diff --git a/Cargo.lock b/Cargo.lock index dfc3087..44c9457 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -482,6 +482,7 @@ dependencies = [ "libc", "rmp-serde", "serde", + "serde_json", "sha2", "tempfile", "toml", diff --git a/Cargo.toml b/Cargo.toml index 713ad33..ee797ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,10 @@ path = "src/bin/idunn-provision.rs" name = "idunn-host" path = "src/bin/idunn-host.rs" +[[bin]] +name = "idunn-launch" +path = "src/bin/idunn-launch.rs" + [lib] name = "idunn_daemon" path = "src/lib.rs" @@ -34,6 +38,7 @@ serde = { version = "1", features = ["derive"] } ed25519-dalek = "2.2" toml = "0.8" uuid = { version = "1", features = ["v4"] } +serde_json = "1" [dev-dependencies] tempfile = "3" diff --git a/README.md b/README.md index 503409e..8c6935c 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,64 @@ idunn status --command the bindings, so `idunn up profile:full-gamecult` brings up everything bound into that profile. +### Steward orders + +A second root unit from the same binary, `idunn-orders` (`idunn orders serve`), +takes typed orders from one steward Mind over `/run/idunn-orders/steward.sock`. +It has its own store, `orders.cc`, and its own binding, `steward.toml`; the +survival daemon takes no order. There is still no command string: + +```bash +idunn order package-install [=version]... --reason TEXT # apt-get update, install named packages, check them +idunn order unit --reason TEXT +idunn order deploy --reason TEXT # the same commit as `idunn up` +idunn order show # status and output tails +idunn order list +``` + +A unit order submits exactly one job, for the unit it names: every verb runs +`systemctl --job-mode=ignore-requirements -- `, so systemd builds a +transaction of one job. It does not start what the unit requires or restart what +is part of it: order the dependency first, and order `postgresql@16-main.service` +rather than the `postgresql.service` wrapper. Targets are not orderable. The job is one; what the unit does afterwards is +not: systemd still runs full transactions of its own for `Restart=` after a +failure, for a `StopWhenUnneeded=` dependency left without a user, and for a +`Conflicts=` partner of a unit that fails, so an ordered stop of a unit +that something restarts does not last, and a start of a `Restart=always` +unit can report failure and be active seconds later. A unit +order may act on any unit except Idunn's own (`idunn-*`) and the minds' own +(`mind-self@*`, `mind-persona@*`), and the units that end or take the host +(`systemd-suspend`, `systemd-hibernate`, `systemd-hybrid-sleep`, +`systemd-suspend-then-hibernate`, `rescue`, `emergency`). That set is compiled +into the binary and read from no configuration. An order is refused, recorded +and posted when the ordered unit, or an alias it resolves from, is one of them; +when it has an emergency action (`SuccessAction=`, `FailureAction=`, +`StartLimitAction=`, `JobTimeoutAction=`) other than `none` or an `OnFailure=` +or `OnSuccess=` unit (not for a reload); or, for a stop or restart, when another +unit is `BoundBy=` it, because systemd stops that unit with it whatever the job +mode; for a start or restart, when it has an `Upholds=` unit, which systemd +starts in a full transaction whatever the job mode; or, for a stop, restart or +reload, when it has a `ConsistsOf=` member, a wrapper such as +`postgresql.service` whose verbs would exit 0 and touch nothing. A unit +`systemctl show` cannot resolve fails the order. Slices, scopes, +targets and globs are not units an order can name. + +A package order installs the packages it names and nothing else: apt runs with +`--no-remove`, each name is passed as an exact-name pattern so `.` and `+` +match themselves, and a last step fails the order unless every named package +is installed. + +An order runs once, after the steward's uid is checked and the instance's +brake check (`mind-control --instance check`) exits 0. The check +that the order's principal and uid are the bound steward's runs again at every +execution, so an order accepted before `steward.toml` changed is refused and +recorded, not run. The socket belongs to the steward user alone (mode 0600); +root may `show` and `list` orders but never issue one. An order that had +started when the service died is recorded as lost and never run again. No order releases a brake. Acceptance and result are posted to +`#operator` as a retried projection of `orders.cc`: a chat outage delays the +posts and not the orders. A refusal names the field and a code, never the value +it refused. + ## Authority **Owner.** Deployment admission, and the continuity of installed bodies. @@ -150,6 +208,79 @@ omits one. Unknown fields are rejected on both sides. A repository that gets compromised can therefore change what is built, but not what it is allowed to touch on the host. +An upstream daemon that does not speak Idunn's runtime contract is deployed +with `idunn-launch` as its executable artifact and the upstream tree as a +required adjacent artifact; the launcher finds the child inside the release that +holds it. The launcher is the unit's process: it binds the +candidate endpoint and answers Idunn's challenge there, renders the child's +config, starts a stateful child only once it holds the write lease, and signs +presence. Members never pass through it: the binding's `[route.member]` makes +Idunn route a second listener straight to the child, which binds the address +Idunn gives it in `GAMECULT_IDUNN_MEMBER_BIND`. + +```toml +# recipe fragment — the upstream tree is built or fetched as an artifact +[service] +executable_artifact = "idunn-launch" +arguments = [ + { kind = "literal", value = "--child" }, + { kind = "literal", value = "upstream/bin/daemon" }, + { kind = "literal", value = "--template" }, + { kind = "literal", value = "config.tmpl" }, + { kind = "literal", value = "--health-path" }, + { kind = "literal", value = "/health" }, + { kind = "literal", value = "--health-contract" }, + { kind = "literal", value = "example.presence-launcher.v1" }, +] +required_environment = [ + "GAMECULT_IDUNN_RUNTIME_BUNDLE", + "GAMECULT_IDUNN_CANDIDATE_BIND", + "GAMECULT_IDUNN_MEMBER_BIND", +] +``` + +A directory artifact is sealed read-only: its files are 0444 and its +directories 0555, whatever mode the build or git gave them. A file the tree +must run, such as the launcher's `upstream/bin/daemon` child, is declared in +`executable_members` and sealed 0555. The members are part of the sealed release +address, so changing the list makes a new release. Idunn refuses the release if a +member is absolute, climbs with `..`, is missing, is a symlink or sits behind +one, or is not a regular file; `executable = true` on a directory output and +`executable_members` on a file output are refused too. + +A tree may hold relative symlinks whose `..` steps all come first and climb no +higher than the artifact's top (`share/terminfo/4/4025ex -> ../t/tek4025ex` is +kept). An absolute link, a `..` after a name, or a climb out of the artifact +refuses the release when it is sealed. Host targets (Windows) record +`executable` and `executable_members` in the receipt and do not apply them. + +```toml +# recipe fragment — the upstream artifact declares the child it launches +[[artifacts]] +id = "upstream" +source_kind = "runner-output" +runner = "build" +source = "dist/upstream" +destination = "upstream" +executable_members = ["bin/daemon"] +``` + +```toml +# binding fragment — members connect to the member endpoint, which Idunn +# forwards to the private port derived from the candidate: member +# private_port_start + (candidate port - route private_port_start) +[route] +stable_endpoint = "tcp://10.77.0.1:8008" +private_port_start = 18831 +private_port_end = 18839 +# ... + +[route.member] +stable_endpoint = "tcp://10.77.0.1:8448" +private_port_start = 18841 +# max_connections = 128 # nginx limit_conn on the member listener (1-65535) +``` + ## Contracts Idunn's typed contracts are published by diff --git a/deploy/idunn-orders.service b/deploy/idunn-orders.service new file mode 100644 index 0000000..f2681c2 --- /dev/null +++ b/deploy/idunn-orders.service @@ -0,0 +1,47 @@ +[Unit] +Description=Idunn steward orders: typed package, unit and deploy orders from the steward +# Deliberately no After= or Requires= on any managed target, Synapse or +# idunn-yggdrasil: orders are how an operator repairs those, so none of them +# may be a condition of this unit starting. Posts are a retried projection and +# wait for Synapse on their own. + +[Service] +Type=simple + +# Root, from the same binary as idunn-yggdrasil, with its own store and socket. +# idunn-yggdrasil keeps its no-imperative-input invariant and its write set: +# no order code runs in `idunn serve`. +# +# The unit name matches idunn-*, which is in the always-refused set, so no +# order can stop or restart this service. +User=root +# No Group= and no UMask=: the service gives its socket to the steward user +# (mode 0600) itself and sets Idunn's own umask for the files it writes, so +# nothing here reaches control.cc. The steward user is the only peer the +# service answers: it checks SO_PEERCRED on every connection. +ExecStart=/usr/local/bin/idunn orders serve \ + --store /var/lib/gamecult/idunn-orders \ + --state-store /var/lib/gamecult/idunn/control.cc \ + --steward-binding /etc/gamecult/idunn/steward.toml \ + --socket /run/idunn-orders/steward.sock +Restart=always +RestartSec=5 +TimeoutStopSec=20 + +RuntimeDirectory=idunn-orders +RuntimeDirectoryMode=0755 + +# The Synapse access token of @idunn, copied from a root-only file, and read +# through $CREDENTIALS_DIRECTORY; the service names it in steward.toml. +LoadCredential=idunn-matrix-token:/etc/gamecult/idunn/matrix-token + +PrivateTmp=yes +ProtectHome=yes +ProtectSystem=strict +# Its own order store, and the control store a deploy order commits a command +# to. Transient units started with systemd-run run in the manager's namespace, +# so packages and units are not limited by this list. +ReadWritePaths=/var/lib/gamecult/idunn-orders /var/lib/gamecult/idunn + +[Install] +WantedBy=multi-user.target diff --git a/deploy/idunn-yggdrasil.service b/deploy/idunn-yggdrasil.service index 5474c67..1c45594 100644 --- a/deploy/idunn-yggdrasil.service +++ b/deploy/idunn-yggdrasil.service @@ -41,6 +41,7 @@ ExecStart=/usr/local/bin/idunn serve \ --source-root /var/lib/gamecult/idunn/sources \ --staging-root /var/lib/gamecult/idunn/staging \ --topology-store /var/lib/gamecult/idunn-projection/topology.cc \ + --incident-store /var/lib/gamecult/idunn-projection/incidents.cc \ --idunn-identity-store /var/lib/gamecult/idunn/idunn-service-identity.cc \ --odin-correlation-store /var/lib/gamecult/odin/topology.cc \ --odin-trust-anchor /etc/gamecult/idunn/odin-topology-anchor.cc \ @@ -76,7 +77,10 @@ ReadWritePaths=/etc/ufw # idunn-projection is deliberately outside Idunn's own 0750 directory: the # topology store is Idunn's *published* surface, and every managed target must # read it to verify its own Expected incarnation against the Idunn anchor. It -# was inside the private root, where no workload could traverse to it. +# was inside the private root, where no workload could traverse to it. The +# published topology store is read lock-free (Odin) and incidents.cc is read by +# Bifrost's reader; managed targets get Expected from their runtime bundle. Only +# the stores are readable: each .lock beside them is Idunn's alone, 0600. ReadWritePaths=/var/lib/gamecult/idunn /var/lib/gamecult/idunn-authority /var/lib/gamecult/idunn-projection ReadWritePaths=/srv/build # Each managed target gets only its bound roots. A target Idunn cannot write is @@ -100,6 +104,11 @@ ReadWritePaths=/etc/gamecult/ghostlight/runtime /var/lib/gamecult/ghostlight-wor ReadWritePaths=/etc/gamecult/streampixels-service/runtime /var/lib/gamecult/streampixels-service /srv/streampixels-idunn/releases/service # StreamPixels web is stateless; it needs only its runtime bundle and releases. ReadWritePaths=/etc/gamecult/streampixels-web/runtime /srv/streampixels-idunn/releases/web +# Vili (motion worker): runtime bundle, state, and releases. Prefixed "-" so a +# target whose roots are not created yet cannot stop Idunn itself from starting +# (Idunn startup never depends on a managed target). A root created after Idunn +# starts is writable only from Idunn's next restart. +ReadWritePaths=-/etc/gamecult/vili/runtime -/var/lib/gamecult/vili -/srv/vili-idunn/releases [Install] WantedBy=multi-user.target diff --git a/docs/authority-map.md b/docs/authority-map.md index dc918fd..c8a8546 100644 --- a/docs/authority-map.md +++ b/docs/authority-map.md @@ -55,7 +55,10 @@ A deployable repository publishes one strict declaration, normally - immutable artifact outputs, optionally pinned to an expected SHA-256; - a constrained launch contract; - one health-contract identity; -- an optional typed state contract with state schema generation and slots; +- an optional typed state contract with state schema generation and slots + (kinds `cultcache-file`, `external-file` and `external-directory`; the + external kinds are process-owned state an external runtime imposes, which + Idunn preserves and fences but never interprets); - provided capabilities, dependency kinds, startup requirements, capacity, and conflicts. @@ -67,6 +70,52 @@ recipe fields. Stateless services declare no state block. Their operator binding therefore has no state root, state-transition policy, or process write-lease record. +## Foreign workloads: idunn-launch + +An upstream daemon that cannot speak Idunn's runtime contract (Synapse, MAS) +runs under `idunn-launch`, the unit's own process. Idunn's two-witness rule is +unchanged: Idunn observes the unit, and the launcher, inside the unit and +supervising the child, signs presence with the provider identity and the +activation key. + +- **Owner:** the launcher owns presence, config rendering and the child's + lifetime inside the unit, and the route answers it signs for that child. + Idunn owns admission, lease, route and continuity, as before. +- **Inputs:** the runtime bundle, the activation credential and provider + identity descriptors, the process write lease, the systemd credentials the + binding names, and the child's health over TCP at the member bind. +- **Outputs:** signed `gamecult.runtime_presence_health.v2` (Warming, Active + and failed to Odin; route observations on the candidate endpoint), one child + process, and a rendered config in the unit's private tmp that is never read + back. +- **Route:** Idunn owns both listeners of the route: the Expected route's + stable endpoint (presence) and, when the binding has `[route.member]`, the + member endpoint, rendered as a second server block in the one nginx fragment + with a `limit_conn` on one per-route zone (`max_connections`, default 128), so + members cannot exhaust the connections the presence block shares with them. + The launcher binds the candidate endpoint and answers Idunn's challenge there, + one connection at a time within a one-second deadline; any other first byte + closes the connection. The answer never waits on the child: it reports the + lifecycle's last probe, and a failed, missing or stale probe reads `degraded`. + The child binds the member bind Idunn sets in + `GAMECULT_IDUNN_MEMBER_BIND`, derived from the candidate port. Member bytes + never touch the launcher. +- **Lease:** with `write_lease_required` the launcher starts the child only + after it holds the lease (bound to this Expected, this activation and one of + its own Warming proofs) and keeps the lease store share-locked while the + child lives. Losing the lease stops the child. +- **Config:** one template from the release is rendered with the closed + placeholder set `{{child.listen}}`, `{{child.listen_host}}`, + `{{child.listen_port}}` (all three the member bind), `{{credential:NAME}}`, + `{{credential_path:NAME}}` and `{{state_root}}`. Any other placeholder + refuses to start. +- **Forbidden:** member traffic through the launcher; the child binding the + candidate endpoint; a child of a lease-requiring target running without a held lease; any presence not signed + by both keys. +- **State kind `external-directory`:** a process-owned directory imposed by an + external runtime contract. Same writer, recovery and startup rules as + `external-file`: it is lease-gated, preserved and fenced, never interpreted. + ## Operator binding The root-admitted operator binding supplies the physical policy a repository @@ -173,6 +222,41 @@ health plus Ready correlation. It changes and observes route membership only afterward. Only after the proxy has adopted the candidate may Idunn clear the promotion fence and drain the incumbent. +A target has two slots: the incumbent (the admitted generation) and a +transaction's candidate. Continuity owns the incumbent slot and keeps the +admitted generation alive; it never stops. A live `DeploymentTransaction` owns +its private candidate before Fencing and takes the target itself at Fencing (a +stop-then-start rollout, at Starting) through one `take_target` CAS, so of two +transactions entering from one view exactly one succeeds. A continuity +restart is itself a transaction and takes the target the same way. + +Phases from Warming through Committing carry deadlines, as do aborts and +drains; an expired deadline never waits silently. A stateless deployment +aborts to its incumbent. A deployment whose candidate was issued a write lease +is left to the operator (`operator-required` incident, the second incident +condition beside `continuity-exhausted`; its subject is the transaction id and +it closes when the record ends or its step goes on). A continuity after +Fencing retries its phase forward on a capped backoff and never stops its live +candidate, since that is the only running copy of the admitted body; the +deadline opens the same incident. The one exception is yielding: a continuity +whose route proof has not succeeded gives way, through its own post-fencing +abort, to a deployment the deployment brake has released and that awaits the +handover; a continuity that has proven its route never yields. The +`ContinuityExhausted` incident (six charged attempts in an hour, +`restarts_page`) pages and gates nothing. When continuity next acts is derived +only from its own record in `TargetSupervision` (schema v2): the attempt +streak, the backoff anchor (written by the dead-and-free pass, the only +writer), and the once-per-admission re-materialization bound. The workload +driver alone answers whether a lost unit was a host reboot +(`WorkloadPort::lost_to_host_reboot`). Operator-facing incidents and +`idunn status` are projections that decide nothing. A boot lift returns a +continuity abort left by an older Idunn, one that had not stopped its running +candidate, to forward retry; if it cannot decide, every continuity abort is +held until a lift succeeds. It is a migration with a deletion line (follow-up +`c2c-delete-boot-lift`). Cut line: cuts 3 and 4 of `reincarnation-map.md` +still owe dependency failures not growing the streak and the dead-generation +route refresh. + The deployment brake gates changes to source, release, configuration, schema, unit, or authority binding. The lifecycle brake separately gates restart of an already-admitted incarnation. Neither target brake may gate Idunn itself or an diff --git a/docs/deploy-provenance-map.md b/docs/deploy-provenance-map.md new file mode 100644 index 0000000..95c0eee --- /dev/null +++ b/docs/deploy-provenance-map.md @@ -0,0 +1,394 @@ +# Deployment provenance: map + +Status: map, Imagination, 2026-10-23 (campaign `idunn-deploy-provenance`, +session `self-2026-10-23-eureka-b`). The ends are in +[`deploy-provenance-target.md`](deploy-provenance-target.md). Questions, +rulings, cut specs and follow-ups live in Eureka's mind; this page holds what +has no kind: body facts, rationale and sequencing. Line numbers in T1-T12 are +at `d74ac8d` (origin/main on 2026-10-23); T4, T9 and T13-T16 are at `fd040f0`, +the head of Hands attempt 1, which revision 2 of the spec builds on. + +## Cut `tree-executable-members` + +Spec `idunn-deploy-provenance:cut_spec:cut-tree-executable-members.r2` +(revision 2, after Soul pass 1 found the real interpreter tree refused). It +serves `motion-pipeline:follow_up:idunn-tree-executable-members` (Vili's +admission waits on it) and will serve gamecult-chat C3 too: the presence +launcher runs a child such as `upstream/bin/daemon` from inside a tree artifact +(README, launcher fragment), and Synapse needs a Python interpreter there. + +### Body facts + +- T1. `ArtifactOutput` (`src/deployment.rs:106-117`) carries one `executable` + bool. `ArtifactReceipt` (`src/deployment_plan.rs:731-737`) copies it and + `validate` (`:740-760`) requires it to equal the declaration. +- T2. `harden_installed_release` (`src/drivers.rs:6848-6855`) builds its + executable set from `root.join(receipt.destination)` only. `harden_root_tree` + (`:6858-6892`) seals every directory 0555 and every file 0444 unless that + exact path is in the set. A directory artifact with `executable = true` is + therefore silently meaningless today: its files are all 0444. Nothing refuses + it. No live recipe declares one (CodexConnector, Ghostlight, Heimdall, Odin, + StreamPixels service and web surveyed): `executable = true` is on files only. +- T3. A tree output is copied by `copy_artifact`/`copy_tree` + (`src/drivers.rs:8053-8126`), which recreate symlinks verbatim. The digest + (`digest_tree`, `:8142-8175`) covers names, file bytes and link targets, not + mode bits. So runner or git exec bits never reach the release: harden is the + only mode authority. +- T4. `validate_release_symlink` (`src/drivers.rs:6897-6916`) keeps a symlink + only if its target is relative with no `..`. It runs only inside + `harden_root_tree`, at install, so a tree with a `..` or absolute link copies, + digests and seals, then fails at `install_release` (Soul s1: copying and + digesting the real tree twice took 160 s before harden refused it). The rule + is sound but too narrow: it refuses the real interpreter tree (T13). The + frozen-source path has a physical resolver (`resolve_frozen_source_symlink`, + `:6617`) that walks the filesystem with a 40-link budget and a memo. Its + first version was exponential (Soul S4-1, recorded in `d562a12`), and it needs + the tree on disk to answer. Revision 2 does not reuse it (T14). +- T5. The sealed release id hashes `(schema, plan_id, artifacts, + external_inputs)` (`src/deployment_plan.rs:1000-1011`), and `plan_id` hashes + the whole plan including `recipe_blob`. A changed member list is therefore + already a new plan and a new release. The receipt must still carry the list, + because `harden_installed_release` reads receipts only; its two callers + (`install_release` `:2788`, `validate_installed_release` `:2822`) re-harden + every installed release on each observation. +- T6. Encoding is `rmp_serde::to_vec`, positional. A trailing field with + `#[serde(default, skip_serializing_if = "BTreeSet::is_empty")]` keeps every + existing receipt's bytes and release id: the plan's `deadlines` field + (`:574-577`) is the precedent. +- T7. `digest_artifact` and `digest_tree` (`:8128-8175`) `fs::read` each file + whole. A torch CPU tree's largest file (`libtorch_cpu.so`, hundreds of MB) + is held in Idunn's memory three times per install (seal, install, every + validate). `hash_frozen_source_tree` (`:6788-6840`) already streams in 64 KiB + chunks (S9). The cut collapses both onto one streaming helper. +- T8. `require_leaf_path` (`src/deployment.rs:1951-1958`) only keeps the + artifact destination a single filename. Members are paths inside that + destination, so it does not change. `require_relative_path` (`:1936-1949`) + with `allow_current = false` already refuses absolute, `..`, root and prefix + components. +- T9. The host actuator re-spells the seal (`src/host.rs:1050-1086`): copy, + mode check, digest, pinned-digest check and receipt, the same steps as + `collect_artifact` (`src/drivers.rs:2305-2326`). Soul s1: deleting its mode + check survives every test. Revision 2 collapses both onto one `seal_artifact` + in `drivers.rs`. Each caller keeps only how it finds the source path. +- T10. The transient unit runs with `ProtectSystem=strict` and the release in + `ReadOnlyPaths` (`src/drivers.rs:3675-3698`), `/tmp` mounted `noexec`, and no + `MemoryDenyWriteExecute`. Executing a 0555 file from the release works. + Shared libraries need no exec bit. +- T11. Live body on Yggdrasil: `/usr/local/bin/idunn` sha256 `44cf8a14...`, + installed 2026-10-20 02:34 CEST, before `6e2a31c` (02:40 CEST). `src/` is + identical between `8760a4f` and `6e2a31c`, so the live code is the + reincarnation c2 merge. `origin/main` is 26 commits ahead (+4194/-112 in + `src/`: the presence launcher and member route). Installing this cut ships all + of them. Idunn's own body changes only through gamecult-ops + `scripts/install-idunn-yggdrasil-release.sh` (staged binaries, unit, + bindings). The stage at `/home/gamecultadmin/idunn-stage` is stale (2026-09-11). +- T13. Link census of the real interpreter tree (Yggdrasil probe of + `cpython-3.12.15+20261021-x86_64-unknown-linux-gnu-install_only.tar.gz`, + sha256 `5ff9f060...77191124`; torch adds no links, as Soul s1 counted 1,049 + after pip): 1,049 links. 0 are absolute, 749 have no `..`, and 300 have all + their `..` components before the first name (every one is + `share/terminfo// -> ..//`, one `..` each). 0 have a `..` + after a name, 0 climb above the tree, 0 dangle and 0 resolve outside it. + Blender 5.2.2 has 79 links, none absolute or with `..` (Soul s1). +- T14. Lexically normalising `/` does not check + containment. The kernel resolves a target one component at a time, so a `..` + after a symlinked name climbs from where that link points, not from where its + name sits. Probe on Yggdrasil: `root/a/s -> ..` and `root/a/L -> s/../x` + normalise to `root` and `root/a/x`, both inside, yet `L` reads the file `x` + beside `root`. `root/a/d -> .` with `root/a/M -> d/../../x` escapes the same + way. Idunn has met this before: verify-transaction-cut F5 (`D -> .`, + `L -> D/D/../../../etc/passwd` passed a lexical check). + The sound lexical rule constrains the target's shape: all its `..` components + come before its first name, and they number no more than the link's depth + below the artifact's top. The climb then starts from the link's own + directory, which the walk reached through real directories only (copy and + digest never descend through a link), so it lands on a real ancestor inside + the artifact. After that the target only descends, and any link it passes + through obeys the same rule. By induction on the kernel's hops (capped at 40, + then ELOOP), every hop stays inside. The check is one pass over the target's + components with no filesystem reads, so its cost is linear in the tree and no + resolver is needed. The shapes it refuses (`s/../x`) occur in neither real + tree (T13). + Prior art, from documentation as known to Imagination (high confidence for Go + and securejoin, medium for the tar details): + - Go `path/filepath.IsLocal` (1.20) is lexical and says so: it does not account + for symlinks. It would admit `s/../x`. Used alone, it is the T14 bug. + - Go `os.Root` (1.24) and `cyphar/filepath-securejoin` (used by runc, + containerd and umoci) resolve each component inside the root when a path is + opened, refusing or clamping an escape. They guard one later access, need + the tree on disk, and check again on every open. Idunn instead certifies a + sealed tree once, so that the kernel's own resolution stays inside on every + later access. + - Zip-slip defences and archive extractors check entry names lexically and + protect against link targets by refusing to write through links + (libarchive's `ARCHIVE_EXTRACT_SECURE_SYMLINKS`, and `SECURE_NODOTDOT` for + names; GNU tar creates such links last). They protect the extraction, not + what a kept link points at afterwards. + - OCI layer extraction keeps link targets verbatim, absolute ones included, + because the runtime resolves them inside the container's root. Idunn runs a + release in the host's mount namespace, so absolute targets stay refused. + - Nix keeps link targets verbatim and tracks dependencies by scanning for + references. The whole store is its root, so it has no per-artifact + containment to compare with. + None of these is a static per-link rule for a tree that runs unconfined. The + T14 shape rule fits that case. +- T15. Host targets: `idunn-host` observes, starts and stops processes on + Windows only (the non-Windows `platform` module in `src/host.rs` bails), and + `host.rs` sets no file modes. On Windows a file runs by its type, not by a + mode bit, so `executable` and `executable_members` have nothing to set there. + The file `executable` flag has always been carried and never applied on host + targets. A recipe does not know which workload kind its binding will choose. +- T16. The release root holds only artifact destinations. Runner workspaces, + where external inputs land, are removed before `SealedRelease::new` + (`src/drivers.rs:2421-2423`). `install_release` (`:2781-2788`) and + `validate_installed_release` (`:2815-2822`) digest every receipt's + destination before calling harden. So a rule in the digest walk runs at seal, + at install and at every validate, over every link harden would see. +- T12. Disk: `/` on Yggdrasil has 1.3 TB free. A tree release costs about three + copies (runner workspace, staging, installed). Nothing collects old releases or + staging (`ghostlight-verse:follow_up:idunn-generation-retention`), so every + Vili deploy leaves about 7 GiB behind. + +### Contract, corrected against the body + +- Any directory artifact may declare `executable_members` (a set of relative + paths). The follow-up limited it to runner-output; that limit protects no + invariant, and the launcher's worktree or fetched trees need the same feature. +- `executable = true` names a file artifact. On a directory output the release + is refused, and `executable_members` on a file output is refused (T2). +- A member is refused at recipe parse if it is empty, absolute, or holds `.`, + `..`, a root or a prefix. At seal it is refused if it is missing, is a + symlink, passes through a symlink or non-directory, or is not a regular file. + Refusal happens before `SealedRelease::new`, so no sealed release can name a + member it cannot seal. +- Harden seals each member 0555 and fails if any declared executable path was + not visited as a regular file. "Every declared executable is sealed 0555, or + the install fails" holds structurally, not by a lookup that can miss. +- Symlinks: a link in a tree artifact is kept when its target is relative, its + `..` components all come before its first name, and they number no more than + the link's depth below the artifact's top (T14). One function, + `check_release_link`, decides this. Only the digest walk calls it, so a bad + link refuses the release at seal, before `SealedRelease::new`, and again at + install and validate (T16). Harden no longer judges links. The climb is + bounded by the artifact's top, not the release root: no live release has a + `..` link (the old rule refused them all), so nothing installed is newly + refused, and each artifact's digest describes a closed tree. +- Host targets carry `executable` and `executable_members` in the receipt, + shape-checked at seal by the shared `seal_artifact`, and do not apply them + (T15). Refusing members there would make a recipe's validity depend on its + binding. +- Vili's trees must + not come from `python -m venv`: its `bin/python` is an absolute link and its + `pyvenv.cfg` names the build workspace. Install packages into the relocatable + interpreter tree instead (`motion-pipeline:follow_up:vili-relocatable-python-tree`). + +### Authority + +Owner: the seal (`seal_artifact`, called by `collect_artifact` for systemd +targets and by `materialize` in `host.rs` for host targets) decides +membership from the recipe and writes it into the receipt. Harden enforces the +receipt and decides nothing. Runner exec bits and git `100755` modes are dead +inputs for release modes. Link containment belongs to `check_release_link`, +which only `digest_tree` calls. `validate_release_symlink` is deleted, and +harden's symlink arm leaves links as they are. + +### Acceptance sizing + +The big-tree job (spec `verification.operator`) needs about 12 GiB of scratch +on Yggdrasil: a 3.5 GiB tree, its staging copy, downloads and pip cache. It +removes it at the end. Revision 2 runs it on the trees as shipped, with the 300 +terminfo links (T13). If a tree holds a link the T14 rule refuses, the job +prints it and stops, and the link is reported to Self, not deleted. + +## Cut `isolation-credential-evidence` + +Spec `idunn-deploy-provenance:cut_spec:cut-cut-isolation-credential-evidence.r1`, +at `112a7a5` (origin/main, live on Yggdrasil since 2026-10-24 11:29 UTC). It +serves Vili's admission (`motion-pipeline:cut_report:cut-vili-admission.h4`, +first deviation): every `idunn up vili` is refused before fencing. + +### Body facts (probed read-only on Yggdrasil, 2026-10-24) + +- I1. `idunn up vili` runs 6 and 7 failed with + `systemd workload isolation properties differ from the Idunn contract`, the + message of `observe_unit` (`src/drivers.rs:2975-3005`). Of its conjuncts, + only one differs on the failed Vili unit: `LoadCredential`. Idunn expects + `""` because Vili has no service credential besides the presence identity + (`service_credential_names` empty); systemd shows `[unprintable]`. Every + other conjunct matches, InaccessiblePaths included (one value across all + five idunn-vili/odin units). +- I2. systemd 257.13 `systemctl show -p LoadCredential` prints + `LoadCredential=[unprintable]` for every unit, whatever the credentials: + `ssh.service` and `systemd-journald.service` (not DynamicUser, zero + LoadCredential), the four failed `idunn-vili-*` units (zero), and the Odin + unit (one). The display form carries no information, so the conjunct was + vacuous whenever credentials existed and always false when none did. Odin + and StreamPixels passed only because each carries a regular credential. +- I3. The exact value is readable: D-Bus + `org.freedesktop.systemd1.Service.LoadCredential` (`busctl get-property`) + answers `a(ss) 0` for the Vili units and ssh, `a(ss) 1 + "ODIN_TOPOLOGY_IDENTITY" ""` for Odin. +- I4. What the workload can actually read is the delivered directory. On the + host, `/run/credentials/` holds a directory only for units with credentials + (getty, journald, Odin, StreamPixels service and web). Inside Odin's MainPID + root, `/run/credentials//` holds exactly `ODIN_TOPOLOGY_IDENTITY` + (`-r--r-----+ root root`). The presence identity is a parent-only `OpenFile` + descriptor, never a credential. +- I5. The rule arrived whole in `1fc0dad` (2026-09-03, "Rebuild Idunn + deployment transaction authority"), with its twin clause in + `validate_launch_observation` (`:3468-3472`, `:3513-3516`); no later commit + touched it. No ruling or finding in any campaign concerns workload isolation + verification; gamecult-chat's foreign-workloads cut touched + `CREDENTIALS_DIRECTORY` reservation, not this check. +- I6. `SystemdWorkloadObservation` is persisted by previous Idunns in both + MessagePack encodings, compact (positional) included + (`persisted_systemd_observations_decode_as_the_systemd_variant`, + `:9915-9930`), so deleting the `load_credential` field shifts every later + slot of a live incumbent's record. The field is also in the drift list + (`:2462`) that `stop` (`:4064-4070`) uses to refuse stopping a changed + workload: changing what the field holds without leaving that list would make + the next Idunn refuse to stop the live Odin and StreamPixels incumbents. +- I7. Fixtures: the verbatim 257.13 `systemctl show` output of base's 27 + properties for a Vili unit (presence identity only) and the Odin unit + (presence identity plus one service credential), captured 2026-10-24 into + this Imagination's scratchpad `fixtures/`. +- I8. The new rule holds for every live systemd incumbent: inside each + MainPID root the delivered entries equal its binding's `secret_files` keys + minus `GAMECULT_RUNTIME_PRESENCE_IDENTITY` (Odin 1, StreamPixels service 11, + StreamPixels web 1; key names compared, no values read). An Idunn restart onto + this cut re-observes them (`:3955-3970`) and passes. + +### Decision + +A property systemd cannot display does not belong in the comparison. The +invariant it reached for, that the workload holds exactly the credentials Idunn +delivered, is observed where it is decided: the entries of +`/run/credentials/` in the MainPID's root equal Idunn's service +credential names, absent directory equal to none. That covers +`LoadCredential`, `SetCredential`, `ImportCredential` and drop-ins alike, and +needs no new program. The D-Bus read (I3) is exact but sees only one of those +properties and adds a second systemd client and parser; rejected. + +### Authority + +Owner: `observe_unit` decides the credential rule, from the delivered +directory. `LoadCredential`'s display form is no longer read or compared; the +`load_credential` field is a retired wire slot, written empty, out of the drift +list, kept only so persisted records decode. Forbidden writers: the +`observe_unit` conjunct and the `validate_launch_observation` clause. Dropping +the slot waits on the next schema revision of the observation. + +### Deploy + +Idunn's body changes only through gamecult-ops +`scripts/install-idunn-yggdrasil-release.sh` (stage `idunn` only; unit and +bindings unchanged). After it, Vili's Hands reruns from step 4 of +`motion-pipeline:cut_report:cut-vili-admission.h4`'s undone list. + +## Cut `runner-failure-evidence` + +Spec `idunn-deploy-provenance:cut_spec:cut-runner-failure-evidence.r1`, at +`804ed75` (origin/main, live on Yggdrasil). It serves the StreamPixels-area +Self: `idunn up vili` (command `up-4866467c-ee98-4a2b-b4a9-a008d7c44acd`, +transaction `tx-a04b68c7-c742-4d7d-8220-8aa08b9febae`, 2026-10-25 00:50-00:55 +UTC) failed in Sealing with `Docker runner exited with exit status: 1: ` and +nobody could tell which recipe step failed. + +### Body facts (source read at `804ed75`; host read-only, 2026-10-25) + +- R1. The runner is `docker run --rm` (`src/drivers.rs:2088-2090`), launched by + `DockerRunnerDriver::docker` (`:2173-2195`) with stdin null and + `Command::output()`: stdout and stderr are each buffered whole in the + daemon's memory, unbounded. On a non-zero exit the error is + `"Docker runner exited with {status}: {stderr}"`; stdout is dropped. The + container is gone the moment it exits, so `docker ps -a` and `docker logs` + hold nothing. +- R2. The empty message: the failing process wrote nothing to stderr, and + stdout, the only stream that carried its report, is discarded at + `:2182-2193`. The same interpolation echoes whatever a step does write to + stderr into `last_error`, `pre_fencing_abort.error` and + `completion.error` (`bounded_detail`, 2048 bytes), and from there into + `idunn up`'s `deployment failed:` line and `idunn status`. +- R3. No step identity reaches the error. `materialize` (`:2368-2391`) loops + over `declaration.steps` and calls `run_in_workspace` with argv only; + `step.id` is used only in the environment precondition. External inputs + (`curl` in the runner, `:2247-2258`) fail through the same helper with the + same anonymous message. +- R4. Which step failed on 2026-10-25, inferred from the failed transaction's + staging root, which survives (926 MiB, + `/var/lib/gamecult/idunn/staging/tx-a04b68c7-.../`): in `.runner-node-build`, + `node_modules/.package-lock.json` was written at 00:51:24 (so + `install-vili-dependencies`, `npm ci`, finished), nothing outside + `node_modules` is newer, and `release/vili` (the next step's output) is + absent; the python workspace holds only fetched inputs. The failing step was + `test-daemon` (`node --test ... scripts/test-vili-daemon.mjs`), whose test + reporter writes to stdout: exactly the stream R1 discards. Inference from + residue, not from output. +- R5. The failure path: `resume_one_transaction` (`src/control_plane.rs:6291-6320`) + sends a pre-fence error to `begin_pre_fencing_abort` (`:9916-9935`), which + persists `bounded_detail(format!("{error:#}"))`. Nothing reaches the journal + on this path: the idunn-yggdrasil journal for 00:45-01:10 holds only + provider-exclusion lines. +- R6. Stores. `control.cc` (root 0640) holds commands, transactions + (`idunn.deployment_transaction.v4`, keys 0-36), generations, supervision and + expiry requests; `ControlSnapshot::read` refuses any other type + (`:3420`). Records must re-encode byte-exactly (`decode_record`, + `:3717-3727`), so a new transaction field is a schema bump and a v4 lift + (`:4100-4110`). Terminal transactions move verbatim to `history.cc` + (`:4366-4396`; 39.8 MB on the host), which status reads for `--command`. + `incidents.cc` lives in the published projection directory and by its own + doc (`src/incident.rs:1-11`) carries no free text and is never `control.cc`, + because a type added there stops continuity for every target on a rollback. +- R7. The daemon runs as root (`User=root`). Runner secrets reach the + container as read-only file mounts named by env (`:2041-2047`), so any text + a step derives from them can land in its output. +- R8. Staging residue: 475 transaction roots, 55 GiB under + `/var/lib/gamecult/idunn/staging`; a failed Sealing leaves its workspaces + because `materialize` returns before `remove_tree_inside`. + +### Decision + +The failure becomes a typed value at the runner, `RunnerStepFailure`: the +subject (recipe step or external input, by id), the runner id, the exit code or +signal, and the last 8 KiB of stdout and of stderr with each stream's total +byte count. Its `Display` carries the subject and status, never output. +Capture streams both pipes through bounded tails, so a verbose build no longer +buffers whole in the daemon. + +The tail lives in a new private store, `runner-evidence.cc`, beside +`control.cc` (mode 0600), one record per transaction +(`idunn.runner_failure.v1`), newest 32 kept by the writer. Not in the +transaction record: that is a v5 bump whose rollback makes every older Idunn +refuse its control store (R6), the scar `incident.rs` already names. Not in +`incidents.cc`: published, and free text is barred there. Not in the journal: +journal readers are a wider set than root, so the journal line names the +transaction, the subject, the status and the store, and no output. + +No redaction. Matching secret bytes would make the daemon read every secret, +and still misses any encoding, prefix or transform of one; a claimed redaction +that leaks is worse than a stated rule. The rule instead is the reader set: the +store's only readers (root) can already read every secret the runner was +given. + +Keeping the failed container (dropping `--rm`) is worse: its output then lives +in Docker's json-file log, unbounded, untyped, outside Idunn's state and +readable by the docker group; containers accumulate with no owner to reap them; +`docker system prune` erases the evidence; and nothing ties a container to the +transaction in `idunn status`. + +### Authority + +Owner: the runner decides what failed and keeps the bounded tail; the engine's +pre-fence failure path is the one writer of `runner-evidence.cc`. +`last_error`, `pre_fencing_abort.error` and `completion.error` keep only the +failure's `Display`. Forbidden writers: the `stderr` interpolation in +`DockerRunnerDriver::docker`. Retention is by count inside the writer, so the +store needs no external owner; staging roots (R8) stay with +`ghostlight-verse:follow_up:idunn-generation-retention`. + +### Deploy + +Idunn's body changes only through gamecult-ops +`scripts/install-idunn-yggdrasil-release.sh` (stage `idunn` only; unit and +bindings unchanged). The next failed Sealing then shows its step and tail on +`sudo idunn status --command `. diff --git a/docs/deploy-provenance-target.md b/docs/deploy-provenance-target.md new file mode 100644 index 0000000..74015c7 --- /dev/null +++ b/docs/deploy-provenance-target.md @@ -0,0 +1,61 @@ +# Deployment provenance instead of an approval gate: target + +Status: target, Self, 2026-10-19 (campaign `idunn-deploy-provenance` in +Eureka's mind). It states the ends. The map will live in +`deploy-provenance-map.md`. + +## Why + +Operator, 2026-10-19: + +> The rule about the deployment brake was instituted by a cautious agent, but +> it doesn't actually suit how GameCult operates. The whole point of Eureka is +> to give the Mind, Persona and Self agency over its own Body. It's requiring +> my approval for every deployment that violates our principles. Yes, we +> certainly want to know which agent approved the deployment and why (this is +> the authority provenance which we might want to add to the transaction), but +> that's not the same as gating every deploy on my approval. + +On the same day a StreamPixels deploy (tx-b40c7240) waited on the brake +because only the root-held operator identity could release it. The agent +running the deploy could neither proceed nor record why it should. + +`F:\Projects\CLAUDE.md` says a deployment brake *may* gate artifact, revision, +configuration, schema, unit or authority changes. It does not say that +operator approval is the gate. + +## Invariants + +1. **No deployment waits on a per-transaction operator approval.** An agent + acting for the Mind, Persona or Self may admit a deployment of a target it + is authorized for. +2. **Every transaction carries typed authority provenance.** That is: who + approved it (agent or instance, and faculty), under which session or + ruling, and why. It is written before Fencing, it is immutable, and + `idunn status` and history show it. It is part of the transaction + document, not a log line. +3. **A brake that survives is a stop, not a gate.** The operator or an + authorized agent can halt deployment of a target, and the halt is itself + provenance-bearing. The deployment brake and the continuity lifecycle brake + stay separate (`F:\Projects\CLAUDE.md` Infrastructure). Neither gates Idunn + itself, and the deployment brake never gates continuity. +4. **Existing invariants hold.** These are `idunn-reincarnation` target r2, + `route-continuity-target.md` 1-7 and `authority-map.md`. + +## Scope + +In: + +- The deployment-brake release path in Idunn and `idunn-provision`. +- The provenance record on the transaction. +- `idunn status` and history projection of the provenance. +- The gamecult-ops runbooks that describe a per-transaction operator release + (`heimdall-discord-launch.md`, `idunn-host-raven.md`, the StreamPixels + runbook). + +Out: + +- `idunn-reincarnation`. This campaign is mapped after that campaign's cut 2 + merges, and coordinates with it rather than bundling. +- Harness permission rules on the workstation, which are the operator's. +- Heimdall's onboarding failures. diff --git a/docs/deployment-authority.md b/docs/deployment-authority.md index da7f028..bdc7efc 100644 --- a/docs/deployment-authority.md +++ b/docs/deployment-authority.md @@ -305,6 +305,45 @@ continuity restart does not fetch a branch, rebuild, rerun a migration, alter the launch contract, or consume the deployment brake. An explicit lifecycle brake may suspend this actuation without changing deployment authority. +Continuity never stops. Six charged attempts in an hour only open the +ContinuityExhausted incident (a page; the wire name stays, nothing is +exhausted) and gate no mint, wait or yield. When the next attempt is due is a +function of continuity's own record: a stored attempt streak, a backoff anchor +written when the pass first finds the admitted generation dead and the target +free, and a capped doubling wait from that anchor (5 s base, 300 s cap). A run +of 600 s or more by the admitted body resets the streak. A generation lost to a +host reboot (the workload driver's evidence, `lost_to_host_reboot`) is +re-materialized once per admission without charge or wait. + +Each target has two slots, the incumbent and a transaction's candidate. The +incumbent is continuity's until a transaction takes the target, at Fencing (at +Starting for a stop-then-start rollout), through one `take_target` CAS; of two +transactions entering from one view exactly one succeeds. Before that, a +deployment owns only its private candidate and the target is free for it +between continuity attempts. + +Phases Warming through Committing, post-fencing aborts and drains all carry +deadlines (the plan's frozen `PhaseDeadlines`; `WARMING_DEADLINE_SECONDS`, and +`PRE_FENCING_ABORT_DEADLINE_SECONDS` for a pre-fencing abort). What an expired +post-fencing deadline does depends on the owner. A deployment whose candidate +was never issued a write lease aborts to the incumbent. One whose candidate was +issued a lease may have written under a new state contract and is left to the +operator. A continuity retries its phase forward on a capped backoff and never +stops its live candidate, because that candidate is the only running copy of +the admitted body. Both operator cases open an `operator-required` incident +(subject: the transaction id) that closes when the record ends or its step goes +on. + +The one yield: a continuity whose route proof has not succeeded gives way to a +deployment the deployment brake has released that awaits the handover of the +target. It begins its own post-fencing abort and the deployment fences what is +admitted. A continuity that has proven its route is serving never yields. + +At boot, a lift returns to forward retry any continuity abort an older Idunn +left over a running candidate. If the lift errs or cannot observe the +candidate, the abort is held, not run. The lift is a migration with a deletion +line (follow-up `c2c-delete-boot-lift`). + An admitted generation keeps its Ready and latest Odin receipts as durable proof of the exact activation at their recorded admission times. Dependency selection reauthenticates those receipts at those times; continuity and route diff --git a/docs/guide.md b/docs/guide.md index f018670..e48bc85 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -44,7 +44,8 @@ Idunn centralizes that lifecycle work: - record every deployment/restart request and result as typed state; - record release targets, deployment artifacts, state migrations, and rollout results as typed state; -- escalate to an operator through Bifrost when human action is needed. +- record operator incidents in its published `incidents.cc`, which Bifrost reads + and delivers. Idunn never waits on delivery. The desired shape is simple: diff --git a/docs/host-actuator.md b/docs/host-actuator.md index f2fb1c7..c595c5d 100644 --- a/docs/host-actuator.md +++ b/docs/host-actuator.md @@ -82,8 +82,9 @@ running a script, and a dead `muninn serve` is seen the tick it dies. 2. No path on yggdrasil can start, stop or observe a host workload except through the actuator session bound to that host's anchor. 3. A host with no attached actuator is a failed observation, which - continuity counts and gives up on after six restarts inside one hour, - exactly like a unit that will not start. + continuity counts as a charged attempt and retries on its capped backoff + (it never gives up; six in an hour only pages), exactly like a unit that + will not start. 4. The actuator cannot be told to run a program the recipe did not declare; `allowed_programs` on the host-native runner is enforced on the host. @@ -96,7 +97,8 @@ Each of these was found by running the pipe against Raven, not by reading. bind time. `rollout.strategy = "stop-then-start"` exists for exactly a route-less host-actuator workload: the admitted incumbent is stopped at Starting, before the candidate starts, and supervision leaves the target - alone from Starting on. A systemd-transient workload may not declare it. + alone from Starting on: the transaction takes the target there (one + `take_target` CAS) rather than at Fencing. A systemd-transient workload may not declare it. - **An unobservable host is silence.** The hub returns a typed `HostUnobservable` for not-attached, detached, reconnected and unanswered. Supervision skips that tick and logs it. Before this, an Idunn restart diff --git a/docs/migration.md b/docs/migration.md index 0c810fc..a78b4a0 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -520,7 +520,7 @@ started, ran, and then died on a file it could not open: | `runtime_root` and every directory above it | traversable by the workload: `o+x`, or group-owned by the state group with `g+x` | the workload is a `DynamicUser` and owns nothing, and `ReadOnlyPaths=` binds the bundle's leaf into the namespace without granting traversal above it. Idunn now checks the whole chain and names the closed directory | | `runtime_root` | setgid, group-owned by the state group (`2750 root:`) | `harden_root_authority_file` requires the write-lease record's group to equal its parent's, and only setgid makes a newly created record inherit it. Without it the record lands `root:root` and the workload cannot read its own lease | | the write-lease record | `root:`, mode `0640` | follows from the setgid parent; the workload reads it through the state group. Idunn writes it, so fixing an existing wrong-group record means deleting it, and that is safe only when no transaction is live | -| `/var/lib/gamecult/idunn-projection` | default ACL `u::rw-,g::r--,o::r--` | Idunn runs with `UMask=027`, so each publish creates the store and its `.lock` at `0640`. The code re-opens them after every write, but that leaves a window; a default ACL sets the mode at creation and closes it | +| `/var/lib/gamecult/idunn-projection` | default ACL `u::rw-,g::r--,o::r--` | Idunn runs with `UMask=027`, so a store is created `0640` and each publish re-opens it `0644`; the default ACL sets the mode at creation and closes that window for stores. The `.lock` is not published and no reader opens it: Idunn creates it `0600` (replacing one found wider, so a descriptor another uid already holds is orphaned), and the ACL never decides its mode | | `/etc/nginx/idunn-stream-routes` | `root:root`, not group-writable | "route authority parent is not canonical root-owned and nonwritable". It was `root:idunn 0775` from the generation where Idunn ran unprivileged and needed to write it as a group member | Two shapes are worth stating plainly because they are easy to get backwards. diff --git a/docs/reincarnation-map.md b/docs/reincarnation-map.md new file mode 100644 index 0000000..4f99aed --- /dev/null +++ b/docs/reincarnation-map.md @@ -0,0 +1,600 @@ +# Reincarnation after reboot: map + +Status: map, Imagination, 2026-10-18 (campaign `idunn-reincarnation`, session +`idunn-reincarnation-2026-10-18`). The ends are in +[`reincarnation-target.md`](reincarnation-target.md). Questions, rulings, the +cut spec and follow-ups live in Eureka's mind; this page holds what has +no kind: body facts, the model page, the authority map's rationale and the +cut sequence. Line numbers are at `74a7008`; `src/` is byte-identical to +`c2477d4`, so the earlier anchors still hold. + +Cut 2 pass, Imagination, 2026-10-19 (same campaign and session): body facts +B13-B18, the cut-2 rows of the model page and the cut-2 parts of sections 3-5 +are anchored at `9a0a6a2`, whose `src/` is byte-identical to `ffb4293` (cut 1's +head). The B1-B12 anchors above stay at `74a7008`. + +Read the in-force set with the rulings-in-force and open-follow-ups recipes +for root `idunn-reincarnation`; the cut in flight is the specs-with-no-report +recipe. The first spec (`cut-c1-durable-stop.r1`) was withdrawn when the +operator ruled that continuity never stops; its replacement is cut +`c1-never-stop`. Cited from `idunn-watchdog`: `exhaustion-close-rule-op` and +`paging-conditions`. + +## 1. Body facts + +Evidence: the `idunn-yggdrasil` journal from 11:40 UTC (94,670 lines), the +per-unit journals of the streampixels-service candidates and of Odin +(`idunn-odin-7cfe5cd1…`), `idunn status` and `idunn status --command ` +on Yggdrasil, `journalctl -u nginx`, and the source at `c2477d4`. All times +UTC, 2026-10-18. + +**B1. The reboot did not stop continuity from re-materializing anything.** +Every streampixels-service continuity attempt minted a fresh transient unit +from the installed release `sha256-df663f0a…`: 54 continuity commands from +`continuity-85887ef2` (11:46) to `continuity-52bbbd66` (19:57), most listed by `idunn status` as +`continuity-… failed systemd unit idunn-streampixels-service-.service +is not running`. A continuity transaction is a full `DeploymentTransaction` +built by `from_continuity` (`control_plane.rs:1678`) from the admitted plan, +sealed release, installed release and Expected. It runs every phase: +Starting, Warming, Fencing, Leasing, AwaitingReady, Routing, Committing. The +absent admitted unit `461c6443…` was never the problem. Each new candidate +was. + +**B2. Every candidate died of a dependency, not of itself.** The +streampixels-service process exits about 30 s after start when Odin does not +admit its initial presence. Unit `2bc02905…` at 11:47:21 logged "StreamPixels +service failed to enter its Idunn incarnation: Error: Odin did not admit the +initial Idunn runtime presence ... Timed out waiting for CultNet RUDP response +after 5000 ms." The same is logged by `42ecf363…` at 13:47:13, `03acf44d…` +at 19:47:14, `45276008…` at 19:49:54 and `599b705b…` at 19:51:18. Odin's +RUDP endpoint is `rudp://10.77.0.1:17871`, which nginx proxies. nginx failed +at 11:46:26 ("bind() to 10.77.0.1:17871 failed (99: Cannot assign requested +address)") and started at 19:46:24. Between 11:46 and 19:44 Odin's journal +shows no presence message from streampixels-service at all: none arrived. +After nginx returned, the first one arrived at 19:49:25 and was rejected: +"runtime presence has no exact current activation and provider anchor". The +attempts minted at about 19:53 and 19:55 got past Fencing and then failed +"the Leasing phase passed its deadline" (FailedAfterFencing, RestoreIncumbent). +Other early failures: "observed write lease is not the current Idunn grant" +(11:46:47, the first attempt after boot), and three "reading process stat +/proc/N/stat: No such file or directory" races. + +**B3. Every attempt was charged at mint.** `record_restart(now)` runs inside +the mint CAS (`control_plane.rs:6156-6157`). Nothing reads a continuity's +outcome back into the log. The only exemption anywhere is `HostUnobservable` +(`:5944`), and it applies to an observation, not to a candidate. So the 6-per- +hour window (`CONTINUITY_RESTART_ATTEMPTS` = 6, window 3,600,000 ms, `:71-72`) +filled at 11:50:52 and refilled every hour as entries slid out. The bursts are +in the journal at :47, :48, :49, :51, :52 and :53 of every hour, 39 times +"Idunn stopped restarting admitted streampixels-service". This answers Q2: +the nginx outage counted against the body's budget, and the place that decides +it is the unconditional charge at `:6156`. + +**B4. The deployment was killed by a window slide, not by a live +continuity.** `up-3ad6b05b` (release `sha256-67e227b2…`, a new release) was +recorded at 20:24:51. It was still before Fencing at 20:46:41, when it was +completed `FailedBeforeFencing` with "admitted incumbent failed before +candidate fencing; deployment yielded to continuity: systemd unit +idunn-streampixels-service-461c6443….service is absent". Two seconds later +continuity minted `tx-0804185a` with the OLD release `df663f0a…`, and it was +admitted. The restart that left the window was minted at about 19:46:40 (its +unit `03acf44d…` started at 19:46:42), exactly one window before the abort. +The mechanism, from the source: + +- `:5805-5821`: a live transaction for the target is the `blocker`. From + Fencing on (or Starting for stop-then-start) supervision skips the target + entirely. +- `:6033-6061`: with a blocker present, continuity never restarts and never + reports. It may abort the blocker, and the predicate for that is + `Deploy && !supervision.restarts_exhausted(now) && phase < Fencing && no + abort && not held` (`:6045-6050`). +- `:6062-6084`: only with no blocker does an exhausted continuity open the + incident and say "The target is free for a deployment to replace it". + +So from 20:24:51 the deployment owned the target (`owns_target_authority`, +phase != Complete, `:1772`). Continuity deferred to it through the blocker +`continue` at `:6060`. The deployment deferred to continuity whenever the +window had a free slot, through `:6046`. "Stopped" existed only as a durable +incident (incidents.cc, a projection) and a report. The ownership decision +read the window. When the window gained a slot, continuity reclaimed the +target with no record that it had resumed. This answers Q3: the two owners are +the live `DeploymentTransaction` (it owns while not Complete) and +continuity-as-window (it owns whenever the window is not full). Continuity +defers to the deployment while any transaction is live. The deployment defers +to continuity while the window has room. The "free for a deployment" message +names the first ownership, and the code acts on the second. + +**B5. Fencing an absent incumbent already works.** `advance_fencing` +(`:6869-6952`) stops the exact incumbent and revokes its lease. The systemd +driver's `stop` returns `Ok(())` for an absent unit (`drivers.rs:4003-4007`). +A deployment that is not aborted therefore replaces an absent incumbent with +nothing to fence, which is target invariant 3's last clause, already true. +`exact_incumbent` (`:7556-7569`) fails "target incumbent changed after +transaction creation" if a continuity commit replaced the generation during +the deployment. That is why a phase-split handover needs a rebase +(`handover-pre-fencing`). + +**B6. The two rejection streams are consequences of a dead admitted +generation, not separate faults.** "preserved admitted streampixels-service +after rejecting topology observation: topology correlation does not bind the +current activation" was logged 46,271 times, from 11:46:49 to exactly +20:46:48, about 5,300 an hour. Ghostlight's 42,482 identical lines ran to +19:56:29. Since 20:47, raven-muninn, newly exhausted, shows the same line +621 times. `refresh_admitted_topology` (`:6395-6457`) authenticates Odin's +correlation against the admitted generation's activation. When that +incarnation is dead and Odin's latest record names another one (a +continuity candidate's, or a stale one), authentication fails with an `Err` +rather than `Ok(false)`, and `:5834-5840` prints it on every pass. The pass +order is what does it: route supervision (`:5822`) and topology refresh +(`:5834`) run before the workload is observed (`:5891`). So a dead +generation is "supervised" every tick. "rejected admitted +streampixels-service route continuity: admitted process write lease is no +longer exact" (3,615, from 20:02:23 to 20:46:48) followed the two +post-fencing aborts at 19:54 and 19:57. Fencing revoked the dead incumbent's +lease, "restore incumbent" cannot re-grant a lease to a process that does not +exist, and `supervise_admitted_route` (`:6250-6255`) then refuses the +admitted generation on every challenge. Both streams stopped the moment +`tx-0804185a` was admitted. This answers Q4: both streams follow from one +model gap, that a dead admitted generation is treated as a live incumbent. +That gap is cut 4. They are not the deadlock. + +**B7. Ghostlight is a separate fault.** Its continuity `tx-039b91e9` is +wedged in a post-fencing abort. It was "lease-not-adopted" at its AwaitingReady +deadline, and the abort withdraws the candidate's lease (`:8895-8928`) before +it stops the candidate (`:8929-8946`). The candidate unit `6c51e35e…` has +been running since 19:55:15 and holds the lease's shared lock, so every +attempt fails "process write lease is held by a shared-lock consumer" +(`drivers.rs:4179/4224`). The transaction owns the target, so continuity is +locked out and the route degraded (53 consecutive failures). This is follow-up +`ghostlight-abort-lease-before-stop` (route-continuity invariant 5), not the +stopped-owner deadlock. + +**B8. Unexplained, recorded.** Idunn's record of Odin's stable route reads +"degraded-since 11:46:31, consecutive-failures 1, next-challenge-at 11:47:01" +about nine hours later. It has not been re-challenged since. This is follow-up +`odin-route-record-stale`. It blocks cut 3. Under `budget-never-stop`, a +failure attributed to a declared dependency must not grow the backoff, and the +only dependency-health record Idunn holds for Odin's stable route is this +one. + +**B9. Luna's weekend commits on this path (Q5).** None of them caused the +deadlock. The yield predicate dates from the route-continuity and watchdog +campaigns, not from the weekend. Where they touch: + +| Commit | On the path? | Status | +|---|---|---| +| `9f00e7a` replay provider receipts at admission time | Yes: the deployment's Sealing compile (`advance_sealing` → `current_ready_provider_tokens`) logged "excluded non-current provider ghostlight / raven-muninn: admitted provider's latest receipt is not Ready at its admission time" at 20:24:53 (`:8588`, `:8609`) | Still live: route-continuity B4 has not landed. It did not block this deployment | +| `29946bb` wait for the stable route listener | Yes: the route proofs during the outage were "Connection refused" on 127.0.0.1:8831-8833 | Kept by route-continuity (honest scope) | +| `8ae00a1` cancel live stateless candidates | No: no cancel ran | Still present (`:2888`); route-continuity S2 owns its deletion | +| `ae8ac45`, `8651906`, `e976d32`, `5e94eb8` identity and launch evidence | Only in `observe` and `stop`; nothing here drifted | Kept | +| `a148802` | No | Reverted (`87fa57d`) | + +The target forbids building on them. Cut 1 touches none of their lines. + +**B10. The stop was the only door a deployment had into a crash loop.** +Source read at `74a7008`. A queued `idunn up` becomes a transaction only in +`freeze_command`, which refuses while any transaction for any of its targets +`blocks_new_target_mutation` (`:5741-5747`); the command just stays queued. +The scheduler tick runs `retire_one_terminal_transaction` (returns early when +it retires), then `resume_one_transaction`, then +`supervise_one_admitted_generation`, then `freeze_one_queued_command` +(`:5414-5423`). Today's doubling wait is anchored at the previous attempt's +**mint** (`next_restart_at`, `:1163-1170`). So once an attempt lasts longer than +its wait, the tick after its transaction is retired mints the next attempt +before the freeze runs, and the queued deployment never finds the target +free. At `c2477d4` that ended when the window filled and continuity stopped. +Delete the stop without opening another door and a crash-looping release can +never be replaced. The door the cut uses is kubelet's own anchor +(`docs/research/restart-budget-prior-art.md` §1.1, the exited container's +`FinishedAt`): the wait runs from the first supervision pass that finds the +target dead **and free**. That leaves at least one wait (5 s or more) with no +live transaction, and the freeze runs every tick. This is reasoning over the +source, not a probe. The cut pins it with a tick-level test. + +**B11. The yield cannot outlive the stop.** With the window term gone from +`:6046`, the yield aborts every pre-fencing Deploy whenever the incumbent is +dead (`:6045-6059`). For a crash-looping release the incumbent is always +dead, so the deployment that would fix it is aborted at the first +supervision pass after it is frozen. That is the 2026-10-18 failure: on that +day the operator's `idunn up` existed to replace a release that would not +start. Ruling `handover-phase-split` already says nothing yields before +Fencing. So cut 1 deletes the whole yield. Continuity keeps deferring to any +live transaction for its target (`:6033`, the `continue`). Cut 2 keeps +the incumbent alive alongside the build. Between them, a dead incumbent stays +down while a pre-fencing deployment over it builds. That is bounded by the +deployment's own deadlines, and the deployment then fences it as nothing +(B5). + +**B12. The doubling function already exists once.** `backoff_wait` +(`:4000-4007`) doubles from a base up to `RESUME_BACKOFF_CEILING_MILLIS`, and +has one caller (`back_off`, `:5621`). Continuity's wait is a second spelling +of the same rule (`5_000 << (live.len() - 1)`, `:1169`). The cut gives +`backoff_wait` a ceiling parameter and deletes the second spelling. +`#[cultcache(key = N, default)]` is how an absent key decodes as its default +(precedent `DeploymentTransaction` key 34, `:1552-1557`). New +`TargetSupervision` keys use it, because every record on Yggdrasil predates +them. + +**B13. Two live transactions for one target make the control store +unreadable.** Source read at `9a0a6a2`. `ControlSnapshot::validate` refuses +"multiple transactions claim current-incarnation authority for one target" +over `owns_target_authority` (`:3245-3254`), and that predicate is just +`phase != Complete` (`:1896-1898`). Every reader of the store goes through it: +the scheduler tick, `idunn status`, cancel, expire. So if supervision minted a +continuity beside a pre-fencing deployment today, the next read would fail +the whole engine. Coexistence needs the identity rule changed first, and that +is the cut's first commit. + +**B14. One predicate answers four questions, and the phase rule lives in +one caller.** `blocks_new_target_mutation` (`:1903-1909`, live or post-commit +cleanup pending) is read by the freeze busy check (`:5934-5942`), the +supervision blocker (`:5998-6001`, a single `.find`), candidate-port +reservation (`:7997`) and status (`:4613`). `owns_target_authority` is read by +validation (`:3249`) and by sibling ordering inside one command (`:3384`). +Neither looks at the phase. The only phase-aware ownership rule is inline in +supervision (`:6002-6012`, owns from Fencing, or from Starting for +stop-then-start). The freeze refusal is why a `profile:` command needs every +target free in one tick (follow-up `profile-deploy-joint-gap`). + +**B15. What a pre-fencing deployment reads about its incumbent.** Warming's +semantic warming check uses the incumbent's lease digest +(`:6904-6905`, `is_semantic_warming` `:9530-9560`: Odin reports the candidate's +one disagreement as the lease the incumbent holds). Route preflight uses the +incumbent's route observation (`:7026-7047`). Isolation is proven against the +incumbent's workload (`:7050-7076`). Each goes through `exact_incumbent` +(`:7768-7781`), which bails "target incumbent changed after transaction +creation" once a continuity commit replaced the generation. Sealing, and +Starting for candidate-then-promote, read no incumbent. Stop-then-start's +Starting stops it (`:6857-6863`), which is already the owner's act. The +Sealing-to-Starting write is a `replace_transaction` at `:6806-6808`, not +`transition` (`:8914-8927`), so the two edges at which a transaction can take +a target are written by two different functions today. + +**B16. The commit already guards the incumbent; nothing guards the +handover.** `advance_committing` expects the admitted envelope it read +(`:7701-7702`, CAS `:7743-7764`), so a deployment can never commit over a +generation that moved under it. Entering Fencing is an unguarded +`replace_transaction` of the transaction alone. Continuity's mint CAS +(`:6371-6400`) guards its command, transaction and supervision record, and no +other transaction. The store accepts read-only expectations: an expected +entry with no replacement only guards (CultLib +`packages/cultcache-rs/src/lib.rs:1327-1368`). One engine holds the store +(`ProcessLock::acquire`, `:5514`), but the CLI writes transactions too: +`cancel` sets a pre-fencing abort on a live deployment (`:3014-3083`). So a +handover decided by one CAS over the target's records is exact, and a tick +order is not. + +**B17. What coexisting candidates share, and what they do not.** Continuity +reuses the admitted plan, so its candidate port is the admitted generation's, +already reserved (`:7988-7992`); a deployment picks another. Topology is keyed +per incarnation (`:8495-8498`): a continuity publishes under the admitted key, +a deployment under its own. No candidate holds a write lease before its own +Leasing, which follows Fencing, so two pre-fencing candidates cannot both +write. Route actuation by a continuity is always `Survival`, never refused +(`:4870-4876`); a deployment's `Forward` cannot starve it. The Odin replay +floor `max_odin_sequence` (`:3399-3435`) spans every transaction of the target +and the admitted cursor; incumbent refresh and candidate warming already share +it during every deployment today, so the replay check (`:9498-9501`) gains no +new exposure. Stop-then-start is declared only by route-less host-actuator +bindings (`deployment.rs:756-764`, `:1245-1249`), and no control-plane test +builds one (grep `StopThenStart` in `src/control_plane.rs`: the definition +site only). It is live on Yggdrasil: raven-muninn declares it +(`git grep stop-then-start` in `gamecult-ops`: +`idunn/yggdrasil/bindings/raven-muninn.toml.in`). So the Starting edge of the +handover carries production traffic, and its engine-level proof is owed +before the closing install (follow-up `stop-first-handover-fixture`). + +**B18. A pre-fencing error is an abort.** `resume_candidate` turns any `Err` +from a pre-fencing step into `begin_pre_fencing_abort` unless it is +`LockContended` (`:5768-5783`). So anything the cut makes a deployment wait on +must return `Ok` (a gate wait, `record_gate_wait` `:8170-8182`), or the wait +becomes the yield that ruling `handover-phase-split` forbids. The same path is +how a superseded continuity ends itself, which is wanted. + +## 2. Model page + +One row per persistent kind the campaign touches. + +| Kind | What names it | Lifecycle | Who decides | +|---|---|---|---| +| Paging log: `TargetSupervision.continuity_restarts` (control.cc, `idunn.target_supervision.v1`, key 3) | key = target | The newest 6 charged attempt times, ascending, read through a sliding 1 h window. Outlives every generation. After cut 1 it is the paging threshold only (`restarts_page`): it opens and holds the ContinuityExhausted incident and is shown in status. It gates no mint, wait or yield | Continuity supervision, pushed in the mint CAS | +| Attempt streak (new, cut 1: key 7, `default`) | target | Charged attempts since the admitted body last ran a sustained run. +1 in each charged mint CAS. Reset to 0 when the dead generation was admitted after the last charged attempt and ran ≥ 600 s before it was found dead. Saturating, and the wait is capped, so it can be unbounded | Continuity supervision only | +| Backoff anchor (new, cut 1: key 8, `default`) | target | Written on the first pass that finds the admitted generation dead and the target free after an attempt. The next attempt is due at anchor + `backoff_wait(5 s, streak - 1, 300 s)`. Cleared by every mint. Stale, and ignored, once older than the admitted generation's `admitted_at` | Continuity supervision only | +| Re-materialization bound (new, cut 1: key 9, `default`) | target | Set by the one attempt that is not a backoff step, when the driver reports a reboot loss. Spent while it is not older than the admitted generation's `admitted_at` | Continuity supervision, from the workload driver's evidence | +| Projection deferral (keys 4-5) | target | Unchanged: a failed demotion defers the next mint 30 s. It spaces a re-materialization too | Continuity supervision | +| Admitted generation (`idunn.admitted_generation`) | key = target, `generation_id` = `generation-` | Replaced atomically by every committed deployment or continuity. A dead incarnation keeps its record until replaced | The committing transaction | +| Incident `ContinuityExhausted` (incidents.cc) | condition + subject (target) | Opened at 6 charged attempts in the hour while the target is dead and free. Closed when the target runs and the window no longer holds 6 (`exhaustion-close-rule-op`). Retired after retention. The name is the wire name and stays, though nothing is exhausted any more | Idunn's incident writer, a notification projection: never read by ownership | +| Transaction and pre-fencing abort (`idunn.deployment_transaction.v4`, history.cc once terminal) | `tx-` and its `command_id` (`up-…` or `continuity-…`). Keyed by transaction, so a continuity and a deployment for one target are two records with no shared key | Sealing … Complete. A pre-fencing abort completes `FailedBeforeFencing`, a post-fencing abort `FailedAfterFencing` with a named recovery. Retired to history. After cut 2, per target at any instant: at most one live Deploy, at most one live Continuity, at most one owner (validated by `ControlSnapshot::validate`, B13) | The scheduler advances it. After cut 1, continuity supervision never aborts a transaction; the transaction's own steps do. After cut 2, a continuity whose target was taken, or whose source generation was replaced, aborts itself (superseded) | +| Target ownership (cut 2: derived, no record) | target | A transaction owns its target from `takes_target_at()` (Fencing, or Starting for stop-then-start) until Complete; a Complete transaction with pending post-commit cleanup drains it. Entered only through the handover CAS (`Engine::take_target`) | Derived from the phase. The handover CAS is the one writer of a phase entry into `takes_target_at()` | +| `incumbent_generation_id` on a transaction (key 9, unchanged format) | the generation's id | Deploy: set at freeze; before the handover it names the generation its incumbent-relative evidence was proven against and is rebased when the admitted generation moves (clearing that evidence); pinned by the handover CAS, exact from then on. Continuity: its source generation, never rebased | Deploy: its own pre-handover rebase step and the handover CAS. Continuity: the mint | +| Unit materialization (systemd transient unit `idunn--`) | the activation's hash | Created by Starting. Lost on a host reboot (`/run/systemd/transient`). A failed unit stays loaded; a cleanly stopped one is collected | The workload driver actuates. Whether a loss was a reboot is the driver's evidence (`WorkloadPort::lost_to_host_reboot`, cut 1) | + +No cell is open. Both forks are ruled (`budget-never-stop`, +`handover-phase-split`). The numbers that `budget-never-stop` left open are +the defaulted ruling `continuity-backoff-defaults`. Cut 2 opens one question, +`fencing-over-owning-continuity` (does a deployment ready to fence wait for a +continuity that already owns the target, or preempt it); the spec is written +for the recommended answer, wait. + +## 3. Authority map: continuity and deployment + +The authority map for cut 1 is in its spec. This section is the +campaign-level map, and cut 1 is one step of it. + +- **Owner.** Exactly one authority actuates a target at each instant. A live + `DeploymentTransaction` owns the target from Fencing on, and a stop-first + rollout owns it from Starting. Before Fencing (`handover-phase-split`), a + deployment owns only its private candidate, and continuity owns the + incumbent slot. Continuity owns the admitted generation's survival and + never stops. When it next acts is a function of its own record: the + streak, the backoff anchor and the re-materialization bound. +- **Inputs.** For continuity: the admitted generation (id, `admitted_at`, + workload, plan), the workload observation and the driver's reboot + evidence, its own record, and whether a live transaction blocks the + target. After cut 3 it also reads Idunn's dependency-health observation. + For a deployment: its own plan, its brake, and at Fencing whatever + generation is admitted at that moment (cut 2). +- **Outputs.** Continuity: mints, either charged or the one re-materialization; + backoff anchors; deferrals; and, as notifications only, the + ContinuityExhausted incident and its once-report. Deployment: a new admitted + generation, or an abort that names its recovery. +- **Derived state.** `restarts_page(now)` (6 charged attempts in the hour) is + derived from the paging log and is notification-only: incident open and + close, and status. The next attempt's due time is derived from the + anchor and the streak. The incident and `idunn status` are projections. +- **Forbidden writers.** The paging window gates no mint, no wait and no + yield. Continuity supervision never aborts a deployment: + `begin_pre_fencing_abort` has no caller in it. No deployment path writes + continuity's fields. The incident store decides nothing. No stopped state + exists: no field, and no branch that ends a pass because a count is + reached. +- **Shared paths.** A crash restart, a restart after a sustained run, and a + reboot re-materialization all mint through the one CAS (`:6158-6183`). The + dead-and-free pass is the only writer of the backoff anchor. Every + deployment, operator or scheduled, enters through `freeze_command` and + advances through `advance_transaction`. +- **Deletion line.** Cut 1 deletes the yield (`:6034-6059`), the + exhaustion `continue` and its "free for a deployment" text (`:6062-6085`), + and the mint-anchored wait (`:1160-1184`). Cut 2 (anchors at `9a0a6a2`) + deletes `owns_target_authority` and `blocks_new_target_mutation` + (`:1894-1909`), the inline phase rule and the single-blocker `.find` in + supervision (`:5998-6012`), the blocker `continue` (`:6229-6233`), the + freeze busy check (`:5934-5942`), the one-live-transaction-per-target + validation (`:3245-3254`), and the "target incumbent changed after + transaction creation" text (`:7779`). + +### Cut 2: the two slots and the handover + +Each target has one **incumbent slot**: its `AdmittedGeneration`, the body +running it, its write lease and its stable-route membership. Each live +transaction has its own **candidate slot**: its incarnation key, activation, +candidate port and unit. + +- **Owner.** Before any transaction takes the target, continuity supervision + owns the incumbent slot: it alone decides to mint a restart. From the + handover CAS until Complete, the one transaction that took the target owns + it, fencing, leasing, routing and committing. Its commit hands the slot + back to supervision as a new generation. A transaction owns its own + candidate slot from mint or freeze until it is terminal. A pre-handover + deployment owns nothing else. +- **The handover.** `Engine::take_target` is the only writer of a phase entry + into `takes_target_at()` (Fencing, or Starting for stop-then-start), for + both kinds. It is one CAS. It expects the transaction's own envelope, the + target's admitted envelope (the one `incumbent_generation_id` names, or + absent), and every other non-terminal transaction of the target, each + unchanged, and it writes only the transaction. Of two transactions + entering from one view, exactly one CAS succeeds. The loser's next step + re-reads: a deployment gate-waits for the owner, and a continuity aborts + itself as superseded. +- **Inputs.** Continuity: as cut 1, plus whether any transaction owns or + drains the target, or a continuity is live for it (the mint gate). A + pre-handover deployment: its plan, its brake, and the admitted generation as + it is now, which it rebases onto. The handover: the target's records above. +- **Outputs.** The handover writes the new phase. A rebase rewrites + `incumbent_generation_id` and clears the evidence proven against the old + incumbent: `warming`, `route_preflight`, `isolation`. A superseded continuity + writes its own pre-fencing abort. +- **Derived state.** `owns_target()` is derived from the phase and + `takes_target_at()`. `drains_incumbent()` is derived from the post-commit + cleanup. The freeze refusal is derived from "a live Deploy for this + target". `incumbent_generation_id` on a pre-handover deployment is the + generation its evidence was proven against, not a pin. +- **Forbidden writers.** No path enters `takes_target_at()` except + `take_target`. Continuity supervision aborts nothing, and mints nothing + while another transaction owns or drains the target or a continuity is + live. A deployment never aborts because another transaction owns the + target: it gate-waits (B18). A continuity is never rebased. Freeze reads no + continuity state. No new persistent record decides ownership: it is the + phase. +- **Shared paths.** Candidate-then-promote's Warming-to-Fencing and + stop-then-start's Sealing-to-Starting go through one phase-entry seam, + which calls `take_target` exactly when the entered phase is + `takes_target_at()`. Deployments and continuities use the same CAS. A + single-target and a `profile:` command use the same freeze rule. + +Demotions, named: `incumbent_generation_id` is no longer a pin before the +handover; it is derived from the admitted generation by the rebase. +`blocks_new_target_mutation` is no longer an owner; it splits into +`is_live`, `owns_target` and `drains_incumbent`, each read by the consumer +that asks that question. The freeze busy check is no longer a +target-ownership gate; it guards only one live deployment per target. + +Demotions, named: `restarts_exhausted` is no longer an owner of anything. It +is renamed `restarts_page`, and it opens and holds the incident. The +ContinuityExhausted incident was never read as "stopped", and there is now no +stopped state for it to stand for. The "free for a deployment" report is +deleted: every target is free for a deployment between attempts (B10). + +## 4. Rationale + +The fault is a split brain, not a missing rule. Continuity's "I have stopped" +was written in two places, a durable incident and a log message, and neither +was what the ownership decision read: the yield re-derived "stopped" from a +clock. The withdrawn first spec moved "stopped" into control state. The +operator's ruling is better, and it is also smaller: there is no stopped +state. The window it was derived from becomes a pager. Every case the stop +existed for is then handled by an owner that already exists. A broken release +is replaced by a deployment, which gets in through the gap after each +attempt (B10) and is never yielded (B11). A host-dependency outage costs one +attempt every five minutes or so, and pages. + +Why the wait is anchored at "found dead and free" rather than at the mint: +the anchor is the door. A mint-anchored wait can be swallowed by a long +attempt, and then no tick ever sees the target free (B10). Kubelet anchors +CrashLoopBackOff on `FinishedAt` for the same reason, so this is the +established spelling, not a new one. A streak is stored, not recomputed from +the paging log. The log holds only six entries and slides by the hour, while +the streak must outlive both. Reading the backoff from the window count was +today's spelling, and it is exactly the mutant the cut's tests must kill. + +The reset, "admitted after the last charged attempt and ran at least 600 s +before it was found dead", needs no new writer. Supervision observes the +admitted generation on every pass, so the first pass that finds it dead is +within one poll of the death. A death hidden behind a live transaction or an +unobservable host overstates the run, and so resets early. That is the safe +side, because it shortens one wait. + +The reboot invariant needs evidence of what happened, not a guess. "Unit +absent" alone is not a reboot, because a cleanly stopped transient unit is +collected too. "Host booted after admission" alone is not a loss, because +the unit might have been started this boot. The systemd driver owns both +facts (a `systemctl show` that returns nothing, and the `/proc/stat` btime). +So it answers one question, `lost_to_host_reboot`, and supervision keeps the +budget. The once-per-admission bound is still needed under never-stop. A +failing re-materialization leaves the same admitted generation, unit absent, +booted after admission. Without the bound it would repeat with no step, no +wait and no page, every tick. The clock comparison is safe: admission +happens at least one full transaction, tens of seconds, after boot. + +What cut 1 deliberately does not fix. A dependency failure still grows the +streak (cut 3). A dead incumbent stays down while a pre-fencing deployment +builds over it (cut 2). A dead generation is still route-supervised and +topology-refreshed every tick (cut 4). Paging residual: the cap gives 3600 / +(300 + D) attempts an hour, where D is the attempt's duration, so a crash +loop whose attempts last longer than 300 s settles under 6 an hour. Its +incident opens in the first hour's burst and stays open while the release +never runs, but a release that runs for one pass after each restart can see +it close and not reopen. That is the operator's threshold, applied as ruled. +It is recorded here, not changed. + +Replayed against 2026-10-18 with only cut 1. The reboot loss at 11:46 is +re-materialized at once, with no step. It fails on the nginx outage. The +next attempt resets the streak, because the generation had run for hours, +and then waits 5, 10, 20, 40, 80, 160 s, and 300 s after that. The incident +opens within the first hour and stays open (the release never runs), so the +operator is paged during the outage, not at 20:24. The `idunn up` at 20:24 +freezes in the next gap. Continuity defers to it, nothing aborts it, and it +fences the absent incumbent as nothing and admits the new release. + +Why cut 2 is shaped this way. The handover is the phase, guarded by one CAS +over the target's records, rather than a new per-target "owner" record. A +record would be a second spelling of what the phase already says, and it +would need its own lifecycle, crash repair and migration. A CAS that reads +every other live transaction of the target is enough because a newly minted +or frozen transaction starts at Sealing and cannot own anything (B16). +The rebase clears exactly the evidence that names an incumbent (B15): warming +names its lease, preflight its route and isolation its workload. Keeping any +of them would let the deployment fence one generation with proofs about +another. A deployment waits in Warming while another transaction owns the +target, because Warming's steps read the incumbent that owner is fencing, and +a failed pre-fencing step is an abort (B18). Sealing and Starting keep +running, so the long part, the build, still overlaps. A continuity that loses +the target is aborted, not parked: it restarts a generation that is about to +be replaced, and while it lives it blocks continuity of the generation that +replaces it. + +Prior art on question `fencing-over-owning-continuity` (wait for a +continuity that already owns the target, or preempt it), from +`docs/research/restart-budget-prior-art.md` at `74a7008`. It leans the other +way from the recommendation, and the recommendation says why. Kubernetes' +rolling update scales old ReplicaSets down unhealthy pods first, so that +"unhealthy replicas will block deployment" does not happen (§1.4, +`rolling.go`). That is preemption of a failing old replica. Nomad separates +the two by layer: the client restarts old allocations under their own policy, +and the server reconciler places the new ones (§2.3). Neither waits on an +old replica's restart. But deleting a pod is the kubelet's ordinary teardown. +In Idunn, preempting a continuity past its own Fencing means a deployment +driving another transaction's post-fencing abort. That abort revokes a lease +and restores a projection, and it is the path that wedged Ghostlight (B7). +Waiting is bounded by the owner's own phase deadlines (`PhaseDeadline`, +`:938-960`) and adds no new abort path. Recommendation: wait, at medium +confidence. Preempt is the better answer once post-fencing aborts are known +not to wedge, which is follow-up `ghostlight-abort-lease-before-stop`. + +Residuals, recorded and not fixed. A wedged post-fencing abort of a +continuity (B7, follow-up `ghostlight-abort-lease-before-stop`) holds a +waiting deployment as long as it holds the target. A continuity superseded +after its mint stays charged; that lengthens one wait for a generation that +is being replaced. Stop-then-start has no control-plane fixture (B17), so in +cut 2 its edge is pinned structurally (one seam, one call) and by a unit test +of the phase mapping, not by an engine run. raven-muninn uses it, so the +engine test is owed before cut 5. + +## 5. Cut sequence + +1. **c1-never-stop**: spec in the mind (specs-with-no-report recipe). Depends + on nothing. +2. **c2-phase-split** (`handover-phase-split`): spec + `cut-c2-phase-split.r1` in the mind, branch `idunn/reincarnation-c2` from + `9a0a6a2`. Continuity restarts the incumbent while a pre-fencing + deployment builds. The ownership predicates split (B13, B14). The handover + is one CAS at `takes_target_at()`. A pre-handover deployment rebases onto + whatever is admitted, and a superseded continuity aborts itself. Freeze + refuses only a second live deployment for a target, which dissolves + follow-up `profile-deploy-joint-gap`. It folds in the test-only follow-up + `c1-sustained-reset-test-reach`. Open question: + `fencing-over-owning-continuity`. Depends on cut 1 (merged at `1da150a`). +3. **Dependency failures do not grow the backoff** + (`budget-never-stop`): an attempt that failed because of a declared + dependency (Odin's stable route, the target's own stable route + listener), as Idunn itself observed it, leaves the streak unchanged. + Waits on follow-up `odin-route-record-stale`, because the record it + would read is not trustworthy, and on cut 1. +4. **A dead generation is not supervised as alive** (target invariant 5): + observe the workload before route supervision and topology refresh, so + that a dead admitted generation gets no topology authentication and no + lease-exactness challenge. Its non-binding correlation is reported once, + not every tick. Waits on cut 1, because it rewrites the same function. +5. **Closing proof on Yggdrasil**: an operator-approved install of the merged + cuts. streampixels-service survives the install, and `idunn status` is + checked before and after. Waits on 1-4 and on follow-up + `stop-first-handover-fixture` (raven-muninn is stop-then-start, B17). + +Ops, out of scope: nginx boot order (`gamecult-ops`), and retention of the +roughly 130 dead transient units. + +## Installed + +Cut 1 (`1da150a`, merge of `idunn/reincarnation` at `ffb4293`) was installed on +Yggdrasil on 2026-10-19 at 08:34:45 UTC with the operator's OK. The binary is +`/usr/local/bin/idunn` sha256 `f0e58308…`, built in `/srv/build/idunn-1da150a` +(`cargo test --lib`: 414 passed). The pre-install store backup is +`/root/idunn-store-backup-*-pre-reincarnation-c1`. Restoring it means restoring +its stores and `idunn.running` together, because the cb60965 binary cannot read +`idunn.target_supervision.v2`. At first open Idunn logged +"migrated 5 control record(s) … idunn.target_supervision.v2". streampixels-service +kept running through the restart (unit 73ca40ba, PID 968752, NRestarts 0). The +catalog returned 200, and 4443/udp was listening. + +## Installed: cut 2 + +Cuts c2-phase-split, c2b-owner-deadlines, c2c-post-fence-forward and +c2d-continuity-yields (branch tip `8760a4f`) were installed on Yggdrasil on +2026-10-20 at 00:34:06 UTC, with the operator's OK. + +- **The hotfix before it.** `ec45906` (`bounded_detail`) was installed on its + own on 2026-10-19 at 17:32 UTC. It unwedged tx-f978ab86. +- **Build.** `/srv/build/idunn-8760a4f`; `cargo test --lib` passed 471. +- **Backup.** `/root/idunn-store-backup-*-pre-reincarnation-c2` holds the + stores, `incidents.cc` and the binaries. A rollback must restore them + together: older binaries refuse two live transactions on one target, and + they refuse the `operator-required` incident condition. +- **First boot.** The boot lift returned exactly tx-039b91e9 (ghostlight) and + tx-4eaed669 (odin) to forward retry, as the Soul install gate predicted. + Odin's continuity, wedged in Routing since the 2026-10-18 reboot, committed + 4 s later. Generation tx-4eaed669 is admitted, its route is healthy, and the + same PID 2571 still runs. Ghostlight's continuity is retrying forward in + AwaitingReady. Its daemon's readiness is a ghostlight-side fault + (`Permission denied` reading its session-wrapping key). +- **Live processes.** Every live MainPID was the same before and after the + install. The streampixels catalog returned 200, and 4443/udp was listening. diff --git a/docs/reincarnation-target.md b/docs/reincarnation-target.md new file mode 100644 index 0000000..72e50f7 --- /dev/null +++ b/docs/reincarnation-target.md @@ -0,0 +1,95 @@ +# Reincarnation after reboot: target + +Status: target, Self, 2026-10-18 (campaign `idunn-reincarnation` in Eureka's +mind). It states the ends. The body facts, model page and cut sequence will +live in `reincarnation-map.md`. + +## Why + +Yggdrasil rebooted at 11:46 UTC on 2026-10-18. Admitted bodies run as +transient systemd units under `/run/systemd/transient`, and the reboot wiped +them all. nginx failed at boot, because it bound `10.77.0.1:17871` before +`wg-gamecult` carried that address. Every stable route, Idunn's own +`127.0.0.1:8831-8833` included, stayed down until a manual +`systemctl restart nginx` at about 19:46. + +What Idunn did: + +- From 11:50 onward continuity spent streampixels-service's whole restart + budget (6 per hour) and logged "The target is free for a deployment to + replace it: systemd unit … is absent". The log repeated in hourly bursts as + the window slid. +- When nginx returned, ghostlight and streampixels-web were re-incarnated, and + streampixels-service was not. +- An operator-approved `idunn up streampixels-service` (tx-e69f5402) sat in + Sealing for about 15 minutes. It then failed with "admitted incumbent failed + before candidate fencing; deployment yielded to continuity". Continuity had + freed the target for a deployment. The deployment handed it back to + continuity. Neither actuated. +- At 20:46:43 continuity re-incarnated the service under generation + tx-0804185a, once the window had slid to 4/6. So the outage was not ended by + the deployment the operator approved. The restart window bounded it, a full + hour after its cause had cleared. +- Throughout, several times a second: "preserved admitted … after rejecting + topology observation: topology correlation does not bind the current + activation", and "rejected admitted streampixels-service route continuity: + admitted process write lease is no longer exact". + +## Invariants + +1. **A reboot is not a fault of the body.** If an admitted body's unit is + absent because the host lost its transient units, continuity + re-materializes it from the installed body. It needs no deployment, no + operator, and no part of the restart budget for that absence. +2. **The restart budget measures the body.** A failure caused by a shared host + dependency, such as nginx, the stable route's listener or Odin, never + charges a target's restart budget. A dependency outage can never exhaust + the budget of every target behind it. +3. **Exactly one authority owns each slot at every instant, and nothing + waits on an owner that has stopped.** Continuity never stops (operator, + 2026-10-18: capped backoff, and the 6 per hour count only pages). It never + aborts a deployment. Before Fencing a deployment owns only its candidate, + and continuity owns the incumbent slot. At Fencing the deployment takes the + target and fences whatever is admitted then; an absent incumbent fences as + nothing. Every continuity attempt leaves a free gap in which a queued + deployment can be admitted. There is no reachable state in which + continuity waits for a deployment while that deployment waits for + continuity. A negative test proves it. +4. **Continuity is daemon survival; deployment is change** (`F:\Projects\CLAUDE.md`). + Restarting the admitted installed body never needs a deployment and is + never gated by a deployment brake. A deployment brake gates only artifact, + revision, configuration, schema, unit or authority changes. +5. **Rejections of stale observations are not a log stream.** An observation + that cannot bind the current activation is either resolved by the owner + that can refresh it, or reported once as an incident. It is never + re-rejected several times a second. +6. **Existing invariants hold.** `route-continuity-target.md` 1 to 7 (in + particular no actuation from observation, bounded backoff, every + post-fencing phase ends) and `authority-map.md` (Idunn's own recovery + depends on no managed target). + +## Scope + +In: + +- Idunn's continuity of admitted bodies across a reboot: absent units, the + restart budget and what charges it. +- The handover between continuity and deployment, in both directions. +- Classifying the topology-correlation and write-lease rejections: the same + fault, or a separate one with its own cut. +- Re-admitting streampixels-service on Yggdrasil. It is running again under + tx-0804185a, and the closing proof checks that it survives the fix's + install. + +Out, and not consumers of this change: + +- **nginx's boot order on Yggdrasil.** It is host configuration, owned by + `gamecult-ops`. It gets its own operator-approved change and record, not an + Idunn cut. +- **Ghostlight's degraded route** (53 consecutive failures, incident open) is + in scope only if the map shows the same fault. Otherwise it is a follow-up. +- **Heimdall's onboarding failures and its environment.** +- **Retention of the ~130 dead transient units** on Yggdrasil. It is an ops + follow-up already recorded under route continuity. +- **Luna's 2026-09-27 weekend patches** are not foundation to build on. Where + this map meets them, it cuts them. diff --git a/docs/research/restart-budget-prior-art.md b/docs/research/restart-budget-prior-art.md new file mode 100644 index 0000000..a320c42 --- /dev/null +++ b/docs/research/restart-budget-prior-art.md @@ -0,0 +1,124 @@ +# Restart-budget and handover prior art (Eyes facts pass) + +Serves two operator questions in Eureka campaign `idunn-reincarnation` (session idunn-reincarnation-2026-10-18): + +- `idunn-reincarnation:question:budget-charge` (what charges a target's continuity restart budget) +- `idunn-reincarnation:question:handover-pre-fencing` (who acts when the incumbent dies while a deployment is before Fencing) + +Facts and evidence pointers only. No recommendation. Gathered 2026-10-18 from official docs and upstream source (main branches at fetch time). Quotes are under 15 words. "Not found" means the primary source read does not state it, not that it is false. + +## 1. Kubernetes + +### 1.1 CrashLoopBackOff schedule, cap, reset (kubelet) + +- Docs, pod lifecycle, "Container restarts": backoff is "exponential backoff delay (10s, 20s, 40s, ...)", "capped at 300 seconds (5 minutes)". Reset: after the container has run "for 10 minutes without any problems". https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ +- Source pkg/kubelet/kubelet.go (https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/kubelet.go): `initialCrashLoopBackOff = time.Second * 10`; max is `MaxCrashLoopBackOff = v1beta1.MaxContainerBackOff` (value lives in the kubelet config API package, not re-read; docs say 300 s). Under alpha gate `ReduceDefaultCrashLoopBackOffDecay`: initial 1 s, max 60 s. Gate `KubeletCrashLoopBackOffMax` lets the config set the max (`CrashLoopBackOff.MaxContainerRestartPeriod`). +- Reset rule in code: `klet.crashLoopBackOff.HasExpiredFunc` returns `eventTime.Sub(lastUpdate) > 600*time.Second` (kubelet.go ~line 1102). I.e. the entry resets when more than 600 s have passed since the entry was last updated (last restart event). The client-go default is 2*maxDuration; the kubelet overrides it. +- Growth: client-go `Backoff.Next` doubles (`delay := entry.backoff * 2`) and caps with `min(delay, p.maxDuration)` (https://github.com/kubernetes/client-go/blob/master/util/flowcontrol/backoff.go). +- Window anchor: the exited container's `FinishedAt`; keyed per pod+container; event text "Back-off restarting failed container" (pkg/kubelet/kuberuntime/kuberuntime_manager.go, `doBackOff`). +- The backoff table is an in-memory object created in `NewMainKubelet` (`flowcontrol.NewBackOff(base, boMax)`); no persistence found in the code read. Not found: any doc statement that a node reboot does or does not reset or count toward it. Docs only say restart policy "only applies to restarts that make replacement containers inside the same Pod and running on the same node." Docs on kubelet restarts: pods "continue to run even during the restart" (kubelet restart, not node reboot). + +### 1.2 Probes + +Docs (pod lifecycle "Container probes"; https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/): + +- Liveness: past the failure threshold "the kubelet restarts that container" (per pod restartPolicy). +- Readiness: the EndpointSlice controller "removes the Pod's IP address from the EndpointSlices"; the container is not restarted. +- Startup: until it succeeds "Kubernetes does not execute liveness or readiness probes"; on failure "the kubelet kills the container", subject to restart policy. +- Backoff keys on a container having exited (`doBackOff` reads the last exited container), so crash, liveness kill and startup-probe kill all reach it; readiness failure leaves the container running and does not. The docs do not describe backoff per probe type. + +### 1.3 Dependency down + +Docs, same page, "Dependencies and init containers": init containers "run only once (if successful), during Pod startup"; failed init containers are restarted by the kubelet when pod restartPolicy is OnFailure or Always. Readiness gates: pod is ready only when all `readinessGates` conditions are True. No general documented rule for a main container whose external dependency is down: an app that exits is restarted under backoff; an app that stays up and fails readiness is only removed from endpoints. + +### 1.4 RollingUpdate and the old ReplicaSet + +- Deployment docs (https://kubernetes.io/docs/concepts/workloads/controllers/deployment/): "Once new Pods are ready, old ReplicaSet can be scaled down further"; at the end "all old ReplicaSets is scaled to 0". maxUnavailable and maxSurge default 25%. progressDeadlineSeconds default 600; when exceeded, "Kubernetes takes no action on a stalled Deployment" beyond the Progressing=False condition. +- ReplicaSet docs: it fulfills its purpose by "creating and deleting Pods as needed to reach the desired number." https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ +- Deployment controller source, pkg/controller/deployment/rolling.go (https://github.com/kubernetes/kubernetes/blob/master/pkg/controller/deployment/rolling.go): `reconcileOldReplicaSets` computes `minAvailable = replicas - maxUnavailable`, runs `cleanupUnhealthyReplicas` first (comment: "Clean up unhealthy replicas first, otherwise unhealthy replicas will block deployment"), then `scaleDownOldReplicaSetsForRollingUpdate`. The Deployment controller decides old-RS replica counts during rollout, scaling unavailable ones down first. +- Layering: restarts of a dying container are done by the kubelet (restartPolicy with backoff), not the ReplicaSet. The ReplicaSet recreates a pod only when the pod object is gone or terminal. Not found: a statement in those words that the old ReplicaSet keeps recreating its dying pods until scaled down. + +## 2. Nomad + +### 2.1 restart block (https://developer.hashicorp.com/nomad/docs/job-specification/restart) + +- `attempts` restarts allowed within `interval`; interval "starts when the first task starts". Defaults: service/system attempts 2, interval 30m, delay 15s, mode fail; batch attempts 3, interval 24h, delay 15s, mode fail. Delay has up to 25% jitter. +- `mode = "fail"` (default): after attempts are used the allocation is marked failed and goes to `reschedule`. `mode = "delay"`: "The task waits another full interval before restarting"; the allocation is never failed by restarts. +- Source client/allocrunner/taskrunner/restarts/restarts.go (https://github.com/hashicorp/nomad/blob/main/client/allocrunner/taskrunner/restarts/restarts.go): in-memory `RestartTracker` with `count` and `startTime` set at construction (`time.Now()`); a new interval resets count; `count > Attempts` returns TaskNotRestarting in fail mode, or TaskRestarting with delay to interval end in delay mode. Constructed in `NewTaskRunner` (task_runner.go ~line 468) from the alloc's job type and the task group's policy. `SetKilled` returns TaskKilled without counting. A restart triggered with failure=false (operator/API) returns TaskRestarting without consulting the policy. Unrecoverable start errors return TaskNotRestarting. +- Client restart: restarts.go keeps no persisted counter. `TaskRunner.Restore` (task_runner.go ~line 1244) restores local state and the driver handle (`RecoverTask`); if the handle cannot be restored for a non-system job the task "will not run until server is contacted" (`waitOnServers`). The tracker is rebuilt at task-runner construction. Persisted `TaskState` carries restart counts separately; `GetState` does not read them. + +### 2.2 reschedule block (https://developer.hashicorp.com/nomad/docs/job-specification/reschedule) + +- Applies when an allocation fails to set up or its tasks fail more than `restart` allows. Drains trigger migrations and lost nodes trigger replacements, so reschedule is ignored in both cases. Not applied to system/sysbatch. +- Service defaults: `delay_function = exponential`, delay starting 30s, `max_delay` 1h, `unlimited = true`. Batch defaults: 1 attempt per 24h, constant 5s, unlimited false. `delay_function`: constant, exponential (doubling), fibonacci. `max_delay` ignored for constant. Minimum delay 1 s; docs warn very low delay with `unlimited = true` can cause runaway rescheduling. +- With `unlimited = true`: "The `progress_deadline` in the `update` block still applies". + +### 2.3 Deployments and restarts of old allocations + +- Update block (https://developer.hashicorp.com/nomad/docs/job-specification/update): `max_parallel` 1, `min_healthy_time` 10s, `healthy_deadline` 5m, `progress_deadline` 10m ("the deployment is marked as failed"), `auto_revert` false, `canary` 0. The page says nothing on restarts of old allocations. Not found: docs on deployment/restart interaction. +- Restart is client-local ("Restarts happen on the client that is running the task"); deployment is server-side. Reconciler source (scheduler/reconciler/reconcile_cluster.go, `placeAllocs`): when the deployment is not place-ready (paused/failed/canarying), rescheduling of failed allocs still proceeds, with comment about avoiding "odd user experiences", except where the failed alloc is part of the failed deployment (`partOfFailedDeployment` skips it). + +## 3. systemd + +### 3.1 Start limit (https://www.freedesktop.org/software/systemd/man/latest/systemd.unit.html) + +- A unit started more than `StartLimitBurst=` times within `StartLimitIntervalSec=` may not start again until the interval passes. Applies to all starts, including manual, not only `Restart=`; it is checked after condition checks, so failing conditions do not count. Units with `Restart=` that hit it are no longer restarted automatically; `systemctl reset-failed` flushes the counter. `StartLimitAction=` default none. Counters are cleared when the unit is unloaded. +- Source: `unit_test_start_limit` in src/core/unit.c (`ratelimit_below(&u->start_ratelimit)`; on hit logs "Start request repeated too quickly", returns -ECANCELED); `unit_reset_failed` calls `ratelimit_reset`. + +### 3.2 Job result "dependency" and the start limit + +- src/core/unit.c `unit_start` order: conditions (-ECOMM), asserts (-EPROTO), `unit_verify_deps` (-ENOLINK, "The necessary dependencies are not fulfilled"), then `UNIT_VTABLE(u)->test_startable`. For services, `service_test_startable` (src/core/service.c ~6381) calls `unit_test_start_limit`. A start failing on unfulfilled dependencies returns before the start-limit counter is touched. +- src/core/job.c: `-ENOLINK` maps to `JOB_DEPENDENCY` (`job_run_and_invalidate`); `job_fail_dependencies` also finishes dependent START/VERIFY_ACTIVE jobs with `JOB_DEPENDENCY` ("Dependency failed for %s."). `n_failed_jobs` increments only for JOB_FAILED, JOB_INVALID, JOB_FROZEN, JOB_CONCURRENCY. JOB_DEPENDENCY triggers `OnFailure=` handling. +- Requires=: if the required unit fails to activate and the dependent has After= on it, the dependent is not started; without After= it may still start. Wants=: failure "has no impact on the validity of the transaction as a whole." Dependent is stopped when the required unit is explicitly stopped or restarted; not stopped if the required unit stops on its own. + +### 3.3 Restart= and dependency failure + +- systemd.service(5) (https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html): the restart table is by exit cause of the service's own process (exit code, signal, timeout, watchdog, OOM); no row for dependency failure. Page: "service restart is subject to unit start rate limiting." Not found: a doc statement on whether Restart= acts on a dependency failure. From the code above, a dependency-failed start job finishes with JOB_DEPENDENCY and never passes through the service's own failure state (inference, not an explicit doc statement). +- Backoff: `RestartSec=` default 100ms; `RestartSteps=` with `RestartMaxDelaySec=` give exponential growth to a cap (doc example 10s, 20s, 40s, 80s, 160s, 160s...). `RestartMaxDelaySec` default infinity; both must be set. `RestartRandomizedDelaySec=` (v262) adds jitter. + +### 3.4 Transient units across reboot + +- systemd.unit(5) path table (man/systemd.unit.xml ~lines 403, 465): "Dynamic configuration for transient units" lives in `/run/systemd/transient` (system) and `$XDG_RUNTIME_DIR/systemd/transient` (user). That `/run` is volatile is not stated in this file. +- systemd-run docs: failed transient units are "kept in memory until the user explicitly resets their failure state"; `--collect` is `CollectMode=inactive-or-failed`. systemd-run and https://systemd.io/TRANSIENT-SETTINGS/ do not mention reboot. Not found: an explicit sentence on survival across reboot. + +## 4. Other supervisors + +### 4.1 Erlang/OTP supervisor (https://www.erlang.org/doc/apps/stdlib/supervisor.html, https://www.erlang.org/doc/system/sup_princ.html) + +- `intensity` (MaxR) and `period` (MaxT), defaults 1 and 5. If more than MaxR restarts occur within the last MaxT seconds, the supervisor "terminates all the child processes and then itself" (exit reason `shutdown`). The parent supervisor then restarts it or itself terminates; failure escalates up the tree. Stated purpose: prevent a process from repeatedly crashing for the same reason and being restarted indefinitely. +- Restart types: `permanent` (always), `transient` (only abnormal exit), `temporary` (never). Every restart the strategy performs counts toward intensity; the docs describe no per-cause charging and no backoff delay. Multi-level: total restarts before the top gives up is roughly the product of intensities above the failing child. Strategy `rest_for_one` restarts the failed child and those started after it. + +### 4.2 runit and s6 + +- runit `runsv` (https://smarden.org/runit/runsv.8.html): if ./run exits immediately runsv "waits a second" before restarting; no rate limit or retry cap described; keeps restarting unless told otherwise (`o`, `d`, `x`). +- s6-supervise (https://skarnet.org/software/s6/s6-supervise.html): "There is a minimum 1-second delay between two ./run spawns." A ./finish script exiting 125 makes supervise stop restarting; otherwise it restarts indefinitely. The give-up decision is delegated to a user script, not a built-in counter. + +### 4.3 Borg (Verma et al., EuroSys 2015; https://research.google/pubs/large-scale-cluster-management-at-google-with-borg/, PDF text read) + +- Borglet "restarts them if they fail"; Borg "monitors the health-check URL and restarts tasks that do not respond promptly" (sections 2.6, 3.3). +- Section 4 (Availability): Borg "automatically reschedules evicted tasks"; "limits the allowed rate of task disruptions and the number of tasks from a job that can be simultaneously down"; rate-limits finding new places for tasks from unreachable machines because it cannot tell machine failure from network partition; "avoids repeating task::machine pairings that cause task or machine crashes". +- Section 2.3 (updates): "Updates are generally done in a rolling fashion", with a limit on task disruptions an update causes; changes that would exceed it "are skipped". +- Not found in the Borg paper: a crash-loop backoff schedule, a restart budget, or any distinction between charging task faults and dependency faults. Omega paper not fetched. + +## 5. Facts grouped by the brief's sub-topics + +- (a) Charging only body faults: Kubernetes backoff keys on container exit regardless of cause (readiness failure does not exit and does not advance it). systemd runs dependency verification (-ENOLINK, JOB_DEPENDENCY) before the start-limit test, and JOB_DEPENDENCY is not in `n_failed_jobs`. Nomad's tracker counts any recoverable task failure or start error, with no dependency classification. OTP counts every restart. Borg avoids repeating task::machine pairings that cause crashes (placement, not budget). +- (b) Give up vs capped forever: OTP gives up and escalates. Kubernetes caps at 300 s with no give-up. Nomad restart fail-mode gives up locally, then reschedule continues (service default unlimited, exponential to 1 h); delay-mode never fails. systemd start limit stops auto-restart until the interval passes or reset-failed; `RestartMaxDelaySec` default infinity. runit restarts forever at 1 s; s6 restarts forever unless finish exits 125. +- (c) Rollout vs restart supervisor: Kubernetes splits by layer (kubelet restarts containers; ReplicaSet maintains pod count; Deployment controller scales old RS during rollout). Nomad splits by node vs server (client restart tracker vs server reconciler; reconciler skips rescheduling allocs of a failed deployment). Borg bounds update-induced disruptions with a rate limit. + +## 6. Memory claims vs sources + +budget-charge: + +1. "Kubernetes separates readiness from liveness; only liveness feeds CrashLoopBackOff": CORRECTED. Readiness never restarts and never advances backoff (confirmed). Backoff keys on any container exit (crash, liveness kill, startup-probe kill), so it is not fed only by liveness. +2. "CrashLoopBackOff caps at 5 min": CONFIRMED (docs 300 s; initial 10 s, doubling). Reset: docs say 10 min healthy; code says 600 s since the entry's last update. 60 s cap only under alpha gate. +3. "systemd does not run a unit whose required dependency failed (job result 'dependency')": CONFIRMED with qualifier: Requires= plus After= blocks the start; Requires= without After= may still start; Wants= does not block. +4. "Nomad reschedules with capped exponential delay": CONFIRMED for service jobs (exponential from 30 s, max_delay 1 h, unlimited by default); batch default is constant 5 s, 1 attempt per 24 h. +5. Dependency failure vs start limit (not in the question's claims, material): systemd checks dependencies before counting a start (confirmed by code order). Whether Restart= acts on dependency failure: UNSUPPORTED by docs. +6. Whether restarts after a node reboot count toward CrashLoopBackOff: UNSUPPORTED by docs; code read shows the kubelet backoff map in memory. Nomad's restart tracker is likewise in memory and rebuilt on client restart. + +handover-pre-fencing: + +1. "In a Kubernetes rolling update the old ReplicaSet keeps recreating its dying pods until the rollout scales it down": PARTLY CONFIRMED, PARTLY CORRECTED. ReplicaSets do create pods to reach their desired count, and the Deployment controller scales old RS down, unhealthy first. Container restarts are the kubelet's, not the ReplicaSet's; the RS recreates only pods that are gone. +2. "Nomad restarts old allocations by their own restart policy, independent of the deployment": PARTLY SUPPORTED. Restart is client-local using the alloc's task-group policy (code), so independent of the server-side deployment; no doc statement on the interaction. Server-side, failed allocs are rescheduled even when the deployment is not place-ready, except allocs of a failed deployment. +3. Any source saying an aborted or stalled rollout automatically hands the slot back to the old generation: UNSUPPORTED (Kubernetes takes no action on a stalled Deployment; Nomad `auto_revert` defaults false). diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index baabb18..a278165 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -254,6 +254,27 @@ The rework is a fresh branch from `583b2a7`, not a rebase of `1786ddf`. Its map - **H.** AwaitingReady and Routing keep the F16 shape. It is unreachable at today's heartbeats. - **I.** An `idunn expire` request written while Idunn is stopped lingers, and there is no verb to withdraw it, which is a downgrade hazard. +- **B5 MERGED and INSTALLED (2026-09-30).** Merged to main at `fcb8ba1` (Soul passes 1-3). Built on Yggdrasil: + `/srv/build/idunn-fcb8ba1`, 360 passed. `idunn` sha256 `e96bcefa...`, `idunn-provision` `127902ef...`, unit unchanged. + Installed at 22:35 CEST. + - **Backup**, taken with Idunn stopped and byte-compared: + `/root/idunn-store-backup-20260930T203515Z-pre-route-b5/`. It holds `control.cc` `fab5b66b...`, `history.cc` + `a2627ea4...` and `idunn.running` `848c08e5...` (a96ad9d). + - **Preflight** found one live transaction: the raven-muninn continuity `990dfc25` in AwaitingReady for about 40 + min. Raven had rebooted, and its actuator timed out at 21:36 and 21:49 CEST. The operator ruled "Feel free to + install". + - **After install:** + - No managed target's PID changed. Raven reattached after 15 s, and nginx made 0 reloads. + - B5 aborted the stale continuity at its passed deadline, and supervision opened `continuity-26737b91`. + - A single log line, "could not resume transaction tx-bcfbb693...: transaction disappeared while checking + scheduler progress", appeared at 20:35:38 UTC. It looks like a same-tick ordering quirk between the + resolver and resume. It goes to the next Soul pass. + - **Rollback rule:** stop Idunn and check `control.cc` for any `idunn.expiry_request` envelope. If one exists, + restore the backup instead of swapping the binary. + - **Named tradeoff (Soul batch-3 F1):** a persistent `ufw delete` failure while aborting a first routed deploy now + leaves the aborted candidate routed until ufw recovers. Before B5 the cost was reload churn. Q3-ii's abort + deadline bounds it. + - Test pins F2-F4 are in Hands (`hands/b5-pins`). - **B5 batch 2 (`c06ff2b..aef0da3`) and Soul pass 2 (2026-09-30): merge and install after batch 3.** - A, B, E, F and J hold. Verify: 356 passed. cargo-mutants caught 14 of 14 viable mutants, and Soul's hand mutants killed 16 of 20. diff --git a/docs/route-continuity-target.md b/docs/route-continuity-target.md index 77c201b..7c25d0c 100644 --- a/docs/route-continuity-target.md +++ b/docs/route-continuity-target.md @@ -52,12 +52,18 @@ web app and service through Idunn v2. The live evidence is on Yggdrasil: the admitted incarnation. The proof answers an Idunn challenge and is signed with the credential Idunn issued to that launch. Invariant 3 changes where the proof travels, not what it proves. -5. **Every post-fencing phase ends.** Each phase after fencing has an admitted - deadline. When a deadline expires, Idunn resolves it: stateless targets abort - to the incumbent. Stateful targets get an explicit terminal record with a - named recovery. A granted lease that the process does not pick up within its - window is a named failure with an Idunn-owned recovery. It is never a silent - wait. Resumable-forever is gone. +5. **Every post-fencing phase has a deadline and is never a silent wait.** A + deployment's stateless phase that passes its deadline aborts to its + incumbent. A continuity past its deadline is left to the operator + (operator-required incident) and retries its phase forward on a capped + backoff, because aborting it would stop the only running copy of the admitted + body (ruling post-fence-forward-retry), until a deployment that the + deployment brake has released awaits the handover of its target and the + continuity's route proof has not succeeded: then it yields through its own + post-fencing abort, and the deployment fences what is admitted (rulings + continuity-yields-to-deploy, c2d-yield-only-unproven-to-movable). A + continuity that has proven its route is serving and never yields. A + stateful deployment is left to the operator. 6. **The admitted receipt describes the running incarnation.** Continuity commits update the admitted generation atomically, so the rollback path never meets drift between the receipt and the projection. Rollback demotes the exact diff --git a/docs/secret-files-cut.md b/docs/secret-files-cut.md index 38dd679..71d3a60 100644 --- a/docs/secret-files-cut.md +++ b/docs/secret-files-cut.md @@ -43,6 +43,25 @@ happen again". - **F5.** The recipe exploration's all-at-once run stops early; mutants Z4, Z5 and Z6 survive. - **F6-F8.** An inspected `cause` is not covered, a narrowed predicate has no fixture, and an added NOT NULL on a nullable column is not caught. CHECK constraints and triggers are out of scope, as S5-8 is. +- **Batch 7** (`6c0c876..ca2366c`): F1-F8 fixed. + - Keys must be `indisvalid` with deterministic collations. + - Predicates are printed by Postgres, using a TEMP table in a rolled-back transaction, and compared as sets of + AND conditions. + - Index names are not required. Column privileges are checked with `has_column_privilege`. A nullable column must + stay nullable. + - Results: 241 passed, 39/39 live, 60/61 mutants killed. +- **Soul pass 7: merge.** Nothing blocks. + - 1400 predicate pairs were checked, and no two different predicates compared equal. + - The `indisvalid` argument held under interrupted concurrent DDL. + - TEMP is never refused for the real deployment, because the service role owns the database. +- **MERGED: Heimdall main fast-forwarded to `ca2366c` (2026-09-30).** +- Follow-up branch in Hands: + - S7-1: the recipe exploration never runs the production start shape. + - S7-2: extra unique indexes and similar documented as out of scope. + - S7-3: the column collation must be deterministic. + - S7-4: one more fixture each for Y3, Y4, Y5 and Y8. +- **Recorded:** a least-privilege column-subset grant is refused (F4(b)). That is theoretically servable, but it is + not how Heimdall is deployed. **Rulings and defaults, 2026-09-30 (Self).** - **Q3 A is the operator's ruling:** "I'll rotate before prod". Cuts 1-4 land with the current values; rotation diff --git a/docs/watchdog-reporting-map.md b/docs/watchdog-reporting-map.md index 70bcd93..4bc8814 100644 --- a/docs/watchdog-reporting-map.md +++ b/docs/watchdog-reporting-map.md @@ -18,11 +18,11 @@ probes ran on Yggdrasil over `ssh ygg`, read-only, 2026-09-30 13:51-13:55 UTC | # | What | Where | Result | |---|---|---|---| | I1 | Alert path | `src/control_plane.rs:3798-3812` (`ReportOnce`), `:4599-4610` (`report_once`), `:5114-5138` (`note_fault`, `clear_fault`) | The only operator signal is `eprintln!`. De-dup is `fault_reports: Mutex>` (`:4392`), in memory: an Idunn restart forgets it and reprints. `note_fault` also writes the transaction's `last_error`. | -| I2 | Continuity exhaustion | `:68-71` (6 restarts / 3,600,000 ms, 5 s doubling), `:1066-1072` (`restarts_used`, `restarts_exhausted`), decided `:5709-5722` under key `continuity:` (`:5678`) | Exhaustion is **derived** from `TargetSupervision.continuity_restarts` (newest 6 times, sliding window). No write marks it. The fault clears (`:5723`) as soon as one entry leaves the window, so a permanently dead target leaves and re-enters exhaustion several times each hour. | +| I2 | Continuity exhaustion (since `idunn-reincarnation` cut 1: `restarts_page`, a paging threshold only; nothing stops, and the line numbers below are pre-cut) | `:68-71` (6 restarts / 3,600,000 ms, 5 s doubling), `:1066-1072` (`restarts_used`, `restarts_exhausted`), decided `:5709-5722` under key `continuity:` (`:5678`) | Exhaustion is **derived** from `TargetSupervision.continuity_restarts` (newest 6 times, sliding window). No write marks it. The fault clears (`:5723`) as soon as one entry leaves the window, so a permanently dead target leaves and re-enters exhaustion several times each hour. | | I3 | Route degradation | `RouteSupervisionState` `:890-942`; `record_failed_challenge` sets `degraded_since_unix_millis` on the **first** failed challenge, `record_proved_challenge` clears it; reported at `:5975` under `route:` | Durable per incarnation (inside `admitted_generation.v4`); a new generation starts from `default()`. Backoff doubles from the topology max age to a cap. | | I4 | Other operator-facing faults | `:5645-5651` (`generation-down:`, "holds a dead admitted generation down"), `:6016` (`route-record:`), `note_fault` per transaction | Also stderr plus `last_error`. Not named by the target; listed so the paging-set question sees them. | -| I5 | `OperatorRequired` on main | `TerminalRecovery::OperatorRequired` `:878-885`, validated `:1954`; constructed only in tests (`:12525`, `:12535`) | No production path produces it on main. | -| I6 | `OperatorRequired` on B5 | `hands/b5` `resolve_phase_deadline` (diff hunk at `control_plane.rs` ~8560-8625 on the branch), `PhaseEnd::OperatorRequired` | Deadline passed on a stateful deploy with an issued lease: `report_once("deadline:")`, returns `Ok(None)`, the transaction stays live. No terminal transition and no operator action that ends it were found on the branch (`idunn expire` writes `idunn.expiry_request.v1`, which re-enters the same arm). | +| I5 | `OperatorRequired` on main (pre-cut-2 reading; since `idunn-reincarnation` cut 2 `resolve` reaches it in production through `PhaseEnd::OperatorRequired` and `PhaseEnd::RetryForward`, both calling `leave_to_operator`, which opens the `operator-required` incident `IncidentCondition::OperatorRequired`, subject the transaction id; `close_spent_operator_incidents` closes it when the record no longer `leaves_to_operator`) | `TerminalRecovery::OperatorRequired` `:878-885`, validated `:1954`; constructed only in tests (`:12525`, `:12535`) | No production path produces it on main. | +| I6 | `OperatorRequired` on B5 (superseded by cut 2: B5's deadlines merged as `PhaseDeadline`, and a continuity past its deadline now also lands here) | `hands/b5` `resolve_phase_deadline` (diff hunk at `control_plane.rs` ~8560-8625 on the branch), `PhaseEnd::OperatorRequired` | Deadline passed on a stateful deploy with an issued lease: `report_once("deadline:")`, returns `Ok(None)`, the transaction stays live. No terminal transition and no operator action that ends it were found on the branch (`idunn expire` writes `idunn.expiry_request.v1`, which re-enters the same arm). | | I7 | Store kinds | `:55-61`, `:936-966`; B5 adds `idunn.expiry_request.v1` | `control.cc`: `idunn.deployment_command`, `idunn.deployment_transaction`, `idunn.admitted_generation.v4`, `idunn.target_supervision.v1`. `history.cc` beside it holds terminal transactions (`:3653`), retired one per tick (`:5068-5091`). | | I8 | Single-writer precedent inside `control.cc` | B5 `ExpiryRequest` doc comment | "The CLI only writes this record. The daemon's deadline resolver reads it ... No CLI writes a transaction field." One writer per kind. | | I9 | Serve loop | `:5041-5058`, `run_scheduler_tick` `:5098-5110` | 500 ms poll. A tick error is logged, never propagated. | @@ -118,6 +118,32 @@ file ran alone with the TAP reporter, then the whole glob ran once. | C27 | Suite with the fix | scratch commit `a6770c5` on `42728a67`: the scoped name at the seven live C21 sites, a skip guard on both Rust tests, and the `CARGO_TARGET_DIR` override removed. Same command. | rust-smoke 1/0/0/1 (`# SKIP`); delivery 9/9/0/0; permit 2/2/0/0; rudp 1/0/0/1 (`# SKIP`); feedback-cli 8/8/0/0; idunn-health 3/3/0/0. Glob total 24 tests, 22 pass, 0 fail, 2 skipped, exit 0. `agent-transport.mjs` and `governance-threads.mjs` were checked only by name resolution (C22), because no test loads them. | | C28 | C3 at the reader's install pin | `git show 36ea08d3:packages/cultcache-ts/src/single-file-messagepack-backing-store.ts` | `pullAll` is at `:43`, and the file has no lock. C3 holds at `36ea08d3`, which is the CultLib that cut `ops-notice-deploy` installs. | +### Topology-lock cut probes (2026-09-30, 21:00-21:25 UTC) + +Pinned heads: Idunn origin/main `bca3d85`; `hands/watchdog-idunn-incident` +local `9549d00` (h3, not yet pushed at probe time; origin at `3ed5f26`); +`hands/idunn-projection-mode` `15f7e18` (unmerged); Odin `665dc08`; +CodexConnector `6519289`; Ghostlight `eeb3a68`; gamecult-ops `fdb4356`; +cultcache-rs at Idunn's pin, CultLib `61549aa`. Line numbers marked *trial* +are from a scratch merge of `9549d00` into `bca3d85` (clean, no conflicts), +never in `F:\Projects\Idunn`. Yggdrasil probes were read-only over `ssh ygg`; +the semantics probe (C36) ran in a container through `ygg-verify.sh`. + +| # | What | Where / command | Result | +|---|---|---|---| +| C29 | Every writer of `topology.cc` | trial `git grep` for `CultCacheTopologyDriver {`, `Engine::open(`, `mutate(` | One writer: the `idunn serve` daemon. `CultCacheTopologyDriver::mutate` (trial `src/drivers.rs:4393-4411`) is the only write path; its seven callers are `demote_to_expected_only` `:4455`, `withdraw_stale_incarnation` `:4567`, and the `TopologyPort` methods `:4778, 4846, 4897, 4940, 4992`. `Engine::open` has one production caller, `boot` (`src/control_plane.rs:5330`); the CLI verbs `up`, `status`, `cancel`, `expire` and `validate` write only `control.cc`. `idunn-host` and `idunn-provision` never touch the projection. | +| C30 | Every reader of `topology.cc`, across repos | `git grep -e 'topology\.cc' -e idunn-projection HEAD` in every repo under `F:\Projects` | Readers: Odin (`crates/odin-daemon/src/lib.rs` `CultCacheIdunnProjectionSource::entries`, `pull_all_read_only_snapshot`, lock-free since its first commit `eedce4f`, 2026-09-04), Idunn itself (`snapshot`, trial `drivers.rs:4383-4388`, lock-free; every `control_plane.rs` read is `pull_all_read_only_snapshot`), and the nightly backup (C33). No managed target reads the file: CodexConnector, Ghostlight, Muninn and StreamPixels name only `odin.verse-topology.v1`, and they receive their Expected record in the runtime bundle (`GAMECULT_IDUNN_RUNTIME_BUNDLE`). The unit comment (`deploy/idunn-yggdrasil.service:77-80`) and `docs/migration.md:505-509` say that "every managed target must read it", which is stale. | +| C31 | Shared-lock readers that rely on blocking a writer | cultcache-rs `61549aa` `lib.rs:337-395` (`with_read_only_shared_snapshot`), `:905-910` (`pull_all`, which takes `lock_shared`); cross-repo grep for `with_read_only_shared_snapshot` | That API holds the sibling lock shared so that no writer can replace the snapshot during `action`. Its production users are CodexConnector (`src/idunn_health.rs:441, 562`, `src/daemon.rs:205`) and Ghostlight (`crates/ghostlight-dungeon/src/idunn_health.rs:901`). All of them read the runtime authority record, the process-write-lease record, or their own configuration. None reads `topology.cc`. The write-lease lock is a different file under `idunn-authority` (`CultCacheWriteLeaseDriver`, trial `drivers.rs:4173, 4218`, already `try_compare_exchange_snapshot`). No reader of `topology.cc` takes a lock to get a consistent view. | +| C32 | How a reader stays consistent without the lock | cultcache-rs `61549aa` `lib.rs:666-688` (`write_all_unlocked`) | Every write stages `.tmp`, fsyncs it, `replace_file_atomically` renames it over the store, and fsyncs the directory. A reader that opens the path sees one whole inode. Consistency comes from the rename, not the lock. | +| C33 | The nightly backup's lock use | live `/usr/local/sbin/gamecult-state-backup` sha256 `9d8ebd84...`, identical to gamecult-ops `scripts/gamecult-state-backup-yggdrasil.sh`; lines 98-126; `systemctl cat gamecult-state-backup.service` | Every file with a `.lock` sibling is re-copied under `flock -s -w 20 -E 75 cp -a`. Past the wait it falls back to a double-read. The unit has no `User=`, so it runs as root: a `0600 root` lock still opens (`flock(1)` opens the path), and the backup keeps its shared lock. It skips a store whose lock is absent (`[[ -f lock ]] || continue`) and never creates one. The copy is milliseconds of `cp` once a night. Under a try-lock, a publish in that window is contended and retried on a later pass (C35). The backup does not need the lock for a whole copy (C32). | +| C34 | Live state of the projection dir | `ls -la`, `getfacl -p /var/lib/gamecult/idunn-projection`; `/proc/locks` against the lock's inode; `find /proc/*/fd -lname` | `drwxr-xr-x+ root`, `default:user::rw- default:group::r-- default:other::r--`. `topology.cc` `0644`, 13436 bytes, written 21:07. `topology.cc.lock` is `0644 root`, inode 8227809, created 2026-09-06. No flock is held on it and no process has either file open. `incidents.cc` does not exist yet, because the incident cut is not installed. The default ACL **is** documented, in Idunn `docs/migration.md:523` (it closes the chmod window), though gamecult-ops does not record it. `idunn-yggdrasil.service`: `User=root`, `UMask=0027`, `WatchdogUSec=0`. Odin runs as DynamicUser uid 62025. | +| C35 | What a failed topology publish does inside the tick | trial `src/control_plane.rs` at each call site | Every tick-path write returns `Err` through `?` **before** any evidence of it is persisted, and the step is retried. `advance_starting` records `expected_publication_sha256` or `activation_publication_sha256` only after publish returns `Ok` (`:6605`, `:6664`). `advance_leasing` does the same for the write-lease projection digest (`:7004`, `:7073`). `advance_fencing` withdraws the lease projection before recording (`:6894`, `:6926`). The post-fencing abort and pre-fencing cleanup write `CleanupEvidence::Complete` only after `reconcile_failed_candidate_projection` (`:8956`, `:9046`, `:9180-9204`). A failed continuity demotion (`:6099`) mints no restart and defers continuity. The admitted Expected repair (`:5870`) is logged and re-checked every pass. `retire_stale_incarnations` (`:7914`) returns `Err`. **The one path that is not retried is boot** (`reconcile_failed_continuity_projections`, `:5228-5275`, called once from `boot` `:5332`). It logs a failure and does not retry, and it is outside the tick. | +| C36 | Lock semantics under the live ACL | `ygg-verify.sh Idunn bca3d85 python:3.12-slim` running scratch `probe_lock.py`, as root | (1) A file created with mode 0666 under this default ACL and umask 027 lands `0644`, CultCache's shape. (2) A separate process holding `LOCK_EX` on it makes a nonblocking try from the parent contend. (3) **`fchmod 0600` on the same inode does not revoke that holder**: the try still contends. (4) A 0600 `O_CREAT|O_EXCL` staged file renamed over the lock lands `0600`, and the try on the new inode **succeeds** while the old holder still holds the orphaned inode. (5) uid 65534 cannot open the replaced lock (`Permission denied`). | +| C37 | The incident cut's private-lock mechanism | `git show 9549d00:src/incident.rs` `:229-245` (`exchange`), `:336-375` (`secure_lock`) | `exchange` calls `secure_lock(path)`, then `try_compare_exchange_snapshot`, and turns `Mismatch` and `LockContended` into errors. `secure_lock` opens `.lock` with `create` and mode `0600`, then runs `set_permissions(0600)` on that descriptor. Its doc says that a descriptor another uid already holds is not revoked. So a lock left `0644` (C34's live `topology.cc.lock`) stays holdable by whoever opened it before (C36 step 3). It builds the lock name by hand, duplicating `authority_lock_path` (trial `drivers.rs:8194`). | +| C38 | `hands/b5` | `git merge-base --is-ancestor` | `hands/b5` (`f85e743`) and `hands/b5-pins` (`1cfbeff`) are both ancestors of origin/main (merged at `fcb8ba1` and `ddc031a`). The topology cut has no B5 conflict. | +| C39 | Unmerged branches that touch the topology driver | for each branch not in origin/main: `git diff origin/main... -- src/drivers.rs` | Two. `hands/watchdog-idunn-incident` (the dependency). `hands/idunn-projection-mode` (`15f7e18`, 4 commits, 2026-09-30 01:15-02:40 UTC, no campaign doc) deletes `publish_projection_mode`, has `mutate` set a default ACL on the directory by `setxattr` (`publish_directory_default_mode`), and replaces the mode test with `the_published_projection_is_never_visible_in_another_mode`. That test asserts the **lock** is always `0644`, which contradicts `tick-lock-private`. It conflicts textually with this cut at `mutate` and at `publish_projection_mode`. | +| C40 | The private root's locks | `ls -ld /var/lib/gamecult/idunn`; `sudo ls -la` inside it; `getent passwd idunn`; `ps` | `/var/lib/gamecult/idunn` is `drwxr-x--- idunn:idunn` (uid 986, group 978, which has no members; no process runs as `idunn`). `control.cc.lock` is `0644 root`. The tick's control-store writes use blocking `compare_exchange` (e.g. trial `control_plane.rs:5471`), so root (the backup's `flock -s`) or uid `idunn` could hold one. Nothing runs as `idunn` today. | + ## Model page A row per persistent kind this campaign creates or depends on. "New" means the @@ -127,6 +153,7 @@ cut spec fixes them. | Kind | Identity | Lifecycle | Authority | |---|---|---|---| | **Incident record** (new, `idunn.operator_incident.v1`, its own file `/var/lib/gamecult/idunn-projection/incidents.cc`, published 0644 beside `topology.cc`) | Key `::`. Namespace: `incidents.cc`, one type; the file refuses any other. It is never written to `control.cc`, which bails on unknown types, so a rollback would stop continuity (C1). `condition` is a closed enum (`continuity-exhausted` first; `odin-store-contested`, `odin-store-unpersisted` and `operator-required` join when their sources exist). `subject` is the target for target conditions and the `transaction_id` for `operator-required`. `opened_at` is written once by Idunn's clock. Injective because of one rule: **at most one open incident per `(condition, subject)`**, checked by a compare-exchange create. | **Created** at the site that decides the condition, after that decision and before its trace line. **Closed** by writing `closed_at` and `close_reason` on the same record, never by deleting it. Continuity-exhausted closes when the pass sees the workload running (`recovered`) or the target is no longer admitted, **not** when the window slides (I2). Odin conditions close when the catalog stops reporting them. `operator-required` has **no closing path yet** (I6; follow_up `b5-operator-required-no-exit`). **Reopened**: never; a recurrence is a new record with a new `opened_at`. **Replayed after an Idunn restart**: the durable open record makes the site write nothing. **Pruned**: closed more than 7 days ago, it retires to `history.cc`. **Store failure**: reported once as a fault and never returned into the tick (`survival-independent`). | Idunn's daemon, at the decision site, is the only writer. The same file is the record and what Bifrost reads, so no separate projection exists to drift. Forbidden: the CLI (`status` reads only), Bifrost and every other reader, Odin, `ReportOnce`, `control.cc`, a repair loop that opens incidents from log text. | +| **Projection lock files** (exist: `topology.cc.lock`; new with the incident cut: `incidents.cc.lock`, `incident-history.cc.lock`) | `.lock` beside its store, named by `authority_lock_path`. One per store file. | **Created** by Idunn at mode `0600` before its first exchange on that store. **Replaced** by a staged `0600` file renamed over it whenever Idunn finds it wider than `0600` or not owned by Idunn's euid; a chmod cannot revoke an existing holder (C36). **Held** only for one nonblocking exchange. **Never published**: the store is `0644`, and its lock is not. | Idunn's daemon alone opens it for writing. The root backup may take it shared for its copy (C33), and a contended exchange is retried. Forbidden: any reader opening it (none does, C30-C31), `publish_file_mode` on it, a default ACL deciding its mode. | | **Stderr trace** (exists) | `report_once` key, in memory | Printed once per process lifetime per fault; lost on restart (I1). | **Demoted**: `ReportOnce` is no longer an owner of "has the operator been told". It de-dups stderr only. | | **Delivery request / journal entry** (Bifrost store) | Keyed by the Idunn incident key plus the notice kind (`opened`, and `closed` if Q `closure-notice` says so). Injective because the incident key is. Not derived from payload text or time (BF1's `raisedAt` hash is the counter-example). | **Created** by Bifrost when its reader first sees a notice-worthy incident state in the projection. **Revised**: `pending` -> `sent` or `failed`, with an attempt count and last error. **Retried** until sent, with backoff; a create for an existing key writes nothing (V1's unconditional `put` is the counter-example). **Replayed after a Bifrost restart**: journal is durable; a `pending` entry is retried. **Pruned** only after the incident key has left Idunn's projection. | Bifrost's reader only. Forbidden: Idunn (it never writes Bifrost's store, even though Y2 shows it physically could), VoidBot, the operator by hand. Which Bifrost process: see Rationale. | | **Delivery receipt** (Bifrost, `discord_post_receipt.v1` or the journal's `sent` state) | Same key as the request. | Written once on success with the Discord message id. | Bifrost. **Idunn does nothing with it**: it neither reads it in the daemon nor waits on it (`survival-independent`). `idunn status` does not show delivery state; Bifrost's journal is where a failed delivery is visible. | @@ -139,9 +166,10 @@ cut spec fixes them. Empty cells, stated plainly: -- `operator-required` closure: no path on `hands/b5` ends a transaction left to - the operator (I6). The incident can open and never close until B5 names that - action. +- `operator-required` closure: the incident closes when the transaction ends or + its step goes on (`close_spent_operator_incidents`). A stateful deployment + left to the operator still has no Idunn-owned terminal action; the operator's + `idunn expire` re-enters the same arm. - Delivery-failure visibility: no owner. - Discord `enforce_nonce` window: not verified against Discord's current docs in this pass. @@ -217,10 +245,13 @@ and moves on. An unreadable store means no incident opens (the stderr trace still prints); it never changes what continuity decides. The Idunn cut's survival test pins this with an unwritable store path. -**Bifrost reads without the lock.** Idunn's writes block on an exclusive lock -(C2). A reader that took the lock could stall a tick. cultcache-ts `pullAll` -takes none (C3, C4), and Idunn's rename is atomic, so the reader only ever sees -a whole snapshot. The reader never opens Idunn's file through a CultMesh node, +**Bifrost reads without the lock.** Idunn's writes once blocked on an +exclusive lock (C2). The incident and topology cuts make every write in the +projection directory a nonblocking exchange behind a private lock. A reader +that took the lock is therefore unnecessary. It would also be refused, because +it cannot open a `0600` lock. cultcache-ts `pullAll` takes no lock (C3, C4), +and Idunn's rename is atomic (C32), so the reader only ever sees a whole +snapshot. The reader never opens Idunn's file through a CultMesh node, which could flush. **Delivery at most once, with the nonce as the second guard.** This follows @@ -293,3 +324,29 @@ goes, because it is wrong on every host except one workstation. two branches merge in either order without conflict. The reader depends on both. It is written against a tree with one Discord command vocabulary, and it is verified by a suite that loads. + +**The topology write stops waiting, and the lock stops being published.** The +lock was published `0644` because a comment assumed that readers open it +(`publish_projection_mode`, from `fbb9626`). None does (C30, C31), and +consistency comes from the rename (C32). So the lock is Idunn's alone, and a +contended publish is an error, not a wait. That is safe because every tick path +treats a failed publish as not done: no publication digest, phase transition, +cleanup evidence or continuity restart is persisted before the write returns +`Ok` (C35). A contended publish is retried on a later pass or after backoff, +and Idunn never believes it published what readers do not see. Boot +reconciliation is the exception. It runs once, outside the tick, and it already +logs failures and moves on. Only root processes can contend once the lock is +private: the backup's copy, a few milliseconds a night. No retry machinery is +added for it. + +**One lock rule for the whole projection directory.** The topology cut moves +the incident cut's `secure_lock` into one crate-level exchange primitive that +both stores call, rather than a second copy. It also strengthens the rule: +a lock wider than `0600`, or owned by someone else, is replaced by a rename, +not chmodded. A chmod does not revoke a holder (C36), and the live +`topology.cc.lock` has been `0644` since 2026-09-06 (C34), so tightening it in +place would leave any existing holder able to block every topology publish +forever. The replacement runs only while the lock is not yet private. Once it +is, no other uid can open it, so a later replacement never splits exclusion +between two inodes. A root backup holding the old inode during the first +replacement copies one whole inode anyway (C32). diff --git a/docs/watchdog-reporting-postmortem.md b/docs/watchdog-reporting-postmortem.md new file mode 100644 index 0000000..964698d --- /dev/null +++ b/docs/watchdog-reporting-postmortem.md @@ -0,0 +1,245 @@ +# Watchdog reporting postmortem + +Campaign `idunn-watchdog`, 2026-09-30 to 2026-10-01. This was the first Eureka +campaign whose state was typed in Huginn's mind (instance `eureka`) rather than +kept in a prose map. Document ids below are `idunn-watchdog::`. +Read them with `view`; this page does not restate them. + +## Summary + +Idunn is now a watchdog that reaches the operator on Discord. + +- **Idunn** opens a durable incident at the site where a continuity budget is + exhausted (since `idunn-reincarnation` cut 1 this is a paging threshold + only; continuity keeps restarting). The incident is written to `incidents.cc`, a world-readable file + in its projection directory with a private 0600 lock, and closes when the + target runs again and its restart window is no longer exhausted. +- **Bifrost's watchdog-notice reader** runs from a one-minute timer. It sends + one opening DM and at most one closure DM per incident, keeps its own + journal, and retries a failed send forever with capped backoff. +- **Scale:** 8 cuts across Idunn, Bifrost and gamecult-ops: 13 spec revisions, + 17 Hands reports, 17 Soul verdicts and 55 findings. Measured against the + in-force estimates: + +| Repo | Estimate (lines) | Landed (lines) | +|---|---|---| +| Idunn | +930/−101 | +2618/−354 | +| Bifrost | +246/−335 | +1182/−429 | +| gamecult-ops | +280/−35 | +505/−104 | + +- **Installed on Yggdrasil:** + - Idunn `a909f93` at 2026-09-30 23:21 UTC, then `cb60965` at 2026-10-01 + 02:02 UTC. + - The Bifrost reader at `160fac20` with CultLib `36ea08d3`, at 2026-10-01 + 08:04 UTC. +- **First live notice:** the incident's first real input was raven-muninn, + which exhausted its budget minutes after the incident cut was installed. Its + opening DM was delivered at 08:04:41 UTC, and the operator confirmed it. +- **Still owed:** the closure DM for that same incident + (`follow_up:closure-dm-observed`). It depends on raven-muninn recovering, and + raven-muninn's own fault lies outside this campaign. + +## Scope and invariants + +Target `r4`. Its invariants are: + +- `incident-owner` +- `record-before-delivery` +- `delivery-owner` +- `survival-independent` +- `one-per-incident` +- `no-sensitive-egress` +- `tick-lock-private` + +The operator directions are `ruling:operator-watchdog-direction` and +`ruling:operator-bifrost-bridge`. The second made Bifrost the bridge to +Discord, so Idunn holds no credential and speaks no Discord API. + +Excluded: + +- **Odin's store condition and B5 `OperatorRequired`.** Neither can page yet. + Odin's O2 cut is unbuilt, and B5 has no operator exit + (`follow_up:b5-operator-required-no-exit`). +- **VoidBot's `notify_owner`.** +- **The Persona Discord crossing.** +- **Eve lowering.** + +The earlier attempt was Bifrost `c063f27` (2026-09-04): an alarm was a command +written into Bifrost's store. It never ran, because no pump was deployed and +the deployed pump lacked the DM verb. It was also reversed by ruling: +`ruling:delivery-crossing` has Bifrost read Idunn's projection, so no write +crosses an owner boundary. + +## Timeline + +| Cut | Reports | Soul passes | Notable finding | +|---|---|---|---| +| ops-retire-alarm | h1 | 1 | none | +| bifrost-retire-alarm | h1–h2 | 2 | `silent-skip`: a stray `discord-dm` command sat pending forever | +| bifrost-suite | h1 | 1 | `suite-cultcache-scope` (pre-existing): 9 of 17 tests failed at base on a CultLib rename | +| bifrost-notice-reader | h1–h3 | 3 | `prune-keyed-on-presence`, `bridge-exit-is-failed`: duplicate DMs and blocked closures | +| bifrost-notice-retry | h1–h2 | 2 | `future-stamp-stalls-retry` | +| ops-notice-deploy | h1–h3 | 3 | `retry-window-five-minutes`: a 5-minute outage dropped a notice for good | +| idunn-incident | h1–h3 | 3 | `lock-stalls-tick` (Blocker); `lock-acl-world-openable` (High) | +| idunn-topology-lock | h1–h2 | 2 | `contended-publish-aborts`; the live stall it fixed was pre-existing | + +## Structural delta + +The landed code was about 2.5× its estimate, and the misses were not explained +cut by cut as the skill asks. + +- **idunn-incident:** +1813 against +700. Its fix batches added the lock + primitive, load-independent lock tests, and per-operation fault keys. +- **bifrost-notice-reader:** +891, against r3's estimate of +140. That r3 + figure described only the revision's own delta, not the cut, so r3 carried + the wrong baseline. + +Deleted: + +- Bifrost's `publish-idunn-alarm` and the pump's `discord-dm` verb. +- gamecult-ops' dead `--operator-alarm-command` scripts. +- Idunn's `publish_projection_mode` and its 0644-lock test. +- The reader's prune-on-absence logic and its `WATCHDOG_NOTICE_NOW_MS` + production clock seam. + +Added: + +- `idunn.operator_incident.v1` (`incidents.cc`) and + `bifrost.watchdog_notice_execution.v1` (the journal). +- `bifrost-watchdog-notice.{service,timer}`. + +Nothing was parked. + +## What Soul caught + +Green suites passed every defect below. + +- **Survival and locking.** + - Incident writes took a *blocking* flock inside Idunn's scheduler tick, on + a world-readable lock. Any local uid could freeze continuity for every + target, and `WatchdogUSec=0` meant nothing would restart Idunn + (`s1.lock-stalls-tick`). + - After the fix, the live directory's default ACL still made the lock 0644, + so any uid could stop incidents opening. The test ran in a temp directory + with no ACL (`s2.lock-acl-world-openable`). + - The same probe exposed a pre-existing live hazard: `topology.cc.lock` 0644 + with a blocking write in the tick, measured at 5.07 s at base + (`s2.topology-lock-stalls-tick`). The operator ruled it into the campaign; + after the topology cut the tick takes 0.09 s. +- **Split authority.** + - The reader pruned its "already sent" journal whenever a key was absent + from Idunn's store. Restoring a backup would have re-sent old DMs. + - The reader decided subject validity with its own grammar, narrower than + Idunn's. + - Retention was decided by two predicates in Idunn. +- **Lifecycle.** + - One running observation closed an exhaustion incident. A release that + crashes after each restart would have sent about 12 DMs an hour + (`s1.flap-reopens`). + - A bridge that died after Discord accepted a message was retried up to 5 + times, and the closure was then blocked forever. + - Five attempts on a one-minute timer dropped a notice after a 5-minute + Discord outage. +- **Decorative checks.** + - A compare-exchange expected `None` on a key that embeds `now`, so it + guarded nothing (`s1.cas-decorative`). + - A 2-second wall-clock bound on a lock test flaked under load and counted + equivalent mutants as caught. +- **The verifier itself.** Soul's s3 found that the stopgap forced one + `CARGO_TARGET_DIR` on every job, so cargo-mutants' parallel workers tested + each other's binaries. Two runs of the same two mutants disagreed + (`s3.mutants-share-target-dir`). Every parallel mutation count reported + through the stopgap before 2026-10-01 is unproven, in both sessions' work. + +## Operator corrections + +- **"Bifrost is indeed intended to be the bridge to external services like + Discord."** The agents had found two delivery paths: VoidBot's + `notify_owner` and Bifrost's command store. The ownership boundary was + missing from both repos' summaries. +- **"Stewardship is currently universal…"** Self had filed a gap treating + follow-up repos as needing stewardship. It was withdrawn; the rule was not + written in the skill. +- **"…we seem to be saving some tokens… but we're very very slow."** Self + blamed cold builds, measured, and was wrong. Cold builds took about 51 s and + warm builds 29 s. The cost was slot queueing, repeated verify jobs, a test + that hung a slot for 39 minutes, and a 4-hour job from the other session + holding a fifth of capacity. +- **"Yep, fixing Mind issues is highest priority, immediately once this + campaign closes"** (`ruling:mind-fixes-next`). + +## Incidents + +- **Network outages (ENOTFOUND).** Two outages killed three agents + mid-pass. Each was resumed by `SendMessage` with context intact, and nothing + was lost, because each agent's state was in commits and the mind. +- **A shared stopgap target directory** made mutation evidence noise. It is + fixed in Eureka `a755a60`. +- **Self's own brief inflated runtime.** It told Hands to mutate serially to + dodge a flake, and was corrected to skipping the flaky test by name. +- **A hung test held a verify slot for 39 minutes** + (`s2.route-connect-test-hangs`). It is fixed. +- **A Hands test helper overwrote the container's `/usr/bin/node`** through a + symlink. It was caught by the agent's own baseline and then confined + (`s2.test-clobbers-usr-bin-node`). +- **A worktree directory stays locked** by a finished process + (`F:\Projects\gamecult-ops-watchdog-deploy`, empty). Removing it is left to + the operator. + +## What worked + +- **Soul attacks the layer where the rule fails, not the test.** Every + High/Blocker finding came from a probe the shipped tests could not see: a + separate-process lock holder, the live directory's ACL, or a restored store. +- **Point Soul at the load-bearing invariant in the brief.** Naming "can the + lock hang the tick" produced the Blocker on the first pass. +- **Read before touching live state.** Self installed only after checking the + installed revision, the in-flight transactions and the unit diff. The repo + unit carried a BOM the live one did not. +- **One verify job per pass.** Once briefs required it, a reader fix batch + took 11 minutes, where the first reader Hands took 38. + +## What to change in the pipeline + +Already changed in the skill: + +- The substrate gaps moved from a table to the mind. +- Imagination owns the model page. +- Soul pays for each rerun once. +- The rust image uses sccache. +- The stopgap no longer forces `CARGO_TARGET_DIR`. + +Still to change: + +- **Explain every ledger miss in the next report's deviations.** This campaign + ran at about 2.5× its estimate and nobody explained it per cut. +- **Make a revision's `estimate` the whole cut.** r3's estimate covered only + its own delta. +- **Verify the merged tree of a long-lived branch before pushing.** Self did + this once, for the incident cut over B5, and it should be the rule. +- **Briefs name the standard sibling build command.** The stopgap cannot + provision siblings (`follow_up:gap-verify-no-sibling-repos`), and one + hand-rolled build produced four false failures. + +## Open follow-ups + +Fifty follow-ups are in force. Twenty are substrate gaps (`gap-*`): the work +the operator ruled next. Query them with the substrate-gaps recipe in the +skill. Seven findings remain unresolved: + +- **`cut-idunn-incident.s3.mutants-share-target-dir`.** It is fixed in the + stopgap but awaits a Soul mutation run under the fixed script. +- **Six findings whose resolution keys exceed 64 bytes** + (`follow_up:gap-unresolvable-finding-key`). Their substance is recorded in + `follow_up:notice-retry-hardening`, `follow_up:ops-refusal-test-hardening`, + `follow_up:bifrost-pump-input-hygiene` and + `follow_up:contended-publish-bounds`. + +Operational follow-ups that matter most: + +- `closure-dm-observed` +- `notice-unknown-settle` +- `contended-publish-bounds` +- `idunn-load-flaky-tests` +- `cultcache-rs-lock-mode` +- `cultcache-ts-readonly-pull` diff --git a/docs/watchdog-reporting-target.md b/docs/watchdog-reporting-target.md index 962f118..509cb6e 100644 --- a/docs/watchdog-reporting-target.md +++ b/docs/watchdog-reporting-target.md @@ -23,15 +23,19 @@ Today nobody would hear about it. Idunn already detects several conditions an operator must act on, and for each one it writes a single `eprintln!` line to journald (`report_once`, `control_plane.rs:4599`): -- A continuity restart budget is exhausted (6 per hour; `restarts_exhausted`, - decided at `control_plane.rs:5709`). Idunn stops restarting the target, and - the only trace is one log line. +- A continuity paging threshold is reached (6 charged attempts per hour; + `restarts_page`, formerly `restarts_exhausted`). Continuity never stops: + it keeps restarting on a capped backoff, and this threshold only pages. When + this target was written the only trace was one log line. - An admitted route is degraded. It fails its stable-route challenge (`supervise_admitted_route`, ~5973). The degraded mark is durable, but no one is told. -- A post-fencing phase deadline expires into `OperatorRequired`. This lives on - B5, which is unmerged on `hands/b5`. The failure record is durable and the - notice is a log line. +- A post-fencing phase deadline leaves a transaction to the operator: a + stateful deployment whose candidate was issued a lease, or a continuity + retrying forward past its deadline. Since `idunn-reincarnation` cut 2 it is + the second incident condition, `operator-required` (subject: the transaction + id), beside `continuity-exhausted`; it closes when the record ends or its + step goes on. Odin's own store condition (`odin.store_condition.v1`, `contested`) is mapped but not built. Idunn reads nothing about Odin's store health today. @@ -66,6 +70,15 @@ the machine has a detector with no record, and a deliverer that nothing feeds. clears, one closure notice follows, and only if the opening notice was sent. Repeated delivery attempts for either notice are idempotent at the recipient's end. +- **No tick waits on a lock another uid can hold.** Root holders, such as + the nightly backup's millisecond shared lock, are trusted and bounded. The + operator ruled this scope on 2026-09-30, choosing uid-scoped. Soul found that any + local uid could freeze Idunn's scheduler tick by holding a published + `.lock` file. This was true for incidents during the campaign, and for + `topology.cc` before it. Idunn's writes in its projection directory are + non-blocking exchanges. The locks behind them are private to Idunn, + whatever the directory's default ACL says. The operator ruled on + 2026-09-30 that the topology fix belongs to this campaign. - **Nothing sensitive leaves.** A notice names the target, the condition and the time. It never carries a binding value, a path to a secret, a token, or a line of configuration. @@ -75,8 +88,8 @@ the machine has a detector with no record, and a deliverer that nothing feeds. - Odin's store-recovery cuts, O2 included. They produce `odin.store_condition` and are Odin's campaign, still in prose. This campaign consumes the record once it exists. -- Idunn's B5 phase deadlines. B5 is in prose on `hands/b5`. This campaign - reports its `OperatorRequired` outcome once B5 merges. +- Idunn's phase deadlines. They landed in `idunn-reincarnation` cut 2, which + also owns the `operator-required` incident condition. - VoidBot's `notify_owner` MCP tool and its separate DM code. - Bifrost's Persona delivery crossing. - Eve/TUI lowering of incidents. diff --git a/src/bin/idunn-launch.rs b/src/bin/idunn-launch.rs new file mode 100644 index 0000000..3cf71e6 --- /dev/null +++ b/src/bin/idunn-launch.rs @@ -0,0 +1,96 @@ +//! The unit's process for an upstream daemon: parse the recipe's argv, read +//! what Idunn set, run the launcher, and map the outcome to an exit code. +#[cfg(unix)] +fn main() -> std::process::ExitCode { + use std::sync::atomic::{AtomicBool, Ordering}; + + use anyhow::Context; + use idunn_daemon::deployment::{ + IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, + IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT, + }; + use idunn_daemon::launcher::{self, *}; + + static STOP: AtomicBool = AtomicBool::new(false); + extern "C" fn on_signal(_: libc::c_int) { + STOP.store(true, Ordering::SeqCst); + } + + fn launch() -> anyhow::Result { + let release_root = std::env::current_exe()? + .parent() + .context("the launcher is not inside a release")? + .to_owned(); + let contract = LaunchContract::parse(release_root, std::env::args().skip(1))?; + let bundle = std::env::var_os(IDUNN_RUNTIME_BUNDLE_ENVIRONMENT) + .context("the Idunn runtime bundle is not set")?; + let bind = std::env::var(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT) + .context("the Idunn candidate bind is not set")?; + let member = launcher::parse_member_bind( + &std::env::var(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT) + .context("the Idunn member bind is not set")?, + &bind, + )?; + let (activation, provider) = launcher::inherited_authority_files()?; + let authority = RuntimeAuthority::open(bundle.as_ref(), activation, provider)?; + authority.require_candidate_bind(&bind)?; + let lease_path = std::env::var_os(IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT); + anyhow::ensure!( + lease_path.is_some() == authority.expected().write_lease_required, + "the process write lease binding differs from Expected" + ); + let route = std::net::TcpListener::bind(&bind).context("binding the candidate endpoint")?; + let sink = OdinRudpSink::from_expected(authority.expected())?; + let lease = DriverLeaseReader::new( + &authority.expected().target, + lease_path.as_deref().unwrap_or_default().as_ref(), + ); + let health_path = contract.health_path.clone(); + let ports = Ports { + credentials: std::sync::Arc::new(CredentialsDirectory::from_environment()), + clock: std::sync::Arc::new(SystemClock), + sink: std::sync::Arc::new(sink), + lease: std::sync::Arc::new(lease), + spawner: Box::new(ProcessSpawner), + health: Box::new(move |member| { + std::sync::Arc::new(HttpHealthProbe { + member, + path: health_path.clone(), + }) + }), + }; + // SAFETY: the handler only stores to an atomic. + unsafe { + libc::signal(libc::SIGTERM, on_signal as *const () as libc::sighandler_t); + libc::signal(libc::SIGINT, on_signal as *const () as libc::sighandler_t); + } + Ok(launcher::run( + &contract, + authority, + route, + member, + &std::env::temp_dir(), + ports, + &STOP, + )) + } + + match launch() { + Ok(outcome) => { + if let Outcome::Failed(code) = &outcome { + eprintln!("idunn-launch ended: {code}"); + } + std::process::ExitCode::from(outcome.exit_code() as u8) + } + Err(error) => { + eprintln!("idunn-launch refused: {error}"); + std::process::ExitCode::from(2) + } + } +} + +#[cfg(not(unix))] +fn main() { + eprintln!("idunn-launch runs only on Unix"); + std::process::exit(2); +} diff --git a/src/control_plane.rs b/src/control_plane.rs index 2c5fb3d..cd49073 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1,6 +1,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::env; use std::fs::{self, File, OpenOptions}; +use std::io::Write; use std::net::SocketAddr; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; @@ -34,16 +35,21 @@ use crate::deployment::{ capability_compatible, }; use crate::deployment_plan::{ - CompiledDeploymentPlan, DependencyProviderAuthority, SealedRelease, compile_deployment_plan, + CompiledDeploymentPlan, DependencyProviderAuthority, PRE_FENCING_ABORT_DEADLINE_SECONDS, + SealedRelease, WARMING_DEADLINE_SECONDS, compile_deployment_plan, }; use crate::drivers::{ ChallengeFailure, CultCacheTopologyDriver, CultCacheWriteLeaseDriver, DockerRunnerDriver, FrozenSourceReceipt, GitSourceDriver, InstalledReleaseObservation, IsolationEvidence, NginxRouteDriver, - ProcessIdentity, RouteActuation, RouteActuationGate, RouteActuationRefused, RouteActuators, - RouteObservation, RoutePreflightReceipt, RunnerPort, SourcePort, + LockContended, ProcessIdentity, RouteActuation, RouteActuationGate, RouteActuationRefused, RouteActuators, + RouteObservation, RoutePreflightReceipt, RunnerPort, RunnerStepFailure, SourcePort, SystemdTransientWorkloadDriver, TopologyPort, WorkloadObservation, WorkloadPort, WriteLeasePort, }; +use crate::incident::{CloseReason, IncidentCondition, IncidentStore, render_incidents}; +use crate::runner_evidence::{ + self, Kept, RunnerFailureRecord, runner_evidence_store_path, unreadable_store_path, +}; use crate::host_actuator::{ HostActuatorAccess, HostActuatorHub, HostActuatorRunnerDriver, HostActuatorWorkloadDriver, HostUnobservable, IdunnHostActuatorIdentity, SharedHostActuatorHub, spawn_hub_service, @@ -58,30 +64,43 @@ const ADMITTED_GENERATION_SCHEMA_V3: &str = "idunn.admitted_generation.v3"; const ADMITTED_GENERATION_SCHEMA_V2: &str = "idunn.admitted_generation.v2"; /// The capability whose declaration makes a target Odin-correlated. pub(crate) const ODIN_RENDEZVOUS_CAPABILITY: &str = "odin.verse-rendezvous"; -const TARGET_SUPERVISION_SCHEMA: &str = "idunn.target_supervision.v1"; +const TARGET_SUPERVISION_SCHEMA: &str = "idunn.target_supervision.v2"; +/// The seven-key layout written before continuity's streak, anchor and +/// re-materialization bound. Read and migrated by `read_supervision_record` +/// and `migrate_control_store_to_current_schema`; delete it, `LegacyTargetSupervisionV1` +/// and those two arms once every store has opened under a build that +/// migrates (the key-34 precedent's deletion line). +const TARGET_SUPERVISION_SCHEMA_V1: &str = "idunn.target_supervision.v1"; const EXPIRY_REQUEST_SCHEMA: &str = "idunn.expiry_request.v1"; -/// How many times continuity will restart one target inside one window before -/// it concludes the target itself is the problem. More than one because a -/// start can fail for a passing reason -- a port still held, a peer not yet up -/// -- and bounded because each attempt holds the target against any -/// deployment. Counted per target, so a restart that succeeds and dies again -/// does not start the count over. -const CONTINUITY_RESTART_ATTEMPTS: usize = 6; +/// The number of charged continuity attempts inside +/// `CONTINUITY_RESTART_WINDOW_MILLIS` at which the ContinuityExhausted incident +/// opens and is held. It gates no actuation: continuity never stops +/// (ruling budget-never-stop). +const CONTINUITY_PAGING_RESTARTS: usize = 6; const CONTINUITY_RESTART_WINDOW_MILLIS: u64 = 3_600_000; -/// The wait after the first restart in a window; it doubles with each further one. +/// The base of the streak backoff: the wait after the first charged attempt; +/// it doubles with each further one in the streak. const CONTINUITY_RESTART_BACKOFF_MILLIS: u64 = 5_000; +/// The longest streak backoff between charged continuity attempts (ruling +/// continuity-backoff-defaults). +const CONTINUITY_BACKOFF_CEILING_MILLIS: u64 = 300_000; +/// How long an admitted generation must have run, with no attempt failing on +/// it, before the attempt streak resets (ruling continuity-backoff-defaults). +const CONTINUITY_SUSTAINED_RUN_MILLIS: u64 = 600_000; /// Forward route actuations (fragment write, firewall, `nginx -t`, reload, /// private-mount validation) one target may perform per sliding window. A /// healthy target performs none; a legitimate deployment performs one or two. /// Only deployments count: a Survival actuation is recorded beside this /// ceiling, never counted against it and never refused by it, so a target -/// whose continuity spent its restarts can still be rescued by a deploy. -/// Survival is bounded elsewhere: a continuity's actuations by the restart log -/// times what one transaction can do before its phase deadline ends it (none, -/// over its admitted route), and an admitted-route repair by the challenge -/// backoff. A withdrawal that keeps failing retries, reloading each time, -/// under the resume backoff alone: the abort has no deadline. Nothing bounds +/// that is being restarted can still be rescued by a deploy. +/// Survival is bounded elsewhere: a continuity's actuations by the streak +/// backoff (at most one attempt per `CONTINUITY_BACKOFF_CEILING_MILLIS` once +/// the streak is long) times what one transaction can do before its phase +/// deadline ends it (none, over its admitted route), and an admitted-route +/// repair by the challenge backoff. A withdrawal that keeps failing retries, +/// reloading each time, under the resume backoff alone: the abort has no +/// deadline. Nothing bounds /// reloads across the host. const ROUTE_ACTUATION_CEILING: usize = 12; const ROUTE_ACTUATION_WINDOW_MILLIS: u64 = 3_600_000; @@ -910,8 +929,15 @@ impl LeaseAdoptionEvidence { } } -/// When the transaction entered its current post-fencing phase and when that -/// phase must end. Durations come from the plan's frozen `PhaseDeadlines`. +/// The phases whose entry starts a deadline: the one spelling of it. +const DEADLINE_PHASES: std::ops::RangeInclusive = + DeploymentPhase::Warming..=DeploymentPhase::Committing; + +/// What a live record is doing now and when that must end: a deadline phase +/// from its entry, an abort from its beginning, a drain to its retire. The +/// post-fencing durations come from the plan's frozen `PhaseDeadlines`; the +/// rest are Idunn's. `DeploymentTransaction::deadline_for_current_activity` +/// is the one derivation. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct PhaseDeadline { @@ -921,7 +947,8 @@ struct PhaseDeadline { } impl PhaseDeadline { - /// `None` for a phase that has no deadline (pre-fencing and Complete). + /// `None` for a phase that has no deadline of its own (Sealing, Starting + /// and Complete). fn entering( phase: DeploymentPhase, plan: &CompiledDeploymentPlan, @@ -929,6 +956,7 @@ impl PhaseDeadline { ) -> Option { let deadlines = plan.phase_deadlines(); let seconds = match phase { + DeploymentPhase::Warming => WARMING_DEADLINE_SECONDS, DeploymentPhase::Fencing => deadlines.fencing_seconds, DeploymentPhase::Leasing => deadlines.leasing_seconds, DeploymentPhase::AwaitingReady => deadlines.awaiting_ready_seconds, @@ -944,15 +972,24 @@ impl PhaseDeadline { } } -/// How the deadline resolver ends a post-fencing phase. +/// How the deadline resolver ends a deadline phase. #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum PhaseEnd { + /// Before the fence the transaction owns only its candidate: it ends + /// through its own pre-fencing abort. + AbortBeforeFencing, /// Abort through the post-fencing path. `lease_not_adopted` names a /// granted lease with no adoption evidence. Abort { lease_not_adopted: bool }, /// A stateful deployment whose candidate holds an issued lease: the phase /// is reported and not resolved. OperatorRequired, + /// A continuity at or past Fencing: its candidate is the only running copy + /// of the admitted body, so aborting it restores what it already runs. The + /// resolver leaves it to the operator and its phase step keeps running, + /// until a deployment awaits the handover, when the continuity yields + /// through its own post-fencing abort (ruling continuity-yields-to-deploy). + RetryForward, } /// How a post-fencing failure recovers. `RestoreIncumbent` is what a @@ -1034,7 +1071,7 @@ impl RouteSupervisionState { #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] #[cultcache( type = "idunn.target_supervision", - schema = "idunn.target_supervision.v1" + schema = "idunn.target_supervision.v2" )] struct TargetSupervision { #[cultcache(key = 0)] @@ -1045,7 +1082,8 @@ struct TargetSupervision { /// ascending: the only entries the ceiling counts. #[cultcache(key = 2)] route_actuations: Vec, - /// The newest `CONTINUITY_RESTART_ATTEMPTS` restart times, ascending. + /// The newest `CONTINUITY_PAGING_RESTARTS` restart times, ascending: the + /// paging threshold's log. #[cultcache(key = 3)] continuity_restarts: Vec, /// A restart whose projection could not be prepared waits until this time. @@ -1057,6 +1095,34 @@ struct TargetSupervision { /// Visibility only: never counted, never refused. #[cultcache(key = 6)] route_survivals: Vec, + // Keys 7-9 are why this is v2: a v1 record is lifted by + // `LegacyTargetSupervisionV1` with no streak, no anchor and no + // re-materialization. + /// Charged attempts since the last sustained run. Owned by continuity + /// supervision; it decides the next attempt's backoff step. + #[cultcache(key = 7)] + continuity_attempt_streak: u32, + /// When a dead-and-free pass began waiting out the next attempt. Owned by + /// continuity supervision; it decides when the wait is over. Older than + /// the admitted generation, it decides nothing. + #[cultcache(key = 8)] + continuity_backoff_from_unix_millis: Option, + /// When the admitted generation's one uncharged re-materialization was + /// minted. Owned by continuity supervision; it can only withhold a free + /// re-materialization, never grant one. + #[cultcache(key = 9)] + continuity_rematerialized_at_unix_millis: Option, +} + +/// What continuity supervision does about the next attempt for a dead target. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ContinuityBackoff { + /// Mint now, carrying this streak into the attempt. + Due { streak: u32 }, + /// Start the wait: write the anchor and mint nothing this pass. + Begin, + /// The wait has not run out. + Waiting, } /// The entries of a sliding log still inside `window` at `now`. An entry from @@ -1099,6 +1165,9 @@ impl TargetSupervision { continuity_deferred_until: None, continuity_deferral_reason: None, route_survivals: Vec::new(), + continuity_attempt_streak: 0, + continuity_backoff_from_unix_millis: None, + continuity_rematerialized_at_unix_millis: None, } } @@ -1152,45 +1221,121 @@ impl TargetSupervision { live_entries(&self.continuity_restarts, now, CONTINUITY_RESTART_WINDOW_MILLIS).len() } - fn restarts_exhausted(&self, now: u64) -> bool { - self.restarts_used(now) >= CONTINUITY_RESTART_ATTEMPTS + /// The paging threshold of the ContinuityExhausted incident; it gates no actuation. + fn restarts_page(&self, now: u64) -> bool { + self.restarts_used(now) >= CONTINUITY_PAGING_RESTARTS } - /// The next restart is due one doubling wait after the last, from the - /// number of restarts still inside the window. - fn next_restart_at(&self, now: u64) -> Option { - let live = live_entries(&self.continuity_restarts, now, CONTINUITY_RESTART_WINDOW_MILLIS); - if live.is_empty() { - return None; - } - let last = *self.continuity_restarts.last()?; - Some(last.saturating_add(CONTINUITY_RESTART_BACKOFF_MILLIS << (live.len() - 1))) - } - - /// Whether continuity must still wait: the doubling wait after the last - /// restart, or the deferral after a projection that would not demote. - fn continuity_is_waiting(&self, now: u64) -> bool { - self.next_restart_at(now).is_some_and(|due| { - is_waiting( - now, - due, - CONTINUITY_RESTART_BACKOFF_MILLIS << (CONTINUITY_RESTART_ATTEMPTS - 1), - ) - }) || self - .continuity_deferred_until + /// Whether continuity must still wait out the deferral after a projection + /// that would not demote. + fn continuity_deferred(&self, now: u64) -> bool { + self.continuity_deferred_until .is_some_and(|due| is_waiting(now, due, CONTINUITY_DEFERRAL_MILLIS)) } - /// One restart scheduled. It ends any deferral: the projection was prepared. - fn record_restart(&mut self, now: u64) { - record_entry(&mut self.continuity_restarts, now, CONTINUITY_RESTART_ATTEMPTS); + /// The doubling steps the next attempt waits, if it waits. A streak of `n` + /// waits `n - 1` steps. The generation's one re-materialization is an + /// attempt that failed on it though it charged no streak, so the attempt + /// after it waits the base step: every attempt leaves a gap. + fn owed_steps(&self, admitted_at: u64) -> Option { + match self.continuity_attempt_streak.checked_sub(1) { + None if self.rematerialization_spent(admitted_at) => Some(0), + steps => steps, + } + } + + /// When the wait for the next charged attempt ends, from the anchor and + /// the streak; the one spelling `continuity_backoff` and `status` share. + /// An anchor older than the admitted generation decides nothing. + fn backoff_due(&self, admitted_at: u64) -> Option { + let from = self + .continuity_backoff_from_unix_millis + .filter(|&from| from >= admitted_at)?; + let wait = self.owed_steps(admitted_at).map_or(0, |steps| { + backoff_wait( + CONTINUITY_RESTART_BACKOFF_MILLIS, + steps, + CONTINUITY_BACKOFF_CEILING_MILLIS, + ) + }); + Some(from.saturating_add(wait)) + } + + /// What to do about the next attempt for the dead admitted generation + /// admitted at `admitted_at`. The streak resets only after a sustained + /// run: the generation has lived the sustained time and no attempt, charged + /// or the one re-materialization, has been minted since it was admitted. + /// That rule decides first: a sustained death never began a wait, so a + /// standing anchor, however it came to stand, has no say in it. A wait + /// already running when the generation reaches the sustained time ends + /// there instead: an accepted shortening, always under the cap. + fn continuity_backoff(&self, now: u64, admitted_at: u64) -> ContinuityBackoff { + let streak = self.continuity_attempt_streak; + let sustained = self + .continuity_restarts + .last() + .into_iter() + .chain(&self.continuity_rematerialized_at_unix_millis) + .all(|&last| admitted_at >= last) + && now.saturating_sub(admitted_at) >= CONTINUITY_SUSTAINED_RUN_MILLIS; + if sustained { + return ContinuityBackoff::Due { streak: 0 }; + } + if let Some(due) = self.backoff_due(admitted_at) { + return if self.owed_steps(admitted_at).is_some() + && is_waiting(now, due, CONTINUITY_BACKOFF_CEILING_MILLIS) + { + ContinuityBackoff::Waiting + } else { + ContinuityBackoff::Due { streak } + }; + } + if self.owed_steps(admitted_at).is_none() { + ContinuityBackoff::Due { streak: 0 } + } else { + ContinuityBackoff::Begin + } + } + + /// One charged attempt scheduled at `streak`. It ends the wait and any + /// deferral: the projection was prepared. + fn record_restart(&mut self, now: u64, streak: u32) { + record_entry(&mut self.continuity_restarts, now, CONTINUITY_PAGING_RESTARTS); + self.continuity_attempt_streak = streak.saturating_add(1); + self.continuity_backoff_from_unix_millis = None; self.continuity_deferred_until = None; self.continuity_deferral_reason = None; } + /// The target was found dead and free: the wait for its next attempt runs from now. + fn begin_backoff(&mut self, now: u64) { + self.continuity_backoff_from_unix_millis = Some(now); + } + + /// The admitted workload was seen running: the death the anchor waited + /// out is over, and the next death waits from its own discovery. + fn end_backoff(&mut self) { + self.continuity_backoff_from_unix_millis = None; + } + + /// The one uncharged re-materialization scheduled. The log and the streak + /// are left alone. + fn record_rematerialization(&mut self, now: u64) { + self.continuity_rematerialized_at_unix_millis = Some(now); + self.continuity_backoff_from_unix_millis = None; + self.continuity_deferred_until = None; + self.continuity_deferral_reason = None; + } + + /// Whether the generation admitted at `admitted_at` has had its free re-materialization. + fn rematerialization_spent(&self, admitted_at: u64) -> bool { + self.continuity_rematerialized_at_unix_millis + .is_some_and(|at| at >= admitted_at) + } + fn defer_continuity(&mut self, now: u64, reason: &str) { self.continuity_deferred_until = Some(now.saturating_add(CONTINUITY_DEFERRAL_MILLIS)); - self.continuity_deferral_reason = Some(reason.to_owned()); + self.continuity_deferral_reason = Some(bounded_detail(reason)); } fn validate(&self) -> Result<()> { @@ -1201,7 +1346,7 @@ impl TargetSupervision { require_id(&self.target, "supervised target")?; for (log, limit, what) in [ (&self.route_actuations, ROUTE_ACTUATION_CEILING, "route actuation"), - (&self.continuity_restarts, CONTINUITY_RESTART_ATTEMPTS, "continuity restart"), + (&self.continuity_restarts, CONTINUITY_PAGING_RESTARTS, "continuity restart"), (&self.route_survivals, ROUTE_SURVIVAL_RECORD, "survival route actuation"), ] { ensure!(log.len() <= limit, "{what} log exceeds its bound"); @@ -1210,6 +1355,11 @@ impl TargetSupervision { "{what} log is not an ascending list of times" ); } + ensure!( + self.continuity_backoff_from_unix_millis != Some(0) + && self.continuity_rematerialized_at_unix_millis != Some(0), + "continuity backoff anchor or re-materialization has no time" + ); ensure!( self.continuity_deferred_until.is_some() == self.continuity_deferral_reason.is_some(), "continuity deferral has a time or a reason but not both" @@ -1560,43 +1710,219 @@ struct DeploymentTransaction { lease_adoption: Option, } +/// The note a deployment keeps while another transaction owns its target. +fn waiting_for_owner(target: &str, owner: &str) -> String { + format!("target {target} is owned by {owner}; this deployment fences what is admitted when it completes") +} + +/// What `Engine::take_target` decided about one transaction's entry into the +/// phase where it takes its target. +#[derive(Debug, PartialEq, Eq)] +enum Handover { + /// The CAS wrote the entry: the transaction owns the target. + Took, + /// The CAS lost to a concurrent write; nothing was written. + Lost, + /// Another transaction owns or drains the target; nothing was written. + OwnedBy(String), + /// The admitted generation is no longer the one the transaction names. + Moved, +} + +/// The phase whose entry is the handover of a target: Starting when the rollout +/// stops the incumbent first (it is stopped there by design), else Fencing. The +/// one spelling of the rule; a free function so the stop-then-start arm is +/// testable without a host-actuator fixture. +fn handover_phase(stops_incumbent_first: bool) -> DeploymentPhase { + if stops_incumbent_first { + DeploymentPhase::Starting + } else { + DeploymentPhase::Fencing + } +} + impl DeploymentTransaction { - /// The one writer of `phase`. It stamps the update time and the phase - /// deadline together, so a record outside Fencing..=Committing carries no - /// deadline by construction. No other code assigns `phase`. + /// The one writer of `phase`. It stamps the update time and, through + /// `retime`, the deadline of what the record now does. No other code + /// assigns `phase`. fn enter_phase(&mut self, phase: DeploymentPhase, now: u64) { self.phase = phase; self.updated_at_unix_millis = now; - self.phase_deadline = self - .plan - .as_ref() - .and_then(|plan| PhaseDeadline::entering(phase, plan, now)); + self.retime(now); } - /// A live transaction past the fence that is not already being aborted: - /// the only kind whose phase a deadline, or an operator's expiry, ends. + /// Whether the abort of the pre-fence path still has cleanup to do. + fn pre_fencing_abort_pending(&self) -> bool { + self.completion.is_none() + && self + .pre_fencing_abort + .as_ref() + .is_some_and(|abort| !abort.is_complete()) + } + + /// The one derivation of every duration the record is held to. (1) A + /// pending pre-fencing abort is timed from now by Idunn's constant. (2) A + /// draining Complete is timed from its incumbent's retire time (the route + /// promotion plus the drain, or now when nothing was promoted) for the + /// plan's committing duration. (3) Any other Complete has none. (4) The + /// rest take their phase's duration from now, so a post-fencing abort gets + /// the frozen duration of the phase it aborts, counted from its beginning. + fn deadline_for_current_activity(&self, now: u64) -> Option { + if self.pre_fencing_abort_pending() { + return Some(PhaseDeadline { + phase: self.phase, + entered_at_unix_millis: now, + deadline_at_unix_millis: now.saturating_add( + u64::from(PRE_FENCING_ABORT_DEADLINE_SECONDS) * 1000, + ), + }); + } + if self.phase == DeploymentPhase::Complete { + if !matches!(self.completion, Some(TransactionCompletion::Admitted { .. })) + || !self.drains_incumbent() + { + return None; + } + let plan = self.plan.as_ref()?; + // The retire time needs the binding's drain, which the running + // binary re-parses and re-admits: a plan frozen by an older binary + // may not parse now. That must not leave a drain with no deadline, + // so a retire time that cannot be derived is now, and the + // committing duration from it bounds the drain. + let from = self + .routing + .as_ref() + .and_then(RoutingEvidence::promoted_at_unix_millis) + .and_then(|promoted_at| { + let (_, binding) = plan.parsed_inputs().ok()?; + route_drain_deadline(promoted_at, binding.rollout.drain_seconds).ok() + }) + .unwrap_or(now); + return Some(PhaseDeadline { + phase: self.phase, + entered_at_unix_millis: now, + deadline_at_unix_millis: from + .max(now) + .saturating_add(u64::from(plan.phase_deadlines().committing_seconds) * 1000), + }); + } + PhaseDeadline::entering(self.phase, self.plan.as_ref()?, now) + } + + /// Stamp the deadline of what the record does now. The only non-test + /// writer that sets one: phase entry, an abort's beginning, the commit to a + /// draining Complete and the resolver's lift call it. + fn retime(&mut self, now: u64) { + self.phase_deadline = self.deadline_for_current_activity(now); + } + + /// Whether a deadline can describe the record's current activity: a + /// deadline phase, a pre-fencing abort not yet ended, or an admitted Complete. + fn carries_a_deadline(&self) -> bool { + // An abort carries its deadline until the record ends, including the + // write that completes its last cleanup: a Sealing or Starting record + // has no phase duration to fall back on, and that write is validated. + DEADLINE_PHASES.contains(&self.phase) + || (self.completion.is_none() && self.pre_fencing_abort.is_some()) + || (self.phase == DeploymentPhase::Complete + && matches!(self.completion, Some(TransactionCompletion::Admitted { .. }))) + } + + /// A live transaction in a deadline phase that is not already being + /// aborted: the only kind whose phase a deadline, or an operator's expiry, + /// ends. fn resolves_by_deadline(&self) -> bool { self.completion.is_none() && self.pre_fencing_abort.is_none() && self.post_fencing_abort.is_none() - && (DeploymentPhase::Fencing..=DeploymentPhase::Committing).contains(&self.phase) + && DEADLINE_PHASES.contains(&self.phase) } - /// What an ended post-fencing phase resolves to (Q3). A candidate that was - /// never issued a write lease cannot have written, so the transaction - /// aborts to the incumbent through the post-fencing path, stateless or not. - /// A continuity always aborts: its candidate is the admitted release, and - /// supervision restarts that release under the continuity backoff. A - /// deployment whose candidate was issued a lease may have written under a - /// new state contract, and restoring the incumbent over that is not - /// Idunn's call: it is left for the operator. + /// Whether Idunn has run out of ways to end this record and leaves it to + /// the operator: its deadline has passed in an abort that cannot finish, a + /// drain whose stop keeps failing, or a stateful deploy. Derived, never + /// stored. The step goes on retrying and the target stays owned. + fn leaves_to_operator(&self, now: u64) -> bool { + if self.is_terminal() { + return false; + } + let Some(deadline) = &self.phase_deadline else { + return false; + }; + now >= deadline.deadline_at_unix_millis + && (self.pre_fencing_abort.is_some() + || self.post_fencing_abort.is_some() + || self.drains_incumbent() + || (self.resolves_by_deadline() + && matches!(self.phase_end(), PhaseEnd::OperatorRequired | PhaseEnd::RetryForward))) + } + + /// Whether an operator's expiry can end this record's phase: it resolves by + /// deadline, and is not a continuity past Fencing, whose expiry abort would + /// stop the only running copy of the admitted body (ruling + /// c2c-expire-refuses-continuity-past-fencing). + fn expirable(&self) -> bool { + self.resolves_by_deadline() + && !(self.command_kind == CommandKind::Continuity && self.phase >= DeploymentPhase::Fencing) + } + + /// The operator's only way to restart a forward-retrying continuity, worded + /// from the driver its plan binds: a systemd target has a unit to stop (a + /// stopped transient unit is forgotten, which is permanently stopped); a + /// host-actuator target has no unit, only a process on the managed host + /// that the actuator then reports not running. + fn manual_exit(&self) -> &'static str { + match self + .plan + .as_ref() + .and_then(|plan| plan.parsed_inputs().ok()) + .map(|(_, binding)| binding.workload) + { + Some(WorkloadBinding::HostActuator(_)) => "kill its process on the managed host by hand", + _ => "stop its unit by hand", + } + } + + /// The candidate's route proof has succeeded: it is Committing, or its + /// routing evidence is recorded and only the transition remains. It serves, + /// so no waiting deployment stops it (ruling c2d-yield-only-unproven-to- + /// movable). Derived, never stored. + fn serving_proven(&self) -> bool { + self.phase >= DeploymentPhase::Committing || self.routing.is_some() + } + + /// A continuity at or past Fencing that its deadline leaves to the operator + /// while its step runs, and whose candidate is still unproven: it yields to + /// a deployment that can take the handover (ruling continuity-yields-to- + /// deploy), so a waiter never ends on it. A proven continuity keeps serving + /// and paging and does not yield. Derived, never stored. + fn yields_to_a_deploy(&self) -> bool { + self.resolves_by_deadline() + && self.phase_end() == PhaseEnd::RetryForward + && !self.serving_proven() + } + + /// What an ended post-fencing phase resolves to (Q3). A continuity at or + /// past Fencing retries forward: its incumbent is the admitted generation, + /// which supervision mints a continuity only after seeing dead, and from + /// Fencing the continuity owns the target, so aborting restores the body + /// its candidate already runs (ruling c2c-retry-decided-by-record). A + /// deployment whose candidate was never issued a write lease cannot have + /// written, so it aborts to the incumbent through the post-fencing path, + /// stateless or not. One whose candidate was issued a lease may have + /// written under a new state contract, and restoring the incumbent over + /// that is not Idunn's call: it is left for the operator. fn phase_end(&self) -> PhaseEnd { + if self.phase < DeploymentPhase::Fencing { + return PhaseEnd::AbortBeforeFencing; + } let leasing = self.leasing.as_ref(); let granted = leasing.and_then(LeasingEvidence::lease).is_some(); let issued = leasing.and_then(LeasingEvidence::issued_lease).is_some(); match self.command_kind { + CommandKind::Continuity => PhaseEnd::RetryForward, CommandKind::Deploy if issued => PhaseEnd::OperatorRequired, - _ => PhaseEnd::Abort { + CommandKind::Deploy => PhaseEnd::Abort { lease_not_adopted: granted && self.lease_adoption.is_none(), }, } @@ -1730,7 +2056,7 @@ impl DeploymentTransaction { fn rejected(command: &DeploymentCommand, error: anyhow::Error, now: u64) -> Result { let mut transaction = Self::new(command, command.selector.clone(), 0, None, now)?; - let detail = truncate(&format!("{error:#}"), 2048); + let detail = bounded_detail(&format!("{error:#}")); transaction.enter_phase(DeploymentPhase::Complete, now); transaction.last_error = Some(detail.clone()); transaction.pre_fencing_abort = Some(PreFencingAbort { @@ -1765,21 +2091,47 @@ impl DeploymentTransaction { } } - /// Complete transactions retain retryable cleanup work, but admission has - /// already transferred current-incarnation authority to AdmittedGeneration. - fn owns_target_authority(&self) -> bool { + /// The phase whose entry is the handover: the one transaction that makes + /// it takes the target's incumbent slot from continuity supervision. + fn takes_target_at(&self) -> DeploymentPhase { + handover_phase(self.rollout_stops_incumbent_first()) + } + + /// Not Complete: the transaction still holds its own candidate slot. + /// Read by sibling order inside one command, freeze (a live Deploy + /// refuses a second), the mint gate (a live Continuity refuses a second), + /// candidate-port reservation and validation. + fn is_live(&self) -> bool { self.phase != DeploymentPhase::Complete } + /// A live deployment whose next move is the handover: not ended, not + /// aborting, one phase short of `takes_target_at`. Derived, never stored. + /// Read by `deploy_the_continuity_yields_to`: the continuity's yield and the + /// boot lift. + fn awaits_handover(&self) -> bool { + self.command_kind == CommandKind::Deploy + && self.completion.is_none() + && self.pre_fencing_abort.is_none() + && self.phase as u8 + 1 == self.takes_target_at() as u8 + } + + /// From the handover until Complete the transaction owns the target's + /// incumbent slot: it fences, leases, routes and commits. Derived from the + /// phase, never stored. Read by validation (one owner), supervision (which + /// defers to an owner), the handover CAS, a waiting deployment and status. + fn owns_target(&self) -> bool { + (self.takes_target_at()..DeploymentPhase::Complete).contains(&self.phase) + } + /// Admission has moved to the new generation at Complete, but an exact /// draining incumbent still reserves its process and candidate endpoint. - /// No later mutation for the same target may overlap that cleanup. - fn blocks_new_target_mutation(&self) -> bool { - self.owns_target_authority() - || self - .post_commit_cleanup - .as_ref() - .is_some_and(|cleanup| !cleanup.is_complete()) + /// Read by the mint gate, the handover, candidate-port reservation and + /// status: no later mutation of the target may overlap that cleanup. + fn drains_incumbent(&self) -> bool { + self.post_commit_cleanup + .as_ref() + .is_some_and(|cleanup| !cleanup.is_complete()) } /// Where what this transaction has collected disagrees with what its own @@ -1976,11 +2328,10 @@ impl DeploymentTransaction { if let Some(deadline) = &self.phase_deadline { ensure!( deadline.phase == self.phase - && (DeploymentPhase::Fencing..=DeploymentPhase::Committing) - .contains(&deadline.phase) + && self.carries_a_deadline() && deadline.entered_at_unix_millis > 0 && deadline.deadline_at_unix_millis > deadline.entered_at_unix_millis, - "phase deadline does not describe the current post-fencing phase" + "phase deadline does not describe the record's current activity" ); } if let Some(adoption) = &self.lease_adoption { @@ -2505,6 +2856,9 @@ struct RuntimeOptions { source_root: PathBuf, staging_root: PathBuf, topology_store: PathBuf, + /// Operator incidents, published world-readable beside the topology + /// projection. Not part of the control store: see `incident`. + incident_store: PathBuf, odin_correlation_store: PathBuf, odin_trust_anchor: PathBuf, idunn_identity_store: PathBuf, @@ -2530,6 +2884,7 @@ impl Default for RuntimeOptions { source_root: PathBuf::from("/var/lib/gamecult/idunn/sources"), staging_root: PathBuf::from("/var/lib/gamecult/idunn/staging"), topology_store: PathBuf::from("/var/lib/gamecult/idunn/topology.cc"), + incident_store: PathBuf::from("/var/lib/gamecult/idunn-projection/incidents.cc"), odin_correlation_store: PathBuf::from( "/var/lib/gamecult/odin/idunn-runtime-topology.cc", ), @@ -2562,6 +2917,7 @@ enum Command { }, Status { state_store: PathBuf, + incident_store: PathBuf, command_id: Option, }, Cancel { @@ -2578,6 +2934,10 @@ enum Command { recipe: PathBuf, binding: Option, }, + #[cfg(unix)] + OrdersServe(crate::steward_orders::ServeOptions), + #[cfg(unix)] + Order(crate::steward_orders::OrderCommand), } pub fn run(args: impl Iterator) -> Result<()> { @@ -2598,8 +2958,14 @@ pub fn run(args: impl Iterator) -> Result<()> { ), Command::Status { state_store, + incident_store, command_id, - } => status(&state_store, command_id.as_deref()), + } => status( + &mut std::io::stdout().lock(), + &state_store, + &incident_store, + command_id.as_deref(), + ), Command::Cancel { state_store, command_id, @@ -2611,6 +2977,10 @@ pub fn run(args: impl Iterator) -> Result<()> { requested_by, } => expire(&state_store, &command_id, &requested_by), Command::Validate { recipe, binding } => validate(&recipe, binding.as_deref()), + #[cfg(unix)] + Command::OrdersServe(options) => crate::steward_orders::serve(options), + #[cfg(unix)] + Command::Order(order) => crate::steward_orders::run_order(order), } } @@ -2663,6 +3033,10 @@ fn parse(args: impl Iterator) -> Result { "cancel" => parse_cancel(args), "expire" => parse_expire(args), "validate" => parse_validate(args), + #[cfg(unix)] + "orders" => crate::steward_orders::parse_serve(args).map(Command::OrdersServe), + #[cfg(unix)] + "order" => crate::steward_orders::parse_order(args).map(Command::Order), "--help" | "-h" | "help" => bail!(usage()), _ => bail!("unknown Idunn command {command:?}\n\n{}", usage()), } @@ -2679,6 +3053,7 @@ fn parse_serve(mut args: impl Iterator) -> Result { "--source-root" => options.source_root = path_value(&mut args, &argument)?, "--staging-root" => options.staging_root = path_value(&mut args, &argument)?, "--topology-store" => options.topology_store = path_value(&mut args, &argument)?, + "--incident-store" => options.incident_store = path_value(&mut args, &argument)?, "--odin-correlation-store" => { options.odin_correlation_store = path_value(&mut args, &argument)? } @@ -2762,11 +3137,14 @@ fn parse_up(mut args: impl Iterator) -> Result { } fn parse_status(mut args: impl Iterator) -> Result { - let mut state_store = RuntimeOptions::default().state_store; + let defaults = RuntimeOptions::default(); + let mut state_store = defaults.state_store; + let mut incident_store = defaults.incident_store; let mut command_id = None; while let Some(argument) = args.next() { match argument.as_str() { "--state-store" => state_store = path_value(&mut args, &argument)?, + "--incident-store" => incident_store = path_value(&mut args, &argument)?, "--command" => command_id = Some(string_value(&mut args, &argument)?), "--help" | "-h" => bail!(usage()), _ => bail!("unknown Idunn status option {argument:?}"), @@ -2774,6 +3152,7 @@ fn parse_status(mut args: impl Iterator) -> Result { } Ok(Command::Status { state_store, + incident_store, command_id, }) } @@ -2821,8 +3200,16 @@ fn expire(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> "command has multiple live transactions" ); ensure!( - current.value.resolves_by_deadline(), - "only a live transaction past the fence, and not already aborting, can be expired; before the fence, use idunn cancel" + current.value.expirable(), + "{}", + if current.value.command_kind == CommandKind::Continuity && current.value.phase >= DeploymentPhase::Fencing && current.value.post_fencing_abort.is_none() { + format!( + "a continuity past Fencing retries forward and cannot be expired; to restart it, {} and Idunn aborts it as dead", + current.value.manual_exit() + ) + } else { + "only a live transaction in Warming through Committing, and not already aborting, can be expired; before Warming, use idunn cancel".to_owned() + } ); let now = now_millis()?; let request = ExpiryRequest { @@ -2901,7 +3288,7 @@ fn cancel(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> } else { next.post_fencing_abort = Some(post_fencing_abort_intent(&next, &error)); } - next.last_error = Some(error); + next.last_error = Some(bounded_detail(&error)); next.updated_at_unix_millis = now_millis()?; replace_transaction(store_path, current, &next)?; println!("{command_id} cancellation requested; Idunn is performing exact candidate cleanup"); @@ -3029,16 +3416,7 @@ impl ControlSnapshot { snapshot.admitted.push(Stored { envelope, value }); } TargetSupervision::TYPE => { - ensure!( - envelope.schema_id.as_deref() == Some(TARGET_SUPERVISION_SCHEMA), - "Idunn control store contains an unsupported target supervision record" - ); - let value: TargetSupervision = decode_record(&envelope)?; - value.validate()?; - ensure!( - envelope.key == value.target, - "target supervision key is not its target" - ); + let value = read_supervision_record(&envelope)?; snapshot.targets.push(Stored { envelope, value }); } ExpiryRequest::TYPE => { @@ -3109,16 +3487,27 @@ impl ControlSnapshot { "transaction kind differs from its immutable command" ); } - let mut live_targets = BTreeSet::new(); + // Per target: one owner of the incumbent slot, one live Deploy, one + // live Continuity. A continuity may sit beside a deployment that has + // not yet taken the target (handover-phase-split). + let mut owners = BTreeSet::new(); + let mut live_deploys = BTreeSet::new(); + let mut live_continuities = BTreeSet::new(); for transaction in self .transactions .iter() - .filter(|stored| stored.value.owns_target_authority()) + .filter(|stored| stored.value.is_live()) { + let target = transaction.value.target.as_str(); ensure!( - live_targets.insert(transaction.value.target.as_str()), - "multiple transactions claim current-incarnation authority for one target" + !transaction.value.owns_target() || owners.insert(target), + "multiple transactions own the incumbent slot of one target" ); + let (seen, kind) = match transaction.value.command_kind { + CommandKind::Deploy => (&mut live_deploys, "deployments"), + CommandKind::Continuity => (&mut live_continuities, "continuity transactions"), + }; + ensure!(seen.insert(target), "multiple live {kind} for one target"); if let Some(fencing) = transaction .value .fencing @@ -3212,6 +3601,43 @@ impl ControlSnapshot { .find(|stored| stored.value.target == target) } + /// The transaction, other than `except`, that holds the target's + /// incumbent slot or is still draining it. The one lookup supervision, a + /// waiting deployment and the handover share. + fn target_owner(&self, target: &str, except: Option<&str>) -> Option<&Stored> { + self.on_target(target).find(|stored| { + Some(stored.value.transaction_id.as_str()) != except + && (stored.value.owns_target() || stored.value.drains_incumbent()) + }) + } + + /// Every transaction of the target, in whatever state. + fn on_target<'a>( + &'a self, + target: &str, + ) -> impl Iterator> + use<'a> { + let target = target.to_owned(); + self.transactions + .iter() + .filter(move |stored| stored.value.target == target) + } + + /// The candidate ports taken by admitted generations and by transactions + /// that are live or still draining; a finished one has released its port. + fn candidate_ports_in_use(&self) -> BTreeSet { + self.admitted + .iter() + .map(|stored| &stored.value.plan) + .chain( + self.transactions + .iter() + .filter(|stored| stored.value.is_live() || stored.value.drains_incumbent()) + .filter_map(|stored| stored.value.plan.as_ref()), + ) + .filter_map(|plan| plan.candidate_port) + .collect() + } + fn supervision_for(&self, target: &str) -> Option<&Stored> { self.targets .iter() @@ -3248,7 +3674,7 @@ impl ControlSnapshot { fn has_earlier_authority_sibling(&self, transaction: &DeploymentTransaction) -> bool { self.transactions.iter().any(|stored| { - stored.value.owns_target_authority() + stored.value.is_live() && stored.value.command_id == transaction.command_id && stored.value.ordinal < transaction.ordinal }) @@ -3303,7 +3729,7 @@ impl ControlSnapshot { } } -fn decode_record(envelope: &CultCacheEnvelope) -> Result +pub(crate) fn decode_record(envelope: &CultCacheEnvelope) -> Result where T: for<'de> Deserialize<'de> + Serialize, { @@ -3748,6 +4174,74 @@ fn read_generation_record(envelope: &CultCacheEnvelope) -> Result, + #[cultcache(key = 3)] + continuity_restarts: Vec, + #[cultcache(key = 4)] + continuity_deferred_until: Option, + #[cultcache(key = 5)] + continuity_deferral_reason: Option, + #[cultcache(key = 6)] + route_survivals: Vec, +} + +impl LegacyTargetSupervisionV1 { + /// No streak, no anchor, no re-materialization: the same defaults a fresh + /// record starts with. + fn into_current(self) -> TargetSupervision { + TargetSupervision { + schema_version: TARGET_SUPERVISION_SCHEMA.into(), + target: self.target, + route_actuations: self.route_actuations, + continuity_restarts: self.continuity_restarts, + continuity_deferred_until: self.continuity_deferred_until, + continuity_deferral_reason: self.continuity_deferral_reason, + route_survivals: self.route_survivals, + continuity_attempt_streak: 0, + continuity_backoff_from_unix_millis: None, + continuity_rematerialized_at_unix_millis: None, + } + } +} + +/// Read one stored target supervision record, lifting a v1 record to the +/// current shape by the same rule as `read_generation_record`. One schema id +/// names one shape: a v1 record that carries ten keys, or a v2 record that +/// carries seven, is refused. +fn read_supervision_record(envelope: &CultCacheEnvelope) -> Result { + let value = match envelope.schema_id.as_deref() { + Some(TARGET_SUPERVISION_SCHEMA) => decode_record::(envelope)?, + Some(TARGET_SUPERVISION_SCHEMA_V1) => { + let legacy: LegacyTargetSupervisionV1 = decode_record(envelope) + .context("decoding a v1 target supervision record")?; + ensure!( + legacy.schema_version == TARGET_SUPERVISION_SCHEMA_V1, + "stored supervision schema differs from its envelope" + ); + legacy.into_current() + } + _ => bail!("Idunn control store contains an unsupported target supervision record"), + }; + value.validate()?; + ensure!( + envelope.key == value.target, + "target supervision key is not its target" + ); + Ok(value) +} + /// Rewrite every older record as current, once, before the engine runs. /// /// The read path lifts older records on its own, so this is convergence rather @@ -3776,6 +4270,8 @@ fn migrate_control_store_to_current_schema(store_path: &Path) -> Result { schema, Some(ADMITTED_GENERATION_SCHEMA_V2 | ADMITTED_GENERATION_SCHEMA_V3) )) + || (envelope.r#type == TargetSupervision::TYPE + && schema == Some(TARGET_SUPERVISION_SCHEMA_V1)) }) .collect::>(); let mut migrated = 0; @@ -3783,6 +4279,10 @@ fn migrate_control_store_to_current_schema(store_path: &Path) -> Result { let (record_type, key, mut next) = if envelope.r#type == DeploymentTransaction::TYPE { let (key, next) = cleanup_evidence::migrate_transaction_record(&envelope)?; (DeploymentTransaction::TYPE, key, next) + } else if envelope.r#type == TargetSupervision::TYPE { + let value = read_supervision_record(&envelope)?; + let next = target_supervision_envelope(&value, now_millis()?)?; + (TargetSupervision::TYPE, value.target, next) } else { let value = read_generation_record(&envelope)?; let next = admitted_envelope(&value, value.admitted_at_unix_millis)?; @@ -3846,7 +4346,7 @@ fn target_supervision_envelope(value: &TargetSupervision, now: u64) -> Result( +pub(crate) fn typed_envelope( key: &str, record_type: &str, schema: &str, @@ -3982,10 +4482,10 @@ fn read_history_transactions(state_store: &Path) -> Vec { /// How long to wait after `failures` consecutive failures: one poll interval /// after the first, doubling each time, never past the ceiling. -fn backoff_wait(poll_millis: u64, failures: u32) -> u64 { - poll_millis +fn backoff_wait(base_millis: u64, failures: u32, ceiling_millis: u64) -> u64 { + base_millis .saturating_mul(1u64 << failures.min(20)) - .min(RESUME_BACKOFF_CEILING_MILLIS) + .min(ceiling_millis) } /// Whether an attempt due at `not_before` must still wait at `now`. A due time @@ -4004,8 +4504,12 @@ fn state_advanced(before: &DeploymentTransaction, after: &DeploymentTransaction) &after != before } -/// Says a fault once while it lasts. A scheduler tick that hits the same +/// Says a fault once on stderr while it lasts. A scheduler tick that hits the same /// unreadable file every half second must not print it every half second. +/// +/// It is a trace and owns nothing else: it is in memory, keyed by free-form +/// strings, and an Idunn restart forgets it. Whether the operator is told +/// belongs to the incident record (`incident`), never to this map. #[derive(Default)] struct ReportOnce { last: Mutex>, @@ -4238,20 +4742,21 @@ fn resolve_selector( Ok(targets) } -fn submit( +/// The one commit primitive for a deployment command: `idunn up` and a steward +/// deploy order both write theirs here. Returns the commit time. +pub(crate) fn commit_deployment_command( store_path: &Path, + command_id: &str, selector: &str, requested_by: &str, - wait: bool, - timeout_seconds: u64, -) -> Result<()> { +) -> Result { if let Some(parent) = store_path.parent() { fs::create_dir_all(parent)?; } let now = now_millis()?; let command = DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: format!("up-{}", Uuid::new_v4()), + command_id: command_id.into(), kind: CommandKind::Deploy, selector: selector.into(), requested_by: requested_by.into(), @@ -4269,7 +4774,33 @@ fn submit( )?, "deployment command id collided" ); - println!("{}", command.command_id); + Ok(now) +} + +/// A command's present state as `idunn status --command` derives it, for the +/// steward order poster: `queued` and `running` are not terminal. +#[cfg(unix)] +pub(crate) fn command_state( + state_store: &Path, + command_id: &str, +) -> Result<(&'static str, String)> { + let snapshot = ControlSnapshot::read(state_store)?; + let archived = read_history_transactions(state_store); + let mut transactions = snapshot.transaction_for_command(command_id); + transactions.extend(archived.iter().filter(|value| value.command_id == command_id)); + Ok(derived_command_status(&transactions)) +} + +fn submit( + store_path: &Path, + selector: &str, + requested_by: &str, + wait: bool, + timeout_seconds: u64, +) -> Result<()> { + let command_id = format!("up-{}", Uuid::new_v4()); + let now = commit_deployment_command(store_path, &command_id, selector, requested_by)?; + println!("{command_id}"); if !wait { return Ok(()); } @@ -4280,10 +4811,10 @@ fn submit( snapshot .commands .iter() - .any(|stored| stored.value.command_id == command.command_id), + .any(|stored| stored.value.command_id == command_id), "submitted deployment command disappeared" ); - let transactions = snapshot.transaction_for_command(&command.command_id); + let transactions = snapshot.transaction_for_command(&command_id); if !transactions.is_empty() && transactions.iter().all(|value| value.is_terminal()) { if let Some(error) = transactions @@ -4308,7 +4839,12 @@ fn submit( } } -fn status(store_path: &Path, command_id: Option<&str>) -> Result<()> { +pub(crate) fn status( + out: &mut impl Write, + store_path: &Path, + incident_store: &Path, + command_id: Option<&str>, +) -> Result<()> { let snapshot = ControlSnapshot::read(store_path)?; // Commands stay resident; their finished transactions do not. Without // history a completed command would report as though it had never run, @@ -4336,10 +4872,11 @@ fn status(store_path: &Path, command_id: Option<&str>) -> Result<()> { ); transactions.sort_by_key(|value| value.ordinal); let (state, detail) = derived_command_status(&transactions); - println!( + writeln!( + out, "{} {} {} {}", command.command_id, command.selector, state, detail - ); + )?; // Naming one command asks about that command, so print what a stuck // transaction is actually waiting on. A gate reason lives in // `last_error` and was never rendered anywhere, which left "Sealing" @@ -4348,67 +4885,93 @@ fn status(store_path: &Path, command_id: Option<&str>) -> Result<()> { // ids are here because they are exactly what a brake release must name. if command_id.is_some() { for transaction in transactions { - println!(" transaction {}", transaction.transaction_id); - println!( + writeln!(out, " transaction {}", transaction.transaction_id)?; + writeln!( + out, " target {} phase {:?}", transaction.target, transaction.phase - ); + )?; if let Some(expected) = &transaction.expected { - println!(" runtime {}", expected.runtime_id); - println!(" release {}", expected.sealed_release_id); + writeln!(out, " runtime {}", expected.runtime_id)?; + writeln!(out, " release {}", expected.sealed_release_id)?; } - println!( + writeln!( + out, " odin publisher cursor {}", transaction.odin_publisher_sequence_cursor - ); + )?; if let Some(evidence) = &transaction.latest_odin_observation { - println!( + writeln!( + out, " latest odin observation sequence {}", evidence.publisher_sequence - ); + )?; } if let Some(reason) = &transaction.last_error { - println!(" waiting on {reason}"); + writeln!(out, " waiting on {reason}")?; } // A Complete transaction that is not terminal still owns its // target through unfinished cleanup. Without this an operator // sees "Complete" and a stale reason and cannot tell what is // holding the target. if let Some(completion) = &transaction.completion { - println!(" completion {completion:?}"); + writeln!(out, " completion {completion:?}")?; } if let Some(abort) = &transaction.pre_fencing_abort { - println!(" pre-fencing abort {abort:?}"); + writeln!(out, " pre-fencing abort {abort:?}")?; + // The step that failed and its output tail live in their + // own private store. An unreadable store costs its lines. + runner_evidence::show_status( + out, + &runner_evidence_store_path(store_path), + &transaction.transaction_id, + )?; } if let Some(abort) = &transaction.post_fencing_abort { - println!(" post-fencing abort {abort:?}"); + writeln!(out, " post-fencing abort {abort:?}")?; } if let Some(cleanup) = &transaction.post_commit_cleanup { - println!(" post-commit cleanup {cleanup:?}"); + writeln!(out, " post-commit cleanup {cleanup:?}")?; } - println!( - " terminal {} owns-target {}", + writeln!( + out, + " terminal {} owns-target {} drains {}", transaction.is_terminal(), - transaction.blocks_new_target_mutation() - ); + transaction.owns_target(), + transaction.drains_incumbent() + )?; } } } // What supervision holds for each target: the union of the admitted // generations and the metered targets, so a target that has only been // deployed to (no generation yet) still shows what it has used. - let now = now_millis()?; - for target in supervised_targets(&snapshot) { - for line in render_supervision( + for line in render_targets(&snapshot, incident_store, now_millis()?) { + writeln!(out, "{line}")?; + } + Ok(()) +} + +/// Every supervised target with its meters and its incidents. An incident +/// store that cannot be read costs the incident lines and nothing else. +fn render_targets(snapshot: &ControlSnapshot, incident_store: &Path, now: u64) -> Vec { + let incidents = IncidentStore::new(incident_store).read(); + let mut lines = Vec::new(); + for target in supervised_targets(snapshot) { + lines.extend(render_supervision( target, snapshot.admitted_for(target).map(|stored| &stored.value), snapshot.supervision_for(target).map(|stored| &stored.value), now, - ) { - println!("{line}"); + )); + if let Ok(records) = &incidents { + lines.extend(render_incidents(records, target, now)); } } - Ok(()) + if let Err(error) = &incidents { + lines.push(format!("incidents unreadable: {}", error.root_cause())); + } + lines } /// Every target supervision holds something for: the admitted generations @@ -4440,11 +5003,15 @@ fn render_supervision( lines.push(format!(" held: {reason}")); } lines.push(format!( - " continuity restarts {}/{} next-restart-at {}", + " continuity attempts-in-hour {} pages-at {} streak {} next-attempt-at {}", meters.restarts_used(now), - CONTINUITY_RESTART_ATTEMPTS, - at(meters.next_restart_at(now)) + CONTINUITY_PAGING_RESTARTS, + meters.continuity_attempt_streak, + at(generation.and_then(|value| meters.backoff_due(value.admitted_at_unix_millis))) )); + if let Some(rematerialized) = meters.continuity_rematerialized_at_unix_millis { + lines.push(format!(" continuity rematerialized-at {rematerialized}")); + } if let (Some(until), Some(reason)) = ( meters.continuity_deferred_until, &meters.continuity_deferral_reason, @@ -4598,6 +5165,10 @@ struct Engine { docker_runner: DockerRunnerDriver, systemd_workload: Arc, host_actuators: Option, + /// A test world that serves no host actuator runs a host-actuator + /// binding's workload through this port instead. + #[cfg(test)] + host_workload: Mutex>>, history_report: ReportOnce, /// One `ReportOnce` per faulting record, so one wedged transaction says /// its fault once and never drowns another's. @@ -4607,6 +5178,14 @@ struct Engine { /// Process-local on purpose: a restart retries at once, and no schema /// carries a clock that only pacing reads. resume_backoff: Mutex>, + /// Continuity aborts the boot lift could not decide (their candidate could + /// not be observed): no step of them runs until `retry_held_lift` decides. + /// Process-local: a restart asks again at boot. + lift_held: Mutex>, + /// Set when the boot lift itself failed, so no record's decision exists: + /// every continuity with a post-fencing abort is held until a later lift + /// succeeds. Process-local, like `lift_held`. + lift_unsettled: std::sync::atomic::AtomicBool, /// When each routed target's next challenge is due, held only when the /// stored `RouteSupervisionState` could not record the last one. The stored /// value owns the schedule and survives restarts; this spaces attempts @@ -4642,11 +5221,26 @@ impl RouteActuationGate for EngineRouteGate<'_> { struct ResumeBackoff { failures: u32, not_before_unix_millis: u64, + /// The longest wait this backoff was armed with: the skip check reads it, + /// so the check and the wait cannot disagree. + ceiling_millis: u64, } /// Longest wait between attempts at a failing transaction step. const RESUME_BACKOFF_CEILING_MILLIS: u64 = 60_000; +/// The longest wait between attempts at one transaction's step: the continuity +/// cap for a record left to the operator, which retries on the slow cadence its +/// page promises (ruling c2c-forward-retry-cadence), and the shorter one for +/// every other. +fn resume_ceiling(transaction: &DeploymentTransaction, now: u64) -> u64 { + if transaction.leaves_to_operator(now) { + CONTINUITY_BACKOFF_CEILING_MILLIS + } else { + RESUME_BACKOFF_CEILING_MILLIS + } +} + impl Engine { fn open(options: RuntimeOptions) -> Result { Self::open_with_systemd_workload( @@ -4699,9 +5293,13 @@ impl Engine { docker_runner: DockerRunnerDriver::default(), systemd_workload, host_actuators, + #[cfg(test)] + host_workload: Mutex::default(), history_report: ReportOnce::default(), fault_reports: Mutex::default(), resume_backoff: Mutex::default(), + lift_held: Mutex::default(), + lift_unsettled: std::sync::atomic::AtomicBool::new(false), challenge_wait: Mutex::default(), }) } @@ -4821,6 +5419,82 @@ impl Engine { } } + fn incidents(&self) -> IncidentStore { + IncidentStore::new(&self.options.incident_store) + } + + /// A failed runner step keeps its output tail in the runner evidence + /// store and says so on the journal, naming no output. The store write is + /// best effort: its failure is named in the line and never stops the + /// abort that follows. + fn keep_runner_failure( + &self, + transaction: &DeploymentTransaction, + error: &anyhow::Error, + ) -> Option { + let failure = error.downcast_ref::()?; + let path = runner_evidence_store_path(&self.options.state_store); + let recorded = now_millis().and_then(|now| { + runner_evidence::record( + &path, + &RunnerFailureRecord::from_failure( + &transaction.transaction_id, + &transaction.command_id, + &transaction.target, + failure, + now, + ), + ) + }); + Some(runner_failure_journal_line(transaction, failure, &path, &recorded)) + } + + /// Settle the outcome of one incident operation. A failure never leaves + /// this function: it is traced once per episode, under the operation and + /// the subject it was decided for (`*` for a pass over the whole store), + /// and continuity decides exactly as it would have with a healthy store. + /// Each decision site has its own key, so one site's success never ends + /// another site's episode. The message names the operation and the root + /// cause and never a path or a value. Incident writes are never progress. + fn record_incident(&self, operation: &str, subject: &str, outcome: Result) { + let key = format!("incident-store:{operation}:{subject}"); + match outcome { + Ok(_) => self.clear_fault(&key), + Err(error) => self.report_once( + &key, + format!( + "Idunn could not {operation} an operator incident: {}", + error.root_cause() + ), + ), + } + } + + /// Close every open incident whose subject is a target that is no longer + /// admitted, then retire what has been closed past retention. + fn reconcile_incidents(&self, snapshot: &ControlSnapshot) { + let store = self.incidents(); + let close_unadmitted = || -> Result<()> { + let now = now_millis()?; + for record in store.read()? { + if record.is_open() + && record.condition.ends_with_admission() + && snapshot.admitted_for(&record.subject).is_none() + { + store.close( + record.condition, + &record.subject, + CloseReason::NoLongerAdmitted, + now, + )?; + } + } + Ok(()) + }; + self.record_incident("close", "*", close_unadmitted()); + self.record_incident("retire", "*", now_millis().and_then(|now| store.retire_closed(now))); + } + fn host_access(&self) -> Result> { let hub = self .host_actuators @@ -4884,9 +5558,15 @@ impl Engine { let (_, binding) = plan.parsed_inputs()?; Ok(match &binding.workload { WorkloadBinding::SystemdTransient(_) => Arc::clone(&self.systemd_workload), - WorkloadBinding::HostActuator(_) => Arc::new(HostActuatorWorkloadDriver { - access: self.host_access()?, - }), + WorkloadBinding::HostActuator(_) => { + #[cfg(test)] + if let Some(port) = self.host_workload.lock().expect("host workload mutex").clone() { + return Ok(port); + } + Arc::new(HostActuatorWorkloadDriver { + access: self.host_access()?, + }) + } }) } @@ -5119,7 +5799,202 @@ struct ProjectionReconciliation { unexplained: Vec, } +/// What the lift decided about one continuity abort. +enum LiftDecision { + /// The abort was withdrawn and the record retimed. + Lifted, + /// The abort stands and runs: not a lift candidate, or its candidate is + /// provably dead. + KeepAborting, + /// The candidate could not be observed and is not provably dead. + Undecided(anyhow::Error), +} + impl Engine { + /// Boot's call of the lift. A lift that errs decided nothing, so boot fails + /// closed: every continuity abort is held, none of its steps run, until + /// `retry_held_lift` completes a lift. + fn boot_lift(&self) { + match self.return_wedged_continuity_aborts_forward() { + Ok(returned) => { + for transaction_id in &returned { + println!( + "returned continuity {transaction_id} to forward retry: its abort had not stopped its running candidate" + ); + } + } + Err(error) => { + self.lift_unsettled + .store(true, std::sync::atomic::Ordering::SeqCst); + eprintln!( + "Idunn boot could not lift wedged continuity aborts; every continuity abort is held until a lift succeeds: {error:#}" + ); + } + } + } + + /// Once at boot: withdraw the abort of a continuity that has not stopped + /// its running candidate, and retime the record so its phase retries + /// forward. A pre-c2c Idunn aborted a continuity at its deadline; that + /// abort stops the only running copy of the admitted body (ruling + /// c2c-boot-returns-wedged-aborts-forward). The result is the exact inverse + /// of `begin_post_fencing_abort`. + /// + /// A continuity abort now begins only over a permanently stopped candidate + /// or to yield to a deployment awaiting the handover, so this predicate + /// matches only aborts over a running candidate with nothing awaiting the + /// target, which Idunn no longer creates: it is a migration with a deletion + /// line (follow-up c2c-delete-boot-lift). A record whose candidate cannot + /// be observed is held, not aborted: `resume_candidate` runs no step of it + /// and asks `retry_held_lift` each pass, until observation answers (ruling + /// c2c-lift-defers-on-observe-error). + fn return_wedged_continuity_aborts_forward(&self) -> Result> { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let mut returned = Vec::new(); + for stored in &snapshot.transactions { + let id = &stored.value.transaction_id; + match self.consider_lifting_continuity_abort(&snapshot, stored) { + LiftDecision::Lifted => returned.push(id.clone()), + LiftDecision::KeepAborting => {} + LiftDecision::Undecided(error) => { + eprintln!( + "Idunn could not decide whether continuity {id} may retry forward; its abort is held, not run: {error:#}" + ); + self.lift_held.lock().expect("lift held mutex").insert(id.clone()); + } + } + } + Ok(returned) + } + + /// A held record's later pass: lift it, release it to its abort over + /// positive evidence of death, or keep holding. `Some(moved)` means the + /// pass is spent on the record; `None` lets the pass go on. + fn retry_held_lift(&self, current: &Stored) -> Result> { + let id = current.value.transaction_id.as_str(); + if current.value.command_kind == CommandKind::Continuity + && current.value.post_fencing_abort.is_some() + && self.lift_unsettled.load(std::sync::atomic::Ordering::SeqCst) + { + match self.return_wedged_continuity_aborts_forward() { + Err(error) => { + self.note_fault("holds continuity abort", id, &error); + return Ok(Some(false)); + } + Ok(returned) => { + self.lift_unsettled + .store(false, std::sync::atomic::Ordering::SeqCst); + for transaction_id in &returned { + println!( + "returned continuity {transaction_id} to forward retry: its abort had not stopped its running candidate" + ); + } + if returned.iter().any(|lifted| lifted == id) { + return Ok(Some(true)); + } + } + } + } + if !self.lift_held.lock().expect("lift held mutex").contains(id) { + return Ok(None); + } + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + match self.consider_lifting_continuity_abort(&snapshot, current) { + LiftDecision::Undecided(error) => { + self.note_fault("holds continuity abort", id, &error); + Ok(Some(false)) + } + LiftDecision::Lifted => { + self.lift_held.lock().expect("lift held mutex").remove(id); + self.clear_fault(id); + println!( + "returned continuity {id} to forward retry: its abort had not stopped its running candidate" + ); + Ok(Some(true)) + } + LiftDecision::KeepAborting => { + self.lift_held.lock().expect("lift held mutex").remove(id); + self.clear_fault(id); + Ok(None) + } + } + } + + /// The lift's predicate and write for one record. An abort proceeds only + /// over positive evidence that its candidate is dead; an observation that + /// merely errs is undecided. + fn consider_lifting_continuity_abort( + &self, + snapshot: &ControlSnapshot, + stored: &Stored, + ) -> LiftDecision { + let transaction = &stored.value; + let Some(abort) = &transaction.post_fencing_abort else { + return LiftDecision::KeepAborting; + }; + if transaction.command_kind != CommandKind::Continuity + || transaction.completion.is_some() + || abort.candidate_cleanup != CleanupEvidence::Pending + || abort.lease_withdrawal == CleanupEvidence::Complete + { + return LiftDecision::KeepAborting; + } + match now_millis() { + Ok(now) => { + if self + .deploy_the_continuity_yields_to(snapshot, transaction, now) + .is_some() + { + return LiftDecision::KeepAborting; + } + } + Err(error) => return LiftDecision::Undecided(error), + } + let lifted = (|| -> Result { + let plan = required(&transaction.plan, "transaction plan")?; + let expected = required(&transaction.expected, "Expected projection")?; + let activation = required(&transaction.activation, "activation")?; + let Some(workload) = &transaction.workload else { + return Ok(false); + }; + self.workload_for(plan)? + .observe(expected, activation, workload)?; + if let Some(lease) = transaction + .leasing + .as_ref() + .and_then(LeasingEvidence::issued_lease) + { + let path = plan + .parsed_inputs()? + .1 + .process_write_lease + .context("leased target has no write-lease binding")? + .record_path; + if !CultCacheWriteLeaseDriver::new(&transaction.target, path) + .observe_exact(lease)? + { + return Ok(false); + } + } + let now = now_millis()?; + let mut next = transaction.clone(); + next.post_fencing_abort = None; + next.last_error = Some(abort.error.clone()); + next.updated_at_unix_millis = now; + next.retime(now); + replace_transaction(&self.options.state_store, stored, &next)?; + Ok(true) + })(); + match lifted { + Ok(true) => LiftDecision::Lifted, + Ok(false) => LiftDecision::KeepAborting, + Err(error) => match self.candidate_is_permanently_stopped(transaction) { + Ok(true) => LiftDecision::KeepAborting, + _ => LiftDecision::Undecided(error), + }, + } + } + /// Once at boot: demote the activation a failed continuity left standing, /// where the projection still names exactly the activation that failed /// transaction issued. A continuity abort before the single resolution @@ -5227,12 +6102,13 @@ fn boot(options: RuntimeOptions) -> Result<(ProcessLock, Engine)> { .context("migrating Idunn control records to the current schema")?; if migrated > 0 { println!( - "migrated {migrated} control record(s) to {DEPLOYMENT_TRANSACTION_SCHEMA} / {ADMITTED_GENERATION_SCHEMA}" + "migrated {migrated} control record(s) to {DEPLOYMENT_TRANSACTION_SCHEMA} / {ADMITTED_GENERATION_SCHEMA} / {TARGET_SUPERVISION_SCHEMA}" ); } ControlSnapshot::read(&options.state_store).context("validating all Idunn records")?; let engine = Engine::open(options)?; engine.validate_durable_authority(&ControlSnapshot::read(&engine.options.state_store)?)?; + engine.boot_lift(); match engine.reconcile_failed_continuity_projections() { Ok(outcome) => { for transaction_id in &outcome.demoted { @@ -5252,7 +6128,13 @@ fn boot(options: RuntimeOptions) -> Result<(ProcessLock, Engine)> { fn serve(options: RuntimeOptions) -> Result<()> { let (_lock, engine) = boot(options)?; - loop { + serve_while(&engine, || true) +} + +/// The daemon's loop: tick, sleep, and go again while `again` says so. `serve` +/// says so forever; a test says so a few times. +fn serve_while(engine: &Engine, mut again: impl FnMut() -> bool) -> Result<()> { + while again() { match engine.run_scheduler_tick() { Ok(true) => continue, Ok(false) => {} @@ -5267,6 +6149,7 @@ fn serve(options: RuntimeOptions) -> Result<()> { } thread::sleep(Duration::from_millis(engine.options.poll_millis)); } + Ok(()) } impl Engine { @@ -5315,8 +6198,8 @@ impl Engine { let continuity_progress = self.supervise_one_admitted_generation()?; // Freezing is not gated on the others' progress: a queued command // whose target is free must not wait behind an unrelated target's - // wedge. `freeze_command` already refuses a busy target, so this is - // still at most one freeze per tick and never a second claimant. + // wedge. `freeze_command` refuses only a second live deployment of a + // target, and this freezes at most one command per tick. let froze = self.freeze_one_queued_command()?; Ok(transaction_progress || continuity_progress || froze) } @@ -5325,7 +6208,7 @@ impl Engine { /// record's `last_error`, which status renders. Recording is best effort: /// a store too broken to take the note is already in stderr. fn note_fault(&self, what: &str, transaction_id: &str, error: &anyhow::Error) { - let detail = truncate(&format!("{error:#}"), 2048); + let detail = bounded_detail(&format!("{error:#}")); let offered = { let mut reports = self.fault_reports.lock().expect("fault report mutex"); reports @@ -5352,6 +6235,8 @@ impl Engine { /// Write `detail` as the record's `last_error`, touching nothing else and /// not archiving: a terminal record whose archive is the fault stays put. fn record_last_error(&self, transaction_id: &str, detail: &str) -> Result<()> { + let detail = bounded_detail(detail); + let detail = detail.as_str(); let snapshot = ControlSnapshot::read(&self.options.state_store)?; let Some(stored) = snapshot .transactions @@ -5385,6 +6270,8 @@ impl Engine { fn resume_one_transaction(&self) -> Result { let snapshot = ControlSnapshot::read(&self.options.state_store)?; let mut progressed = self.retire_spent_expiry_requests(&snapshot)?; + // Incident writes are never progress. + self.close_spent_operator_incidents(&snapshot, now_millis()?); let mut candidates = snapshot .transactions .iter() @@ -5424,7 +6311,15 @@ impl Engine { /// transaction would report progress every tick. fn resume_candidate(&self, current: &Stored) -> Result { let id = current.value.transaction_id.as_str(); + if let Some(moved) = self.retry_held_lift(current)? { + return Ok(moved); + } if let Some(moved) = self.resolve_phase_deadline(current)? { + if moved { + // A phase-ending write is movement: the next step is not held + // behind the backoff a forward retry armed. + self.resume_backoff.lock().expect("resume backoff mutex").remove(id); + } return Ok(moved); } if self @@ -5434,7 +6329,7 @@ impl Engine { .get(id) .is_some_and(|backoff| { now_millis().is_ok_and(|now| { - is_waiting(now, backoff.not_before_unix_millis, RESUME_BACKOFF_CEILING_MILLIS) + is_waiting(now, backoff.not_before_unix_millis, backoff.ceiling_millis) }) }) { @@ -5444,15 +6339,29 @@ impl Engine { self.note_fault("holds transaction", id, &anyhow::Error::new(disagreement)); return Ok(false); } + let left = now_millis().is_ok_and(|now| current.value.leaves_to_operator(now)); let advanced = self.advance_transaction(current); if advanced.is_ok() { - // The step ran and succeeded: the fault, if any, has recovered, so - // a later failure is news. A skipped or failed attempt clears nothing. - self.resume_backoff - .lock() - .expect("resume backoff mutex") - .remove(id); - self.clear_fault(id); + // A step that returns Ok without moving a record left to the + // operator (a gate-wait on a silent candidate) is an attempt like + // a failed one: it backs off, and the fault it recorded stands. + let unmoved = left + && ControlSnapshot::read(&self.options.state_store)? + .transactions + .iter() + .find(|stored| stored.value.transaction_id == id) + .is_some_and(|live| !state_advanced(¤t.value, &live.value)); + if unmoved { + self.back_off(¤t.value)?; + } else { + // The step ran and succeeded: the fault, if any, has recovered, so + // a later failure is news. A skipped or failed attempt clears nothing. + self.resume_backoff + .lock() + .expect("resume backoff mutex") + .remove(id); + self.clear_fault(id); + } } if let Err(error) = advanced { let latest_snapshot = ControlSnapshot::read(&self.options.state_store)?; @@ -5468,12 +6377,22 @@ impl Engine { .remove(id); return Ok(true); } - self.back_off(id)?; + self.back_off(&latest.value)?; if latest.value.phase < DeploymentPhase::Fencing { // Before the fence an abort that cannot finish a step is // resumable, never a post-fence abort: that path refuses a // pre-fence phase, so choosing it only wedged the tick. - if latest.value.pre_fencing_abort.is_none() { + // + // A publish that lost to a held lock is not a failed step: the + // holder lets go, so the transaction stays in its phase with + // the error recorded and the backoff above retries it. The + // phase deadline, not an abort, bounds the wait. + if latest.value.pre_fencing_abort.is_none() + && error.downcast_ref::().is_none() + { + if let Some(line) = self.keep_runner_failure(&latest.value, &error) { + eprintln!("{line}"); + } self.begin_pre_fencing_abort(latest, error)?; } else { self.record_resumable_error(latest, &error)?; @@ -5494,26 +6413,41 @@ impl Engine { } } let after = ControlSnapshot::read(&self.options.state_store)?; - let live = after + let Some(live) = after .transactions .iter() .find(|stored| stored.value.transaction_id == current.value.transaction_id) - .context("transaction disappeared while checking scheduler progress")?; + else { + // The step that ends a transaction archives it inside its own + // write, so it is gone from the live set because it finished: that + // is the step's progress. Gone from both is a fault. + ensure!( + read_history_transactions(&self.options.state_store) + .iter() + .any(|archived| archived.transaction_id == id && archived.is_terminal()), + "transaction disappeared while checking scheduler progress" + ); + return Ok(true); + }; Ok(state_advanced(¤t.value, &live.value)) } /// Wait twice as long after each consecutive failure of a transaction's /// step, from one poll interval up to a ceiling. - fn back_off(&self, transaction_id: &str) -> Result<()> { + fn back_off(&self, transaction: &DeploymentTransaction) -> Result<()> { let now = now_millis()?; + let ceiling_millis = resume_ceiling(transaction, now); let mut all = self.resume_backoff.lock().expect("resume backoff mutex"); - let failures = all.get(transaction_id).map_or(0, |backoff| backoff.failures); - let wait = backoff_wait(self.options.poll_millis, failures); + let failures = all + .get(&transaction.transaction_id) + .map_or(0, |backoff| backoff.failures); + let wait = backoff_wait(self.options.poll_millis, failures, ceiling_millis); all.insert( - transaction_id.to_string(), + transaction.transaction_id.clone(), ResumeBackoff { failures: failures.saturating_add(1), not_before_unix_millis: now.saturating_add(wait), + ceiling_millis, }, ); Ok(()) @@ -5591,8 +6525,10 @@ impl Engine { Ok(false) } - /// Freeze one queued command into its transactions. `Ok(false)` means its - /// target is busy and nothing was written. + /// Freeze one queued command into its transactions. `Ok(false)` means a + /// live deployment already exists for one of its targets and nothing was + /// written. Continuity, owners and drains never refuse a freeze: a + /// deployment that has not taken its target owns nothing. fn freeze_command( &self, snapshot: &ControlSnapshot, @@ -5631,7 +6567,9 @@ impl Engine { let busy = snapshot .transactions .iter() - .filter(|stored| stored.value.blocks_new_target_mutation()) + .filter(|stored| { + stored.value.command_kind == CommandKind::Deploy && stored.value.is_live() + }) .map(|stored| stored.value.target.as_str()) .collect::>(); if targets.iter().any(|target| busy.contains(target.as_str())) { @@ -5680,6 +6618,7 @@ impl Engine { .lock() .expect("challenge wait mutex") .retain(|target, wait| snapshot.target_stamp(target) == wait.stamp); + self.reconcile_incidents(&snapshot); let mut progressed = false; let mut admitted = snapshot.admitted.iter().collect::>(); admitted.sort_by_key(|stored| stored.value.target.as_str()); @@ -5691,23 +6630,21 @@ impl Engine { &anyhow::Error::new(disagreement), ); } - let blocker = snapshot.transactions.iter().find(|stored| { - stored.value.target == current.value.target - && stored.value.blocks_new_target_mutation() - }); - if blocker.is_some_and(|stored| { - let owns_from = if stored.value.rollout_stops_incumbent_first() { - // The incumbent is stopped at Starting by design; a - // continuity restart in that window would fight the - // candidate for the same host resources. - DeploymentPhase::Starting - } else { - DeploymentPhase::Fencing - }; - stored.value.phase >= owns_from && stored.value.phase < DeploymentPhase::Complete - }) { + // A transaction that owns the incumbent slot fences, stops or + // replaces this generation: nothing here touches it meanwhile. + if snapshot + .on_target(¤t.value.target) + .any(|stored| stored.value.owns_target()) + { continue; } + // Continuity defers to an owner, a drain or its own live attempt, + // never to a deployment that has not taken the target, and aborts + // nothing (handover-phase-split). + let mint_blocked = snapshot.target_owner(¤t.value.target, None).is_some() + || snapshot.on_target(¤t.value.target).any(|stored| { + stored.value.is_live() && stored.value.command_kind == CommandKind::Continuity + }); match self.supervise_admitted_route(&snapshot, current) { Ok(true) => { progressed = true; @@ -5752,30 +6689,26 @@ impl Engine { // the admitted one leaves the projection naming an incarnation that // no longer exists -- which the workload rightly refuses to adopt. // That record is published from observation once the process is up. - if blocker.is_none() - || blocker.is_some_and(|stored| stored.value.phase == DeploymentPhase::Complete) - { - let repair = (|| -> Result<()> { - let topology = self.topology(); - let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?; - if !topology.admitted_expected_projection_is_exact( - ¤t.value.expected, - &provider_anchor, - )? { - ensure!( - topology.publish_expected(¤t.value.expected, &provider_anchor)? - == current.value.expected.canonical_sha256()?, - "admitted Expected projection repair differs" - ); - } - Ok(()) - })(); - if let Err(error) = repair { - eprintln!( - "Idunn preserved admitted {} after refusing topology projection repair: {error:#}", - current.value.target + let repair = (|| -> Result<()> { + let topology = self.topology(); + let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?; + if !topology.admitted_expected_projection_is_exact( + ¤t.value.expected, + &provider_anchor, + )? { + ensure!( + topology.publish_expected(¤t.value.expected, &provider_anchor)? + == current.value.expected.canonical_sha256()?, + "admitted Expected projection repair differs" ); } + Ok(()) + })(); + if let Err(error) = repair { + eprintln!( + "Idunn preserved admitted {} after refusing topology projection repair: {error:#}", + current.value.target + ); } let mut operational_error = None; let observation = match self.workload_for(¤t.value.plan).and_then(|workload| { @@ -5848,6 +6781,39 @@ impl Engine { } }; if observation.is_some() { + let seen = snapshot.supervision_for(¤t.value.target); + if let Some(stored) = seen.filter(|stored| { + stored.value.continuity_backoff_from_unix_millis.is_some() + }) { + let mut next = stored.value.clone(); + next.end_backoff(); + self.write_target_supervision(Some(stored), &next, now_millis()?)?; + } + // The admitted workload was seen running: the only evidence + // that ends an exhaustion incident, once the window no longer holds it. + // The window sliding alone is not health and never closes one. + self.record_incident( + "close", + ¤t.value.target, + now_millis().and_then(|now| { + // Running is not enough while the window still holds + // the restarts that exhausted it: a release that + // survives one poll after each restart is still the + // same incident. + if snapshot + .supervision_or_new(¤t.value.target) + .restarts_page(now) + { + return Ok(false); + } + self.incidents().close( + IncidentCondition::ContinuityExhausted, + ¤t.value.target, + CloseReason::Recovered, + now, + ) + }), + ); continue; } let workload_error = operational_error @@ -5857,8 +6823,7 @@ impl Engine { // continuity minted over it copies the undeclared Expected, is // held at once, and owns the target forever, so the declaring // redeploy that would clear the hold could never freeze. The dead - // held target stays down and free. Nothing yields to it either: - // there is no continuity to yield to. + // held target stays down and free. if current.value.readiness().is_err() { self.note_fault( "holds a dead admitted generation down; declare readiness in its recipe and redeploy", @@ -5867,16 +6832,12 @@ impl Engine { ); continue; } - // Continuity restarts a release; it cannot repair one. When the - // admitted release will not start, rescheduling it forever keeps - // the target permanently occupied -- and a target with a live - // transaction accepts no deployment, so the one action that could - // fix it is exactly the one that is locked out. Restarts are - // counted in the target's own log, in a sliding window, and spaced - // by a doubling wait. The log outlives every generation, so a - // release that starts, dies and starts again is bounded the same as - // one that never starts. Nothing here reads history: an unreadable - // history file must not stop crash recovery. + // Continuity restarts a release; it cannot repair one, and it + // never stops (budget-never-stop). Every charged attempt is spaced + // by a doubling, capped wait, so a queued deployment always gets + // a free gap (map B10). The log outlives every generation and only + // pages. Nothing here reads history: an unreadable history file + // must not stop crash recovery. let now = now_millis()?; let supervision = snapshot.supervision_or_new(¤t.value.target); // A stepped-back clock leaves entries in the future. Settle them @@ -5894,52 +6855,72 @@ impl Engine { } let continuity_key = format!("continuity:{}", current.value.target); - if let Some(blocker) = blocker { - // Yielding a *deployment* to continuity is right: the incumbent - // died, so changing it can wait. Doing the same to a continuity - // transaction is self-defeating -- that transaction exists to - // restart the very workload whose absence triggers the yield, - // so aborting it schedules another, which is aborted in turn. - // Odin sat in that loop, down, while Idunn cancelled its own - // recovery every few seconds. - // Only yield to a continuity that is still trying. Once it has - // given up on this release there is nothing to yield to, and - // the deployment is the only thing left that can fix the - // target -- aborting it would close the last door. - if blocker.value.command_kind == CommandKind::Deploy - && !supervision.restarts_exhausted(now) - && blocker.value.phase < DeploymentPhase::Fencing - && blocker.value.pre_fencing_abort.is_none() - // A held record is reported and never aborted by Idunn. - && blocker.value.held_disagreement().is_none() - { - self.begin_pre_fencing_abort( - blocker, - anyhow!( - "admitted incumbent failed before candidate fencing; deployment yielded to continuity: {workload_error:#}" - ), - )?; - progressed = true; - } + if mint_blocked { continue; } - if supervision.restarts_exhausted(now) { + if supervision.restarts_page(now) { + // The record comes first and the trace after it: the trace is + // this record's shadow, never its substitute. + self.record_incident( + "open", + ¤t.value.target, + self.incidents().open( + IncidentCondition::ContinuityExhausted, + ¤t.value.target, + now, + ), + ); self.report_once( &continuity_key, format!( - "Idunn stopped restarting admitted {}: {} restarts inside the window. The target is free for a deployment to replace it{}: {workload_error:#}", + "Idunn has restarted admitted {} {} times inside the hour and keeps restarting it on a capped backoff (streak {}): {workload_error:#}", current.value.target, supervision.restarts_used(now), - supervision.route_reopens_at(now).map_or_else(String::new, |at| format!( - ", but its route ceiling refuses the route change until it reopens at unix ms {at}" - )), + supervision.continuity_attempt_streak, ), ); + } else { + self.clear_fault(&continuity_key); + } + if supervision.continuity_deferred(now) { continue; } - self.clear_fault(&continuity_key); - if supervision.continuity_is_waiting(now) { - continue; + // A unit the host lost to a reboot is re-materialized once per + // generation, uncharged and with no wait; the driver owns the + // evidence and supervision owns the budget. An unanswerable + // driver is not evidence of a reboot. + let admitted_at = current.value.admitted_at_unix_millis; + let rematerialize = !supervision.rematerialization_spent(admitted_at) + && match self.workload_for(¤t.value.plan).and_then(|workload| { + workload.lost_to_host_reboot(¤t.value.workload, admitted_at) + }) { + Ok(lost) => lost, + Err(error) => { + self.note_fault( + "could not tell whether the host rebooted under", + &format!("reboot:{}", current.value.target), + &error, + ); + false + } + }; + let mut streak = 0; + if !rematerialize { + match supervision.continuity_backoff(now, admitted_at) { + ContinuityBackoff::Waiting => continue, + ContinuityBackoff::Begin => { + let mut began = supervision.clone(); + began.begin_backoff(now); + self.write_target_supervision( + snapshot.supervision_for(¤t.value.target), + &began, + now, + )?; + progressed = true; + continue; + } + ContinuityBackoff::Due { streak: due } => streak = due, + } } // The projected activation names an incarnation that is gone. Every @@ -5976,7 +6957,7 @@ impl Engine { current.value.target ); let mut deferred = supervision.clone(); - deferred.defer_continuity(now, &truncate(&format!("{error:#}"), 2048)); + deferred.defer_continuity(now, &bounded_detail(&format!("{error:#}"))); self.write_target_supervision( snapshot.supervision_for(¤t.value.target), &deferred, @@ -6005,10 +6986,15 @@ impl Engine { command.validate()?; let transaction = DeploymentTransaction::from_continuity(&command, ¤t.value, now)?; - // The restart is counted in the same CAS that mints it, so a crash - // cannot schedule one the ceiling never saw. + // The attempt is charged, or recorded as the one re-materialization, + // in the same CAS that mints it, so a crash cannot schedule one the + // meters never saw. let mut counted = supervision; - counted.record_restart(now); + if rematerialize { + counted.record_rematerialization(now); + } else { + counted.record_restart(now, streak); + } ensure!( SingleFileMessagePackBackingStore::new(&self.options.state_store) .compare_exchange( @@ -6321,6 +7307,11 @@ impl Engine { if current.value.pre_fencing_abort.is_some() && current.value.completion.is_none() { return self.advance_pre_fencing_abort(current); } + if current.value.phase < current.value.takes_target_at() + && self.settle_before_handover(current)? + { + return Ok(()); + } match current.value.phase { DeploymentPhase::Sealing => self.advance_sealing(current), DeploymentPhase::Starting => self.advance_starting(current), @@ -6446,7 +7437,9 @@ impl Engine { })?; next.enter_phase(DeploymentPhase::Starting, now); next.last_error = None; - replace_transaction(&self.options.state_store, current, &next) + // Starting is the handover for stop-then-start, so this write goes + // through the seam that decides it. + self.enter(current, next) } fn advance_starting(&self, current: &Stored) -> Result<()> { @@ -7379,6 +8372,9 @@ impl Engine { generation_id: generation.generation_id.clone(), }); complete.post_commit_cleanup = Some(post_commit_cleanup); + // enter_phase ran before the cleanup was set: a draining record + // carries its drain's deadline. + complete.retime(now); complete.validate()?; let admitted_expected = CultCacheExpectedEnvelope { @@ -7406,6 +8402,10 @@ impl Engine { Ok(()) } + /// The admitted generation this transaction was proven against. A + /// pre-handover deployment is rebased onto the admitted generation before + /// any step reads its incumbent, and the handover pins it after, so a + /// mismatch here is a fault, not a race. fn exact_incumbent<'a>( &self, snapshot: &'a ControlSnapshot, @@ -7417,7 +8417,9 @@ impl Engine { (Some(expected), Some(current)) if current.value.generation_id == *expected => { Ok(Some(current)) } - _ => bail!("target incumbent changed after transaction creation"), + _ => bail!( + "admitted generation changed under a transaction that owns the target or names it as its source" + ), } } @@ -7625,24 +8627,7 @@ impl Engine { let Some(route) = &binding.route else { return Ok(None); }; - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let mut used = BTreeSet::new(); - for plan in snapshot - .admitted - .iter() - .map(|stored| &stored.value.plan) - .chain( - snapshot - .transactions - .iter() - .filter(|stored| stored.value.blocks_new_target_mutation()) - .filter_map(|stored| stored.value.plan.as_ref()), - ) - { - if let Some(port) = plan.candidate_port { - used.insert(port); - } - } + let used = ControlSnapshot::read(&self.options.state_store)?.candidate_ports_in_use(); (route.private_port_start..=route.private_port_end) .find(|port| !used.contains(port)) .map(Some) @@ -7700,6 +8685,36 @@ impl Engine { })) } + /// Whether a deployment awaiting the handover is free to take it: the + /// deployment brake has released its exact transaction, so the stop a yield + /// makes is followed by the deployment's own landing. A Deploy at its + /// Sealing tail has not passed the brake yet and is read live; one that is + /// still building, or whose brake cannot be read, is held. An operator's + /// hold is a reason to leave a serving or retrying body alone. + fn deploy_can_move(&self, deploy: &Stored, now: u64) -> bool { + deploy.value.deployment_authorization.is_some() + || matches!(self.deployment_authorization(deploy, now), Ok(Some(_))) + } + + /// The deployment a continuity gives way to: one on its target that awaits + /// the handover and can take it, while the continuity's own candidate is + /// unproven (ruling c2d-yield-only-unproven-to-movable). The one test the + /// deadline's yield and the boot lift's abort both ask, so neither stops a + /// serving candidate, or one a braked Deploy cannot replace. + fn deploy_the_continuity_yields_to<'a>( + &self, + snapshot: &'a ControlSnapshot, + continuity: &DeploymentTransaction, + now: u64, + ) -> Option<&'a Stored> { + if continuity.serving_proven() { + return None; + } + snapshot.on_target(&continuity.target).find(|other| { + other.value.awaits_handover() && self.deploy_can_move(other, now) + }) + } + fn lifecycle_allows(&self, current: &Stored, now: u64) -> Result { let plan = required(¤t.value.plan, "continuity plan")?; let binding = plan.parsed_inputs()?.1; @@ -7808,16 +8823,74 @@ impl Engine { } } + /// A deployment waits on the transaction that owns or drains its target. + /// The owner bounds the wait, so the waiter's own Warming deadline is + /// disarmed (the resolver re-arms it when the target is free). When the + /// owner's deadline has passed in a state Idunn will not end by itself and + /// will not yield (a wedged abort or drain), the waiter ends through its own + /// pre-fencing abort (the caller's Err path): it never aborts the owner. A + /// waiter keeps waiting on an owner that retries forward, because that + /// owner yields to it on its next pass. + fn wait_for_owner( + &self, + current: &Stored, + owner_id: &str, + ) -> Result<()> { + let held = ¤t.value; + let now = now_millis()?; + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + if snapshot.transactions.iter().any(|stored| { + stored.value.transaction_id == owner_id + && stored.value.leaves_to_operator(now) + && !stored.value.yields_to_a_deploy() + }) { + bail!( + "owner-left-to-operator: target {} is owned by {owner_id}, which Idunn has left to the operator; this deployment ends before fencing", + held.target + ); + } + let detail = bounded_detail(&waiting_for_owner(&held.target, owner_id)); + if held.last_error.as_deref() == Some(detail.as_str()) && held.phase_deadline.is_none() { + return Ok(()); + } + let mut next = held.clone(); + next.last_error = Some(detail); + next.phase_deadline = None; + next.updated_at_unix_millis = now; + replace_transaction(&self.options.state_store, current, &next) + } + + /// Idunn has run out of ways to end this record. It stays owned and its + /// step keeps retrying; the operator is told. The incident comes first and + /// the trace second, and only for a record that is left to the operator + /// (an operator's expiry of a stateful deploy is reported, not paged). + fn leave_to_operator(&self, transaction: &DeploymentTransaction, cause: &str) { + let id = &transaction.transaction_id; + let now = now_millis(); + if now.as_ref().is_ok_and(|now| transaction.leaves_to_operator(*now)) { + self.record_incident( + "open", + id, + now.and_then(|now| self.incidents().open(IncidentCondition::OperatorRequired, id, now)), + ); + } + self.report_once( + &format!("deadline:{id}"), + format!("Idunn leaves {id} to the operator: {cause}; it keeps retrying the step"), + ); + } + fn record_gate_wait( &self, current: &Stored, detail: &str, ) -> Result<()> { - if current.value.last_error.as_deref() == Some(detail) { + let detail = bounded_detail(detail); + if current.value.last_error.as_deref() == Some(detail.as_str()) { return Ok(()); } let mut next = current.value.clone(); - next.last_error = Some(detail.into()); + next.last_error = Some(detail); next.updated_at_unix_millis = now_millis()?; replace_transaction(&self.options.state_store, current, &next) } @@ -8557,46 +9630,252 @@ impl Engine { current: &Stored, next_phase: DeploymentPhase, ) -> Result<()> { - ensure!( - next_phase as u8 == current.value.phase as u8 + 1, - "deployment phase transition is not adjacent" - ); let mut next = current.value.clone(); next.enter_phase(next_phase, now_millis()?); next.last_error = None; - replace_transaction(&self.options.state_store, current, &next) + self.enter(current, next) } - /// Every post-fencing phase ends. Before a live transaction past the - /// fence runs its step, a phase whose deadline has passed, or whose - /// command an operator asked to expire, is resolved as `phase_end` says. + /// Before its handover a Deploy is proven against the generation that was + /// admitted when the evidence was gathered. When another generation has + /// been admitted since (continuity restarted the incumbent), rewrite the + /// incumbent and drop exactly the evidence that names the old one: the + /// warming (its lease), the route preflight (its route) and the isolation + /// (its workload). A continuity is never rebased. Returns whether it wrote. + fn rebase_onto_admitted(&self, current: &Stored) -> Result { + let held = ¤t.value; + if held.command_kind != CommandKind::Deploy + || held.pre_fencing_abort.is_some() + || held.phase >= held.takes_target_at() + { + return Ok(false); + } + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let admitted = snapshot.admitted_for(&held.target); + let admitted_id = admitted.map(|stored| stored.value.generation_id.as_str()); + if admitted_id == held.incumbent_generation_id.as_deref() { + return Ok(false); + } + self.persist_same_phase(current, |next| { + next.incumbent_generation_id = admitted_id.map(str::to_owned); + next.warming = None; + next.route_preflight = None; + next.isolation = None; + next.odin_publisher_sequence_cursor = next + .odin_publisher_sequence_cursor + .max(admitted.map_or(0, |stored| stored.value.odin_publisher_sequence_cursor)); + Ok(()) + })?; + Ok(true) + } + + /// What a transaction that has not yet taken its target does first, when + /// the world moved under it. `true` means this decided the step. + /// + /// A Deploy rebases onto the generation now admitted; in Warming, whose + /// steps read the incumbent an owner is fencing, it waits while another + /// transaction owns or drains the target. Both return Ok: a pre-fencing + /// error is an abort, and no deployment is aborted for continuity. A + /// Continuity restarts one generation and no other, so it ends itself + /// (through the ordinary pre-fencing abort) when its target is taken or + /// its generation replaced. + fn settle_before_handover(&self, current: &Stored) -> Result { + let held = ¤t.value; + match held.command_kind { + CommandKind::Deploy => { + if self.rebase_onto_admitted(current)? { + return Ok(true); + } + if held.phase != DeploymentPhase::Warming { + return Ok(false); + } + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let Some(owner) = snapshot.target_owner(&held.target, Some(&held.transaction_id)) + else { + return Ok(false); + }; + self.wait_for_owner(current, &owner.value.transaction_id) + .map(|()| true) + } + CommandKind::Continuity => { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + if let Some(owner) = snapshot.target_owner(&held.target, Some(&held.transaction_id)) + { + bail!( + "continuity superseded: target {} is owned by {}", + held.target, + owner.value.transaction_id + ); + } + ensure!( + snapshot + .admitted_for(&held.target) + .map(|stored| stored.value.generation_id.as_str()) + == held.incumbent_generation_id.as_deref(), + "continuity superseded: the generation it restarts is no longer admitted" + ); + Ok(false) + } + } + } + + /// The seam for the forward moves that can be the handover (`transition` + /// and the end of Sealing). Entering the phase where the transaction takes + /// its target (`takes_target_at`) is decided by `take_target`: that CAS is + /// the handover, and nothing else writes that entry. The other writes + /// (Starting to Warming, completions, aborts, same-phase evidence) go to + /// `replace_transaction` directly because none of them enters that phase. + fn enter( + &self, + current: &Stored, + next: DeploymentTransaction, + ) -> Result<()> { + ensure!( + next.phase as u8 == current.value.phase as u8 + 1, + "deployment phase transition is not adjacent" + ); + if next.phase != next.takes_target_at() { + return replace_transaction(&self.options.state_store, current, &next); + } + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let target = ¤t.value.target; + match (self.take_target(&snapshot, current, &next)?, current.value.command_kind) { + // Lost to a concurrent writer: the next step reads the new world. + (Handover::Took | Handover::Lost, _) => Ok(()), + // Wait, never abort: a pre-fencing error would end the deployment. + (Handover::OwnedBy(owner), CommandKind::Deploy) => { + self.wait_for_owner(current, &owner) + } + // The next step rebases onto the generation now admitted. + (Handover::Moved, CommandKind::Deploy) => Ok(()), + (Handover::OwnedBy(owner), CommandKind::Continuity) => { + bail!("continuity superseded: target {target} is owned by {owner}") + } + (Handover::Moved, CommandKind::Continuity) => { + bail!("continuity superseded: the generation it restarts is no longer admitted") + } + } + } + + /// The handover: one CAS that writes `next` (the entry into + /// `takes_target_at`) only if nothing else moved. It expects the + /// transaction's own envelope, the target's admitted envelope as read + /// (or its absence), and every other non-terminal transaction of the + /// target as read, so of two transactions entering from one view exactly + /// one succeeds. + fn take_target( + &self, + snapshot: &ControlSnapshot, + current: &Stored, + next: &DeploymentTransaction, + ) -> Result { + let target = current.value.target.as_str(); + let id = current.value.transaction_id.as_str(); + if let Some(owner) = snapshot.target_owner(target, Some(id)) { + return Ok(Handover::OwnedBy(owner.value.transaction_id.clone())); + } + let admitted = snapshot.admitted_for(target); + if admitted.map(|stored| stored.value.generation_id.as_str()) + != current.value.incumbent_generation_id.as_deref() + { + return Ok(Handover::Moved); + } + next.validate()?; + let mut expected = vec![ + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: id.into(), + current: Some(current.envelope.clone()), + }, + CultCacheExpectedEnvelope { + r#type: AdmittedGeneration::TYPE.into(), + key: target.into(), + current: admitted.map(|stored| stored.envelope.clone()), + }, + ]; + expected.extend( + snapshot + .on_target(target) + .filter(|stored| stored.value.transaction_id != id && !stored.value.is_terminal()) + .map(|stored| CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: stored.value.transaction_id.clone(), + current: Some(stored.envelope.clone()), + }), + ); + let took = SingleFileMessagePackBackingStore::new(&self.options.state_store) + .compare_exchange( + &expected, + &[transaction_envelope(next, next.updated_at_unix_millis)?], + )?; + Ok(if took { Handover::Took } else { Handover::Lost }) + } + + /// Every live record is held to a deadline. Before a transaction runs its + /// step, a deadline phase whose deadline has passed, or whose command an + /// operator asked to expire, is resolved as `phase_end` says: aborted before + /// the fence, aborted after it, or left to the operator with its step + /// retrying (a stateful deploy, or a continuity, whose candidate is the only + /// running copy of its body). An abort or a drain past its deadline is left + /// to the operator the same way. A continuity left to the operator yields, + /// through its own post-fencing abort, to a deployment awaiting the + /// handover; it never writes the deployment. /// Returns `Some(moved)` when it decided the tick, `None` to run the step. /// - /// The deadline is written only by `enter_phase`, so nothing here or in a - /// resume path extends it. A record lifted from before deadlines existed - /// has none; its phase is timed once, from the first time this sees it. + /// Only `retime` sets a deadline (at phase entry, an abort's beginning, the + /// commit and this lift), so nothing in a step or a resume path extends + /// one. A record with none whose activity needs one is timed once, from + /// the first time this sees it: that is also how a waiter is re-armed once + /// the target is free. fn resolve_phase_deadline( &self, current: &Stored, ) -> Result> { let transaction = ¤t.value; - if !transaction.resolves_by_deadline() { + if transaction.is_terminal() { + return Ok(None); + } + // An owned waiter is timed by its owner. + if transaction.phase < DeploymentPhase::Fencing + && transaction.pre_fencing_abort.is_none() + && ControlSnapshot::read(&self.options.state_store)? + .target_owner(&transaction.target, Some(&transaction.transaction_id)) + .is_some() + { return Ok(None); } let now = now_millis()?; let Some(deadline) = transaction.phase_deadline else { let mut next = transaction.clone(); - next.enter_phase(transaction.phase, now); + next.retime(now); if next.phase_deadline.is_none() { return Ok(None); } replace_transaction(&self.options.state_store, current, &next)?; return Ok(Some(true)); }; - let requested = ControlSnapshot::read(&self.options.state_store)? - .expiry_requests - .iter() - .any(|request| request.value.command_id == transaction.command_id); + if !transaction.resolves_by_deadline() { + if transaction.leaves_to_operator(now) { + let what = if transaction.drains_incumbent() { + "draining its incumbent" + } else { + "its abort" + }; + self.leave_to_operator( + transaction, + &format!( + "{what} in the {:?} phase passed its deadline at unix ms {}", + transaction.phase, deadline.deadline_at_unix_millis + ), + ); + } + return Ok(None); + } + let requested = transaction.expirable() + && ControlSnapshot::read(&self.options.state_store)? + .expiry_requests + .iter() + .any(|request| request.value.command_id == transaction.command_id); let cause = if now >= deadline.deadline_at_unix_millis { format!( "the {:?} phase passed its deadline at unix ms {}", @@ -8608,6 +9887,10 @@ impl Engine { return Ok(None); }; match transaction.phase_end() { + PhaseEnd::AbortBeforeFencing => { + self.begin_pre_fencing_abort(current, anyhow!("{cause}"))?; + Ok(Some(true)) + } PhaseEnd::Abort { lease_not_adopted } => { let error = if lease_not_adopted { anyhow!("lease-not-adopted: {cause}, and the granted write lease has no adoption evidence") @@ -8618,11 +9901,35 @@ impl Engine { Ok(Some(true)) } PhaseEnd::OperatorRequired => { - self.report_once( - &format!("deadline:{}", transaction.transaction_id), - format!( - "Idunn leaves {} to the operator: {cause}, and its stateful candidate was issued a write lease", - transaction.transaction_id + self.leave_to_operator( + transaction, + &format!("{cause}, and its stateful candidate was issued a write lease"), + ); + Ok(None) + } + PhaseEnd::RetryForward => { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let now = now_millis()?; + if let Some(deploy) = transaction + .yields_to_a_deploy() + .then(|| self.deploy_the_continuity_yields_to(&snapshot, transaction, now)) + .flatten() + { + self.begin_post_fencing_abort( + current, + anyhow!( + "continuity-yields: {cause}; deployment {} awaits the handover of {}", + deploy.value.transaction_id, + transaction.target + ), + )?; + return Ok(Some(true)); + } + self.leave_to_operator( + transaction, + &format!( + "{cause}; its candidate is the only running copy of the admitted body, so Idunn retries the phase forward and never stops it; to restart it, {}", + transaction.manual_exit() ), ); Ok(None) @@ -8630,6 +9937,24 @@ impl Engine { } } + /// Close the operator-required incident of every transaction that is no + /// longer left to the operator: it ended, or its step went on. A + /// transaction that is still left keeps its incident open. + fn close_spent_operator_incidents(&self, snapshot: &ControlSnapshot, now: u64) { + let store = self.incidents(); + let close = || -> Result<()> { + for subject in store.open_subjects(IncidentCondition::OperatorRequired)? { + if !snapshot.transactions.iter().any(|stored| { + stored.value.transaction_id == subject && stored.value.leaves_to_operator(now) + }) { + store.close(IncidentCondition::OperatorRequired, &subject, CloseReason::Recovered, now)?; + } + } + Ok(()) + }; + self.record_incident("close", "*", close()); + } + /// Delete expiry requests whose command has no phase left to end: it was /// resolved, it finished, or its command is gone. fn retire_spent_expiry_requests(&self, snapshot: &ControlSnapshot) -> Result { @@ -8639,7 +9964,7 @@ impl Engine { .filter(|request| { !snapshot.transactions.iter().any(|stored| { stored.value.command_id == request.value.command_id - && stored.value.resolves_by_deadline() + && stored.value.expirable() }) }) .map(|request| request.envelope.clone()) @@ -8656,7 +9981,7 @@ impl Engine { current: &Stored, error: &anyhow::Error, ) -> Result<()> { - let detail = truncate(&format!("{error:#}"), 2048); + let detail = bounded_detail(&format!("{error:#}")); if current.value.last_error.as_deref() == Some(detail.as_str()) { return Ok(()); } @@ -8680,8 +10005,10 @@ impl Engine { "pre-fencing abort intent is already durable" ); let abort = pre_fencing_abort_intent(¤t.value, &format!("{error:#}")); + let now = now_millis()?; self.persist_same_phase(current, |next| { next.pre_fencing_abort = Some(abort); + next.retime(now); Ok(()) }) } @@ -8714,16 +10041,19 @@ impl Engine { current.value.post_fencing_abort.is_none(), "post-fencing abort intent is already durable" ); - let abort = post_fencing_abort_intent(¤t.value, &truncate(&format!("{error:#}"), 2048)); + let now = now_millis()?; + let abort = post_fencing_abort_intent(¤t.value, &bounded_detail(&format!("{error:#}"))); self.persist_same_phase(current, |next| { next.post_fencing_abort = Some(abort); + next.retime(now); Ok(()) }) } /// Withdraw the candidate in the one order that never leaves two writers: - /// take away its route, then its write lease, then the process itself, and - /// only then reconcile the projection. + /// take away its route, then stop the process and withdraw its write lease + /// (the lease is lifetime-held, so the holder stops first), then clean up, + /// and only then reconcile the projection. fn advance_post_fencing_abort(&self, current: &Stored) -> Result<()> { let abort = required( ¤t.value.post_fencing_abort, @@ -8763,6 +10093,15 @@ impl Engine { .process_write_lease .context("leased target has no write-lease binding")? .record_path; + // The candidate holds its lifetime write lease; revoking it under + // a live holder fails on the shared lock forever. A stopped + // process writes nothing, so stop first (as advance_fencing does). + if let Some(workload) = ¤t.value.workload { + let plan = required(¤t.value.plan, "abandoned candidate plan")?; + self.workload_for(plan)?.stop(workload).context( + "stopping the abandoned candidate before revoking its lifetime-held write lease", + )?; + } let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); driver .revoke_exact(Some(lease)) @@ -9080,7 +10419,7 @@ fn incumbent_was_stopped_during_fencing(fencing: &FencingEvidence) -> bool { fn pre_fencing_abort_intent(transaction: &DeploymentTransaction, error: &str) -> PreFencingAbort { PreFencingAbort { - error: truncate(error, 2048), + error: bounded_detail(error), candidate_cleanup: transaction.candidate_cleanup_owed(), topology_reconciliation: transaction.abort_topology_reconciliation(), source_cleanup: if transaction.command_kind == CommandKind::Deploy { @@ -9096,7 +10435,7 @@ fn post_fencing_abort_intent( error: &str, ) -> PostFencingAbort { PostFencingAbort { - error: truncate(error, 2048), + error: bounded_detail(error), route_restoration: if transaction.route_preflight.is_some() { CleanupEvidence::Pending } else { @@ -9292,7 +10631,13 @@ fn required<'a, T>(value: &'a Option, label: &str) -> Result<&'a T> { value.as_ref().with_context(|| format!("{label} is absent")) } -fn now_millis() -> Result { +/// The control store `idunn up` and `idunn serve` use unless told otherwise. +#[cfg(unix)] +pub(crate) fn default_state_store() -> PathBuf { + RuntimeOptions::default().state_store +} + +pub(crate) fn now_millis() -> Result { Ok(SystemTime::now() .duration_since(UNIX_EPOCH) .context("system clock predates Unix epoch")? @@ -9312,12 +10657,12 @@ fn sibling_path(path: &Path, suffix: &str) -> PathBuf { PathBuf::from(value) } -fn string_value(args: &mut impl Iterator, name: &str) -> Result { +pub(crate) fn string_value(args: &mut impl Iterator, name: &str) -> Result { args.next() .ok_or_else(|| anyhow!("{name} requires a value")) } -fn path_value(args: &mut impl Iterator, name: &str) -> Result { +pub(crate) fn path_value(args: &mut impl Iterator, name: &str) -> Result { Ok(PathBuf::from(string_value(args, name)?)) } @@ -9333,12 +10678,12 @@ fn u64_value(args: &mut impl Iterator, name: &str) -> Result .with_context(|| format!("{name} requires a u64")) } -fn require_selector(value: &str) -> Result<()> { +pub(crate) fn require_selector(value: &str) -> Result<()> { let value = value.strip_prefix("profile:").unwrap_or(value); require_id(value, "deployment selector") } -fn require_id(value: &str, label: &str) -> Result<()> { +pub(crate) fn require_id(value: &str, label: &str) -> Result<()> { ensure!( !value.is_empty() && value.len() <= 256 @@ -9369,8 +10714,67 @@ fn require_detail(value: &str, label: &str) -> Result<()> { Ok(()) } -fn truncate(value: &str, length: usize) -> String { - value.chars().take(length).collect() +const DETAIL_LIMIT: usize = 2048; +const DETAIL_HEAD: usize = 512; +const DETAIL_ELISION: &str = " +...[elided]... +"; +const DETAIL_EMPTY: &str = "(no detail)"; + +/// The only producer of persisted free-text details; `require_detail` is the +/// only judge, and this output always satisfies it. NUL is dropped, empty +/// input gets a fixed placeholder, and an over-long value keeps its head and +/// its tail (where a build log's error is) cut on char boundaries. +/// The journal line for a failed runner step: the transaction, the subject and +/// status (the failure's `Display`), and where the evidence is. Never output. +fn runner_failure_journal_line( + transaction: &DeploymentTransaction, + failure: &RunnerStepFailure, + store_path: &Path, + recorded: &Result, +) -> String { + let head = format!( + "Idunn transaction {} (command {}, target {}) {failure}", + transaction.transaction_id, transaction.command_id, transaction.target + ); + match recorded { + Ok(kept) => { + let aside = match kept { + Kept::Stored => String::new(), + Kept::SetAside { replaced } => format!( + "; an undecodable evidence store was set aside as {}{}", + unreadable_store_path(store_path).display(), + match replaced { + Some(last_written) => format!( + ", replacing an earlier set-aside store last written at unix millis {last_written}" + ), + None => String::new(), + } + ), + }; + format!( + "{head}; evidence in {}{aside}; read it with idunn status --command {}", + store_path.display(), + transaction.command_id + ) + } + Err(error) => format!("{head}; evidence was not kept: {error:#}"), + } +} + +fn bounded_detail(value: &str) -> String { + let value = value.replace('', ""); + if value.is_empty() { + return DETAIL_EMPTY.to_owned(); + } + if value.len() <= DETAIL_LIMIT { + return value; + } + let head = value.floor_char_boundary(DETAIL_HEAD); + let tail = value.ceil_char_boundary( + value.len() - (DETAIL_LIMIT - DETAIL_HEAD - DETAIL_ELISION.len()), + ); + format!("{}{DETAIL_ELISION}{}", &value[..head], &value[tail..]) } fn sha256_id(bytes: &[u8]) -> String { @@ -9395,10 +10799,13 @@ fn usage() -> &'static str { "Idunn deployment, admission, and continuity control plane\n\n\ idunn serve [runtime options] [--host-actuator-bind ADDR]\n\ idunn up [--state-store PATH] [--no-wait]\n\ - idunn status [--state-store PATH] [--command ID]\n\ + idunn status [--state-store PATH] [--incident-store PATH] [--command ID]\n\ idunn cancel [--state-store PATH]\n\ idunn expire [--state-store PATH]\n\ - idunn validate --recipe PATH [--binding PATH]\n\n\ + idunn validate --recipe PATH [--binding PATH]\n\ + idunn orders serve [--store DIR] [--steward-binding PATH] [--socket PATH]\n\ + idunn order --reason TEXT\n\ + idunn order show ID | idunn order list\n\n\ Recipes describe capability and process requirements. Idunn seals exact\n\ source and artifacts, admits one incarnation, and delegates execution to\n\ systemd and routing mechanics to the configured proxy." @@ -9415,6 +10822,7 @@ mod tests { use super::*; use crate::drivers::{SystemdWorkloadObservation, incarnation_key}; + use crate::incident::INCIDENT_RETENTION_MILLIS; /// Fixed clock for the signing fixture; correlations must land inside /// DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS of it to authenticate. @@ -9439,6 +10847,7 @@ mod tests { private_port_end, config_path: PathBuf::from(format!("/etc/nginx/idunn-stream-routes/{route_id}.conf")), reload_unit: "nginx.service".into(), + member: None, } } @@ -9650,6 +11059,7 @@ mod tests { source_root: root.join("sources"), staging_root: root.join("staging"), topology_store: root.join("topology.cc"), + incident_store: root.join("incidents.cc"), odin_correlation_store: root.join("odin-correlation.cc"), odin_trust_anchor: odin_anchor, idunn_identity_store: idunn, @@ -10607,9 +12017,45 @@ mod tests { vec!["serve", "--swarm-profile", "yggdrasil-local"], vec!["serve", "--restart-command", "bad"], vec!["validate-runtime-admission"], + vec!["order", "unit", "restart", "nginx.service", "--reason", "r", "--command", "sh -c bad"], + vec!["order", "unit", "restart", "nginx.service", "--reason", "r", "--shell"], + vec!["order", "run", "sh", "--reason", "r"], + vec!["order", "package-install", "git-lfs", "--reason", "r", "--deploy-command", "x"], + vec!["order", "unit", "restart", "nginx.service", "extra", "--reason", "r"], + vec!["order", "deploy", "vili", "--reason", "r", "--requested-by", "someone"], + vec!["orders", "serve", "--allow-units", "nginx.service"], + vec!["orders", "run"], ] { assert!(parse(arguments.into_iter().map(str::to_owned)).is_err()); } + // The declarative order forms parse, and only as order commands. + #[cfg(unix)] + for arguments in [ + vec!["order", "unit", "restart", "nginx.service", "--reason", "r"], + vec!["order", "package-install", "git-lfs", "--reason", "r"], + vec!["order", "deploy", "vili", "--reason", "r"], + vec!["order", "list"], + vec!["orders", "serve"], + ] { + let parsed = parse(arguments.into_iter().map(str::to_owned)).unwrap(); + assert!(matches!(parsed, Command::Order(_) | Command::OrdersServe(_))); + } + } + + #[test] + fn deployment_commit_primitive_is_shared_and_never_overwrites() -> Result<()> { + let temp = TempDir::new()?; + let store = temp.path().join("control.cc"); + commit_deployment_command(&store, "up-one", "vili", "steward:eureka order ord-1")?; + let snapshot = ControlSnapshot::read(&store)?; + assert_eq!(snapshot.commands.len(), 1); + let command = &snapshot.commands[0].value; + assert_eq!(command.command_id, "up-one"); + assert_eq!(command.selector, "vili"); + assert_eq!(command.requested_by, "steward:eureka order ord-1"); + assert!(commit_deployment_command(&store, "up-one", "other", "x").is_err()); + assert_eq!(ControlSnapshot::read(&store)?.commands.len(), 1); + Ok(()) } #[test] @@ -11234,8 +12680,9 @@ mod tests { }); assert!(!transaction.is_terminal()); - assert!(!transaction.owns_target_authority()); - assert!(transaction.blocks_new_target_mutation()); + assert!(!transaction.is_live()); + assert!(!transaction.owns_target()); + assert!(transaction.drains_incumbent()); Ok(()) } @@ -11474,7 +12921,7 @@ mod tests { ); assert_eq!(lifted.phase_deadline, None); assert_eq!(lifted.lease_adoption, None); - assert_eq!(lifted.owns_target_authority(), legacy.phase != DeploymentPhase::Complete); + assert_eq!(lifted.is_live(), legacy.phase != DeploymentPhase::Complete); } assert_eq!( phases.iter().map(|(_, phase)| *phase).collect::>(), @@ -11745,11 +13192,12 @@ mod tests { } #[test] - fn phase_deadlines_follow_the_plan_and_only_cover_post_fencing_phases() -> Result<()> { + fn phase_deadlines_follow_the_plan_and_cover_warming_through_committing() -> Result<()> { let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; let plan = &generation.plan; let defaults = plan.phase_deadlines(); for (phase, seconds) in [ + (DeploymentPhase::Warming, WARMING_DEADLINE_SECONDS), (DeploymentPhase::Fencing, defaults.fencing_seconds), (DeploymentPhase::Leasing, defaults.leasing_seconds), (DeploymentPhase::AwaitingReady, defaults.awaiting_ready_seconds), @@ -11764,7 +13212,6 @@ mod tests { for phase in [ DeploymentPhase::Sealing, DeploymentPhase::Starting, - DeploymentPhase::Warming, DeploymentPhase::Complete, ] { assert!(PhaseDeadline::entering(phase, plan, 1_000).is_none()); @@ -11821,7 +13268,7 @@ mod tests { } #[test] - fn a_deadline_must_describe_the_current_post_fencing_phase() -> Result<()> { + fn a_deadline_must_describe_the_records_current_activity() -> Result<()> { let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; let plan = transaction.plan.clone().unwrap(); let now = transaction.updated_at_unix_millis; @@ -11836,10 +13283,25 @@ mod tests { backwards.phase_deadline.as_mut().unwrap().deadline_at_unix_millis = now; assert!(backwards.validate().is_err()); // Entry is not ordered against the wall-clock stamp of later writes. - let mut entered_after_the_stamp = transaction; + let mut entered_after_the_stamp = transaction.clone(); entered_after_the_stamp.phase_deadline = PhaseDeadline::entering(DeploymentPhase::AwaitingReady, &plan, now + 1); entered_after_the_stamp.validate()?; + + // The activity rule: Warming carries a deadline; Sealing only while + // its abort is pending; Complete only once admitted. + let mut other = transaction.clone(); + other.phase = DeploymentPhase::Warming; + assert!(other.carries_a_deadline()); + other.phase = DeploymentPhase::Sealing; + assert!(!other.carries_a_deadline(), "Sealing has no activity to time"); + other.pre_fencing_abort = Some(pre_fencing_abort_intent(&other, "refused")); + assert!(other.carries_a_deadline()); + other.pre_fencing_abort = None; + other.phase = DeploymentPhase::Complete; + assert!(!other.carries_a_deadline()); + other.completion = Some(TransactionCompletion::Admitted { generation_id: "g".into() }); + assert!(other.carries_a_deadline()); Ok(()) } @@ -11900,6 +13362,9 @@ mod tests { fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { Ok(false) } + fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { + Ok(false) + } } /// A Continuity transaction for an unrouted, stateless target with no @@ -11931,10 +13396,17 @@ mod tests { IDUNN_DEPLOYMENT_BRAKE_SCOPE, IDUNN_DEPLOYMENT_RELEASE_PURPOSE, IdunnDeploymentBrakeReleasePurpose, enroll_service_identity_at, }; - let signer = enroll_service_identity_at::( - &world.root.join("identities/brake-operator.cc"), - )?; - export_service_identity_trust_anchor(&signer, &world.root.join("brake-anchor.cc"))?; + // One operator per world, however many deployments it releases. + let signer_path = world.root.join("identities/brake-operator.cc"); + let signer = if signer_path.exists() { + open_service_identity_at::(&signer_path)? + } else { + let signer = + enroll_service_identity_at::(&signer_path)?; + export_service_identity_trust_anchor(&signer, &world.root.join("brake-anchor.cc"))?; + signer + }; + let authorization_id = format!("authorization-{transaction_id}"); let mut record = IdunnDeploymentBrakeRecord { schema_version: IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into(), brake_id: IDUNN_DEPLOYMENT_BRAKE_ID.into(), @@ -11945,7 +13417,7 @@ mod tests { reason: "test rollout".into(), observed_at_unix_millis: now, expires_at_unix_millis: Some(now + 600_000), - authorization_id: Some("authorization-test".into()), + authorization_id: Some(authorization_id.clone()), authorization_purpose: Some(IDUNN_DEPLOYMENT_RELEASE_PURPOSE.into()), authorized_release_id: Some(expected.sealed_release_id.clone()), authorized_deployment_id: Some(transaction_id.to_owned()), @@ -11965,13 +13437,34 @@ mod tests { record.validate()?; let canonical_brake_bytes = rmp_serde::to_vec(&record)?; Ok(DeploymentAuthorization { - authorization_id: "authorization-test".into(), + authorization_id, brake_sha256: sha256_id(&canonical_brake_bytes), canonical_brake_bytes, authorized_at_unix_millis: now, }) } + /// `binding` with its provider trust anchor and signer pointed at this + /// world's provider identity, enrolled on first use. + fn bind_provider(world: &EngineFixture, binding: &str) -> Result { + let provider_path = world.root.join("identities/provider.cc"); + let provider_anchor = world.root.join("identities/provider-anchor.cc"); + let provider = if provider_path.exists() { + open_service_identity_at::(&provider_path)? + } else { + let provider = + enroll_service_identity_at::(&provider_path)?; + export_service_identity_trust_anchor(&provider, &provider_anchor)?; + provider + }; + Ok(binding + .replace( + "/etc/gamecult/trust/service.cc", + &provider_anchor.display().to_string(), + ) + .replace("service-runtime-signer", &provider.entry().identity_id)) + } + /// What makes a target Odin: it provides the rendezvous. fn provide_odin(expected: &mut IdunnExpectedIncarnationRecord) { expected.capabilities.push(cultnet_rs::IdunnExpectedCapability { @@ -12024,32 +13517,19 @@ mod tests { use crate::deployment_plan::tests::{ BINDING, RECIPE, artifact_receipt, external_input_receipt, source, }; - let provider_path = world.root.join("identities/provider.cc"); - let provider_anchor = world.root.join("identities/provider-anchor.cc"); - let provider = if provider_path.exists() { - open_service_identity_at::(&provider_path)? - } else { - let provider = - enroll_service_identity_at::(&provider_path)?; - export_service_identity_trust_anchor(&provider, &provider_anchor)?; - provider - }; - let (binding_head, binding_tail) = BINDING .split_once("[route]") .context("binding has no route table")?; - let binding = format!( - "{binding_head}[brakes]{}", - binding_tail - .split_once("[brakes]") - .context("binding has no brakes table")? - .1 - ) - .replace( - "/etc/gamecult/trust/service.cc", - &provider_anchor.display().to_string(), - ) - .replace("service-runtime-signer", &provider.entry().identity_id); + let binding = bind_provider( + world, + &format!( + "{binding_head}[brakes]{}", + binding_tail + .split_once("[brakes]") + .context("binding has no brakes table")? + .1 + ), + )?; let recipe = RECIPE .split("[[dependencies]]") .next() @@ -12093,16 +13573,23 @@ mod tests { declare_odin(&mut expected); let now = now_millis()?; + // A second seeding of one kind into one world is a second command. + let base_id = format!( + "{}-service{}", + match kind { + CommandKind::Deploy => "up", + CommandKind::Continuity => "continuity", + }, + if incumbent.is_some() { "-again" } else { "" } + ); + let known = ControlSnapshot::read(&world.state_store)?.commands; + let command_id = (1..) + .map(|n| if n == 1 { base_id.clone() } else { format!("{base_id}-{n}") }) + .find(|id| known.iter().all(|stored| stored.value.command_id != *id)) + .context("no free command id")?; let command = DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: format!( - "{}-service{}", - match kind { - CommandKind::Deploy => "up", - CommandKind::Continuity => "continuity", - }, - if incumbent.is_some() { "-again" } else { "" } - ), + command_id, kind, selector: "service".into(), requested_by: "test".into(), @@ -12324,29 +13811,55 @@ mod tests { } } - /// One engine step at a time until `stop` holds, recording each phase - /// entered and checking after every step that the deadline on the record - /// is exactly the current post-fencing phase's, and absent outside one. + /// The transaction wherever it now lives: live, or archived once terminal. + fn latest_id(world: &EngineFixture, id: &str) -> Result { + match resident_id(world, id) { + Ok(stored) => Ok(stored.value), + Err(_) => read_history_transactions(&world.state_store) + .into_iter() + .find(|transaction| transaction.transaction_id == id) + .context("the transaction is in neither the live set nor history"), + } + } + + /// `drive_id` for the world's only transaction. fn drive( world: &EngineFixture, stop: impl Fn(&DeploymentTransaction) -> bool, + ) -> Result> { + match resident(world) { + Ok(sole) => drive_id(world, &sole.value.transaction_id, stop), + Err(_) => Ok(Vec::new()), + } + } + + /// One engine step of transaction `id` at a time until `stop` holds, + /// recording each phase entered and checking after every step that the + /// deadline on the record is exactly the current phase's, and absent + /// outside a deadline phase. + fn drive_id( + world: &EngineFixture, + id: &str, + stop: impl Fn(&DeploymentTransaction) -> bool, ) -> Result> { let mut phases = Vec::new(); for _ in 0..40 { - let Ok(current) = resident(world) else { + let Ok(current) = resident_id(world, id) else { return Ok(phases); }; if stop(¤t.value) { return Ok(phases); } world.engine.advance_transaction(¤t)?; - let after = latest(world)?; + let after = latest_id(world, id)?; match &after.phase_deadline { Some(deadline) => assert_eq!(deadline.phase, after.phase), + // A waiter's Warming deadline is disarmed while another + // transaction owns the target. None => assert!( - !(DeploymentPhase::Fencing..=DeploymentPhase::Committing) - .contains(&after.phase), - "a post-fencing record lost its deadline" + after.phase == DeploymentPhase::Warming + || !DEADLINE_PHASES.contains(&after.phase), + "a deadline-phase record lost its deadline" ), } if phases.last() != Some(&after.phase) { @@ -12392,15 +13905,28 @@ mod tests { } #[test] - fn a_pre_fence_abort_runs_to_complete_without_a_deadline() -> Result<()> { + fn a_pre_fence_abort_is_timed_from_its_beginning_and_ends_without_a_deadline() -> Result<()> { let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; transaction_at(&world, DeploymentPhase::Warming)?; let warming = resident(&world)?; - assert_eq!(warming.value.phase_deadline, None); + let warming_deadline = warming.value.phase_deadline.context("Warming has no deadline")?; + assert_eq!( + warming_deadline.deadline_at_unix_millis - warming_deadline.entered_at_unix_millis, + u64::from(WARMING_DEADLINE_SECONDS) * 1000 + ); + let before = now_millis()?; world .engine .begin_pre_fencing_abort(&warming, anyhow!("candidate refused"))?; + let aborting = latest(&world)?; + let deadline = aborting.phase_deadline.context("the abort has no deadline")?; + assert_eq!(deadline.phase, DeploymentPhase::Warming); + assert!(deadline.entered_at_unix_millis >= before); + assert_eq!( + deadline.deadline_at_unix_millis - deadline.entered_at_unix_millis, + u64::from(PRE_FENCING_ABORT_DEADLINE_SECONDS) * 1000 + ); drive(&world, |transaction| transaction.completion.is_some())?; let finished = latest(&world)?; assert_eq!(finished.phase, DeploymentPhase::Complete); @@ -12444,6 +13970,118 @@ mod tests { Ok(()) } + #[test] + fn an_abort_is_timed_from_its_beginning_with_its_phases_duration() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; + odin_reports_ready(&world, &seeded, 5)?; + drive(&world, |transaction| { + transaction.phase == DeploymentPhase::Routing + })?; + // The phase's own deadline is long past when the abort begins. + let routing = resident(&world)?; + let mut stale = routing.value.clone(); + let deadline = stale.phase_deadline.as_mut().context("Routing has no deadline")?; + deadline.entered_at_unix_millis = 1; + deadline.deadline_at_unix_millis = 2; + replace_transaction(&world.state_store, &routing, &stale)?; + + let before = now_millis()?; + world + .engine + .begin_post_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + let aborting = latest(&world)?; + let deadline = aborting.phase_deadline.context("the abort has no deadline")?; + assert_eq!(deadline.phase, DeploymentPhase::Routing); + assert!(deadline.entered_at_unix_millis >= before); + let routing_millis = + u64::from(aborting.plan.as_ref().unwrap().phase_deadlines().routing_seconds) * 1000; + assert_eq!( + deadline.deadline_at_unix_millis - deadline.entered_at_unix_millis, + routing_millis + ); + assert!(!aborting.leaves_to_operator(deadline.deadline_at_unix_millis - 1)); + assert!(aborting.leaves_to_operator(deadline.deadline_at_unix_millis)); + Ok(()) + } + + #[test] + fn a_drain_whose_stored_plan_no_longer_parses_is_still_bounded() -> Result<()> { + let (mut draining, promoted_at) = promoted_draining_record()?; + let committing = u64::from(draining.plan.as_ref().unwrap().phase_deadlines().committing_seconds) * 1000; + // A binding the running binary refuses: the drain seconds cannot be read. + draining.plan.as_mut().unwrap().binding_blob = b"not an operator binding".to_vec(); + assert!(draining.plan.as_ref().unwrap().parsed_inputs().is_err()); + draining.retime(promoted_at + 1); + let deadline = draining.phase_deadline.context("an unparsable plan left the drain with no deadline")?; + assert_eq!(deadline.deadline_at_unix_millis, promoted_at + 1 + committing); + assert!(!draining.leaves_to_operator(deadline.deadline_at_unix_millis - 1)); + assert!(draining.leaves_to_operator(deadline.deadline_at_unix_millis)); + Ok(()) + } + + /// A Complete record draining its incumbent, whose route was promoted at + /// the returned time. + fn promoted_draining_record() -> Result<(DeploymentTransaction, u64)> { + let (_, mut draining) = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?; + draining.phase = DeploymentPhase::Complete; + draining.completion = Some(TransactionCompletion::Admitted { + generation_id: "generation-new".into(), + }); + draining.post_commit_cleanup = Some(PostCommitCleanup { + incumbent: IncumbentCleanupEvidence::Pending { + generation_id: "generation-old".into(), + workload: workload(1001, 40, 50), + }, + source: SourceCleanupEvidence::Complete, + }); + let promoted_at = 10_000_000_u64; + match draining.routing.as_mut() { + Some(RoutingEvidence::Promoted { promoted_at_unix_millis, .. }) => { + *promoted_at_unix_millis = promoted_at; + } + other => bail!("the fixture is not promoted: {other:?}"), + } + Ok((draining, promoted_at)) + } + + #[test] + fn a_drain_is_timed_to_its_retire_time_plus_committing() -> Result<()> { + let (mut draining, promoted_at) = promoted_draining_record()?; + let plan = draining.plan.clone().unwrap(); + let drain = u64::from(plan.parsed_inputs()?.1.rollout.drain_seconds) * 1000; + let committing = u64::from(plan.phase_deadlines().committing_seconds) * 1000; + assert!(drain > 0 && committing > 0, "the fixture must make both terms matter"); + + // Before the retire time: the retire time plus the committing duration. + draining.retime(promoted_at + 1); + let deadline = draining.phase_deadline.context("a drain has no deadline")?; + assert_eq!(deadline.phase, DeploymentPhase::Complete); + assert_eq!(deadline.entered_at_unix_millis, promoted_at + 1); + assert_eq!(deadline.deadline_at_unix_millis, promoted_at + drain + committing); + assert!(!draining.leaves_to_operator(deadline.deadline_at_unix_millis - 1)); + assert!(draining.leaves_to_operator(deadline.deadline_at_unix_millis)); + + // After it: the committing duration from now. + let late = promoted_at + drain + 5_000; + draining.retime(late); + assert_eq!(draining.phase_deadline.unwrap().deadline_at_unix_millis, late + committing); + + // Nothing promoted: from now. + draining.routing = Some(RoutingEvidence::SkippedUnrouted); + draining.retime(promoted_at); + assert_eq!(draining.phase_deadline.unwrap().deadline_at_unix_millis, promoted_at + committing); + + // A finished drain, and any other Complete, carry none. + draining.post_commit_cleanup = Some(PostCommitCleanup { + incumbent: IncumbentCleanupEvidence::Complete { generation_id: "generation-old".into() }, + source: SourceCleanupEvidence::Complete, + }); + draining.retime(promoted_at); + assert_eq!(draining.phase_deadline, None); + Ok(()) + } + #[test] fn entering_a_phase_owns_the_deadline_and_a_stepped_clock_cannot_wedge_it() -> Result<()> { let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?; @@ -12879,11 +14517,45 @@ mod tests { Ok(()) } + /// The slow cadence belongs to a record that is left to the operator, + /// whichever way it came to be: it waits up to the continuity cap between + /// attempts from its deadline on, and up to the short cap before. + #[test] + fn a_record_left_to_the_operator_backs_off_to_the_continuity_cap() -> Result<()> { + let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; + let lease = write_lease(); + let leased = LeasingEvidence::Granted { + lease_sha256: lease.canonical_sha256()?, + lease, + }; + transaction.retime(1_000); + let due = transaction.phase_deadline.context("no deadline")?.deadline_at_unix_millis; + let mut aborting = transaction.clone(); + aborting.post_fencing_abort = Some(post_fencing_abort_intent(&aborting, "wedged")); + let mut deploy = transaction.clone(); + deploy.command_kind = CommandKind::Deploy; + let mut stateful = deploy.clone(); + stateful.leasing = Some(leased); + let mut continuity = transaction.clone(); + continuity.command_kind = CommandKind::Continuity; + for (what, record) in [ + ("a forward-retrying continuity", &continuity), + ("a stateful deploy", &stateful), + ("a wedged abort", &aborting), + ] { + assert_eq!(resume_ceiling(record, due - 1), RESUME_BACKOFF_CEILING_MILLIS, "{what} before its deadline"); + assert_eq!(resume_ceiling(record, due), CONTINUITY_BACKOFF_CEILING_MILLIS, "{what} at its deadline"); + } + // A deploy that aborts at its deadline is not left to anyone. + assert_eq!(resume_ceiling(&deploy, due), RESUME_BACKOFF_CEILING_MILLIS); + Ok(()) + } + /// Q3 as a table: which ended phases abort, which name the unadopted /// lease, which are left to the operator, and which recovery the terminal /// record names. #[test] - fn an_ended_phase_aborts_unless_a_deployment_candidate_was_issued_a_lease() -> Result<()> { + fn an_ended_phase_is_resolved_by_command_kind_and_lease() -> Result<()> { let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; let lease = write_lease(); let sha256 = lease.canonical_sha256()?; @@ -12914,16 +14586,27 @@ mod tests { (Deploy, Some(prepared.clone()), None, PhaseEnd::OperatorRequired), (Deploy, Some(granted.clone()), None, PhaseEnd::OperatorRequired), (Deploy, Some(granted.clone()), adoption.clone(), PhaseEnd::OperatorRequired), - (Continuity, Some(LeasingEvidence::SkippedStateless), None, abort(false)), - (Continuity, Some(prepared), None, abort(false)), - (Continuity, Some(granted.clone()), None, abort(true)), - (Continuity, Some(granted), adoption.clone(), abort(false)), + (Continuity, None, None, PhaseEnd::RetryForward), + (Continuity, Some(LeasingEvidence::SkippedStateless), None, PhaseEnd::RetryForward), + (Continuity, Some(prepared), None, PhaseEnd::RetryForward), + (Continuity, Some(granted.clone()), None, PhaseEnd::RetryForward), + (Continuity, Some(granted), adoption.clone(), PhaseEnd::RetryForward), ]; for (kind, leasing, lease_adoption, expected) in cases { transaction.command_kind = kind; transaction.leasing = leasing.clone(); transaction.lease_adoption = lease_adoption.clone(); assert_eq!(transaction.phase_end(), expected, "{kind:?} {leasing:?} {lease_adoption:?}"); + // Only a stateful deploy and a continuity are left to the operator + // when their phase ends; the other deploys abort. + transaction.retime(1_000); + let due = transaction.phase_deadline.context("no deadline")?.deadline_at_unix_millis; + assert_eq!( + transaction.leaves_to_operator(due), + matches!(expected, PhaseEnd::OperatorRequired | PhaseEnd::RetryForward), + "{kind:?} {leasing:?} {lease_adoption:?}" + ); + assert!(!transaction.leaves_to_operator(due - 1)); let recovery = if kind == Continuity && lease_adoption.is_some() { TerminalRecovery::RestartAdmitted } else { @@ -13171,15 +14854,109 @@ mod tests { /// `StillWorkload`'s. struct SwitchWorkload { alive: std::sync::atomic::AtomicBool, + rebooted: std::sync::atomic::AtomicBool, + silent: std::sync::atomic::AtomicBool, + /// Runtime instances that stay alive when the rest are killed: a + /// candidate building beside a dead incumbent. + spared: std::sync::Mutex>, + /// Whether a start launches a process (a fresh private identity) + /// instead of refusing like `StillWorkload`. + launches: std::sync::atomic::AtomicBool, + /// Files whose advisory locks stand for a process that holds a lease + /// for its lifetime: `stop` is what releases them. + holds: std::sync::Mutex>, + /// How many times the process was stopped. + stops: std::sync::atomic::AtomicUsize, + /// Runtime instances whose stop fails, whatever is asked. + failing_stops: std::sync::Mutex>, + /// The unit is absent or failed, which the driver reports as + /// permanently stopped: positive evidence of death, unlike an + /// observation that merely errs. + buried: std::sync::atomic::AtomicBool, } impl SwitchWorkload { fn new() -> Arc { Arc::new(Self { alive: std::sync::atomic::AtomicBool::new(true), + rebooted: std::sync::atomic::AtomicBool::new(false), + silent: std::sync::atomic::AtomicBool::new(false), + spared: std::sync::Mutex::new(Vec::new()), + launches: std::sync::atomic::AtomicBool::new(false), + holds: std::sync::Mutex::new(Vec::new()), + stops: std::sync::atomic::AtomicUsize::new(0), + failing_stops: std::sync::Mutex::new(Vec::new()), + buried: std::sync::atomic::AtomicBool::new(false), }) } + /// The process is dead and the driver can prove it. + fn bury(&self) { + self.kill(); + self.buried.store(true, std::sync::atomic::Ordering::SeqCst); + } + + /// From now on a start launches a process distinct from every recorded one. + fn launching(&self) { + self.launches.store(true, std::sync::atomic::Ordering::SeqCst); + } + + /// The process takes a shared lock on `store`'s authority lock, as a + /// candidate holding its lifetime write lease does, until stopped. + #[cfg(unix)] + fn hold_lease_lock(&self, store: &std::path::Path) { + use std::os::unix::io::AsRawFd; + let mut lock = store.as_os_str().to_os_string(); + lock.push(".lock"); + let file = std::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(false) + .open(lock) + .unwrap(); + // SAFETY: flock on a descriptor this function owns. + assert_eq!(unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_SH) }, 0); + self.holds.lock().unwrap().push(file); + } + + /// Every stop works again. + fn mend_stops(&self) { + self.failing_stops.lock().unwrap().clear(); + } + + /// From now on stopping this runtime instance fails. + fn fail_stop_of(&self, runtime_instance_id: &str) { + self.failing_stops.lock().unwrap().push(runtime_instance_id.to_owned()); + } + + /// `kill` leaves this runtime instance running. + fn spare(&self, runtime_instance_id: &str) { + self.spared.lock().unwrap().push(runtime_instance_id.to_owned()); + } + + /// The host cannot be asked: observing fails as `HostUnobservable` + /// whatever the workload is doing. + fn silence(&self, silent: bool) { + self.silent + .store(silent, std::sync::atomic::Ordering::SeqCst); + } + + fn revive(&self) { + self.buried + .store(false, std::sync::atomic::Ordering::SeqCst); + self.alive + .store(true, std::sync::atomic::Ordering::SeqCst); + self.rebooted + .store(false, std::sync::atomic::Ordering::SeqCst); + } + + /// The host rebooted under the unit: it is gone and the driver can tell. + fn reboot(&self) { + self.rebooted + .store(true, std::sync::atomic::Ordering::SeqCst); + self.kill(); + } + fn kill(&self) { self.alive .store(false, std::sync::atomic::Ordering::SeqCst); @@ -13200,6 +14977,9 @@ mod tests { expected: &IdunnExpectedIncarnationRecord, launch: IdunnRuntimeActivationLaunch, ) -> Result { + if self.launches.load(std::sync::atomic::Ordering::SeqCst) { + return Ok(launch.activation().clone()); + } StillWorkload.prepare_activation(plan, expected, launch) } fn start_prepared( @@ -13210,7 +14990,25 @@ mod tests { expected: &IdunnExpectedIncarnationRecord, activation: &IdunnRuntimeActivationRecord, ) -> Result { - StillWorkload.start_prepared(plan, release, installed, expected, activation) + if !self.launches.load(std::sync::atomic::Ordering::SeqCst) { + return StillWorkload.start_prepared(plan, release, installed, expected, activation); + } + let mut workload = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)? + .1 + .workload + .context("recorded transaction has no workload")?; + match &mut workload { + WorkloadObservation::Systemd(observed) => { + observed.runtime_instance_id = activation.runtime_instance_id.clone(); + observed.executable_sha256 = expected.artifact_sha256.clone(); + // Another UID and other namespaces than any recorded unit. + observed.process_uids[0] += 1_000_000; + observed.pid_namespace_id += 1_000_000; + observed.mount_namespace_id += 1_000_000; + } + WorkloadObservation::Host(_) => bail!("recorded workload is not a systemd unit"), + } + Ok(workload) } fn discard_prepared( &self, @@ -13226,34 +15024,66 @@ mod tests { _: &IdunnRuntimeActivationRecord, prior: &WorkloadObservation, ) -> Result { + if self.silent.load(std::sync::atomic::Ordering::SeqCst) { + return Err(HostUnobservable { + host: "test-host".into(), + reason: "the actuator is between sessions".into(), + } + .into()); + } + let spared = match prior { + WorkloadObservation::Systemd(observed) => { + self.spared.lock().unwrap().contains(&observed.runtime_instance_id) + } + WorkloadObservation::Host(_) => false, + }; ensure!( - self.alive.load(std::sync::atomic::Ordering::SeqCst), + spared || self.alive.load(std::sync::atomic::Ordering::SeqCst), "the workload is gone" ); Ok(prior.clone()) } - fn stop(&self, _: &WorkloadObservation) -> Result<()> { + fn stop(&self, stopping: &WorkloadObservation) -> Result<()> { + self.stops.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + if let WorkloadObservation::Systemd(observed) = stopping { + ensure!( + !self.failing_stops.lock().unwrap().contains(&observed.runtime_instance_id), + "the unit would not stop" + ); + } + self.holds.lock().unwrap().clear(); Ok(()) } fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { - Ok(false) + Ok(self.buried.load(std::sync::atomic::Ordering::SeqCst)) + } + fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { + Ok(self.rebooted.load(std::sync::atomic::Ordering::SeqCst)) } } /// Admit a generation for "service" through the real phase machine and /// retire its transaction, leaving the world with an incumbent. fn admit_incumbent(world: &EngineFixture) -> Result { + Ok(admit_seeded_incumbent(world)?.0) + } + + /// `admit_incumbent`, and the transaction that was admitted. + fn admit_seeded_incumbent( + world: &EngineFixture, + ) -> Result<(AdmittedGeneration, DeploymentTransaction)> { let seeded = transaction_at(world, DeploymentPhase::Fencing)?; odin_reports_ready(world, &seeded, seeded_sequence(None) + 1)?; drive(world, |transaction| { transaction.phase == DeploymentPhase::Complete })?; assert!(world.engine.retire_one_terminal_transaction()?); - Ok(ControlSnapshot::read(&world.state_store)? + let admitted = ControlSnapshot::read(&world.state_store)? .admitted_for("service") .context("commit wrote no admitted generation")? .value - .clone()) + .clone(); + Ok((admitted, seeded)) } #[test] @@ -13345,6 +15175,8 @@ mod tests { let mut meters = TargetSupervision::new("service"); meters.continuity_restarts = ages.iter().map(|age| now - age).collect(); meters.continuity_restarts.sort_unstable(); + // Every logged attempt belongs to one streak. + meters.continuity_attempt_streak = ages.len() as u32; Ok(meters) } @@ -13819,7 +15651,7 @@ mod tests { assert!(lifted.is_terminal(), "{name}: the lift reopened a finished record"); assert_eq!(lifted.phase, archived.phase, "{name}"); assert_eq!(lifted.completion, archived.completion, "{name}"); - assert!(!lifted.owns_target_authority(), "{name}"); + assert!(!lifted.is_live(), "{name}"); } else { assert_eq!(abort_owed(&lifted), Some(CleanupEvidence::Pending), "{name}"); assert!(!lifted.is_terminal(), "{name}"); @@ -14036,6 +15868,9 @@ mod tests { fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { Ok(true) } + fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { + Ok(false) + } } #[test] @@ -14161,6 +15996,9 @@ mod tests { fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { Ok(true) } + fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { + Ok(false) + } } fn queue_deploy_for_nowhere(world: &EngineFixture) -> Result { @@ -14346,6 +16184,9 @@ mod tests { fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { Ok(true) } + fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { + Ok(false) + } } #[test] @@ -14417,12 +16258,17 @@ mod tests { #[test] fn backoff_doubles_caps_and_waits_exactly() { - assert_eq!(backoff_wait(500, 0), 500); - assert_eq!(backoff_wait(500, 1), 1000); - assert_eq!(backoff_wait(500, 2), 2000); - assert_eq!(backoff_wait(500, 3), 4000); - assert_eq!(backoff_wait(500, 30), RESUME_BACKOFF_CEILING_MILLIS); - assert_eq!(backoff_wait(u64::MAX, 5), RESUME_BACKOFF_CEILING_MILLIS); + let ceiling = RESUME_BACKOFF_CEILING_MILLIS; + assert_eq!(backoff_wait(500, 0, ceiling), 500); + assert_eq!(backoff_wait(500, 1, ceiling), 1000); + assert_eq!(backoff_wait(500, 2, ceiling), 2000); + assert_eq!(backoff_wait(500, 3, ceiling), 4000); + assert_eq!(backoff_wait(500, 30, ceiling), ceiling); + assert_eq!(backoff_wait(u64::MAX, 5, ceiling), ceiling); + // The ceiling is the caller's: continuity's is lower than the resume path's. + assert_eq!(backoff_wait(5_000, 5, 300_000), 160_000); + assert_eq!(backoff_wait(5_000, 6, 300_000), 300_000); + assert_eq!(backoff_wait(5_000, 20, 300_000), 300_000); assert!(is_waiting(9, 10, 5)); assert!(!is_waiting(10, 10, 5)); assert!(!is_waiting(11, 10, 5)); @@ -14451,6 +16297,7 @@ mod tests { ResumeBackoff { failures: 3, not_before_unix_millis, + ceiling_millis: RESUME_BACKOFF_CEILING_MILLIS, }, ); }; @@ -14603,6 +16450,120 @@ mod tests { Ok(()) } + /// A workload port whose observation deletes every live transaction from + /// the control store once armed: the step ends, and the record is simply + /// gone, archived nowhere. + struct VanishingWorkload { + store: Mutex>, + armed: std::sync::atomic::AtomicBool, + } + + impl WorkloadPort for VanishingWorkload { + fn install( + &self, + plan: &CompiledDeploymentPlan, + release: &crate::drivers::MaterializedRelease, + ) -> Result { + StillWorkload.install(plan, release) + } + fn prepare_activation( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + launch: IdunnRuntimeActivationLaunch, + ) -> Result { + StillWorkload.prepare_activation(plan, expected, launch) + } + fn start_prepared( + &self, + plan: &CompiledDeploymentPlan, + release: &SealedRelease, + installed: &crate::drivers::InstalledReleaseObservation, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result { + StillWorkload.start_prepared(plan, release, installed, expected, activation) + } + fn discard_prepared( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result<()> { + StillWorkload.discard_prepared(plan, expected, activation) + } + fn observe( + &self, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + prior: &WorkloadObservation, + ) -> Result { + if self.armed.load(std::sync::atomic::Ordering::SeqCst) { + let path = self.store.lock().unwrap().clone().context("no store")?; + let gone = ControlSnapshot::read(&path)? + .transactions + .into_iter() + .map(|stored| stored.envelope) + .collect::>(); + assert!(SingleFileMessagePackBackingStore::new(&path).delete_batch_if_unchanged(&gone)?); + } + StillWorkload.observe(expected, activation, prior) + } + fn stop(&self, observation: &WorkloadObservation) -> Result<()> { + StillWorkload.stop(observation) + } + fn is_permanently_stopped(&self, observation: &WorkloadObservation) -> Result { + StillWorkload.is_permanently_stopped(observation) + } + fn lost_to_host_reboot(&self, observation: &WorkloadObservation, at: u64) -> Result { + StillWorkload.lost_to_host_reboot(observation, at) + } + } + + /// The operator's exit from a forward-retrying continuity is worded from + /// the driver the plan binds: a host-actuator target has no unit to stop. + #[test] + fn the_manual_exit_is_worded_from_the_targets_driver() -> Result<()> { + let (world, _deploy, host_continuity) = stop_first_world(SwitchWorkload::new())?; + let host = resident_id(&world, &host_continuity)?.value; + assert!(host.manual_exit().contains("managed host"), "{}", host.manual_exit()); + assert!(!host.manual_exit().contains("unit"), "{}", host.manual_exit()); + let systemd = EngineFixture::with_workload(SwitchWorkload::new())?; + let incumbent = admit_incumbent(&systemd)?; + let unit = seeded_transaction(&systemd, DeploymentPhase::Fencing, CommandKind::Continuity, Some(&incumbent))?; + assert!(unit.manual_exit().contains("unit"), "{}", unit.manual_exit()); + Ok(()) + } + + /// The step that ends a transaction archives it in its own write, so a + /// record gone from the live set is finished only if history holds it. One + /// that vanished from both is a fault, never progress. + #[test] + fn a_transaction_gone_from_both_the_live_set_and_history_is_a_fault() -> Result<()> { + let workload = Arc::new(VanishingWorkload { + store: Mutex::new(None), + armed: std::sync::atomic::AtomicBool::new(false), + }); + let (world, _deploy, owner) = stop_first_world(workload.clone())?; + *world.engine.host_workload.lock().unwrap() = Some(workload.clone()); + *workload.store.lock().unwrap() = Some(world.state_store.clone()); + warm_stop_first(&world, &owner)?; + // The Warming step waits for Odin; waiting again rewrites nothing, so + // the step succeeds without touching the record it will find missing. + world.engine.resume_candidate(&resident_id(&world, &owner)?)?; + let waited = resident_id(&world, &owner)?.value; + assert!(waited.last_error.is_some(), "the Warming step did not wait: {:?}", waited.phase); + workload.armed.store(true, std::sync::atomic::Ordering::SeqCst); + world.engine.resume_backoff.lock().unwrap().clear(); + let error = world + .engine + .resume_candidate(&resident_id(&world, &owner)?) + .expect_err("a record gone from both stores was read as a finished one"); + assert!(format!("{error:#}").contains("disappeared while checking scheduler progress"), "{error:#}"); + assert!(read_history_transactions(&world.state_store).iter().all(|archived| archived.transaction_id != owner)); + Ok(()) + } + // ---- fix 2: supervision owns the Expected-only precondition ---- #[test] @@ -14968,132 +16929,3242 @@ mod tests { Ok(()) } + // ---- operator incidents ---------------------------------------------- + + fn incident_path(world: &EngineFixture) -> PathBuf { + world.root.join("incidents.cc") + } + + fn incidents_of(world: &EngineFixture) -> Result> { + IncidentStore::new(&incident_path(world)).read() + } + + /// The control store never holds an incident: it would stop every older + /// binary that reads it. + fn assert_control_store_holds_no_incident(world: &EngineFixture) -> Result<()> { + ControlSnapshot::read(&world.state_store)?; + for envelope in SingleFileMessagePackBackingStore::new(&world.state_store) + .pull_all_read_only_snapshot()? + { + assert_ne!(envelope.r#type, "idunn.operator_incident"); + } + Ok(()) + } + + /// Six restarts inside the window, the last long enough ago that its wait is + /// over: only the ceiling holds. + fn spend_the_window(world: &EngineFixture) -> Result<()> { + set_meters( + world, + &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, + ) + } + + /// One entry has left the window, and the newest restart is recent enough + /// that continuity is still waiting: exhaustion is over, nothing restarts. + fn slide_the_window(world: &EngineFixture) -> Result<()> { + set_meters( + world, + &restarted_ago(&[3_700_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 10_000])?, + ) + } + #[test] - fn continuity_stops_when_the_targets_own_log_has_spent_the_window() -> Result<()> { + fn an_exhausted_target_opens_one_incident_across_ticks_and_restarts() -> Result<()> { let workload = SwitchWorkload::new(); let world = EngineFixture::with_workload(workload.clone())?; admit_incumbent(&world)?; workload.kill(); - // Six restarts inside the window, the last long enough ago that its - // doubling wait (160 s) is over: only the ceiling holds. - set_meters( - &world, - &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, - )?; - assert!(!world.engine.supervise_one_admitted_generation()?); + spend_the_window(&world)?; + assert!(!incident_path(&world).exists()); + + // The first pass over the dead target starts its wait: progress, no mint. + assert!(world.engine.supervise_one_admitted_generation()?); + let opened = incidents_of(&world)?; + assert_eq!(opened.len(), 1); + assert_eq!(opened[0].subject, "service"); + assert_eq!(opened[0].condition, IncidentCondition::ContinuityExhausted); + assert!(opened[0].is_open()); + let written = std::fs::read(incident_path(&world))?; + + for _ in 0..3 { + assert!(!world.engine.supervise_one_admitted_generation()?); + } + // An Idunn restart forgets its in-memory trace; the durable record + // keeps the decision from being written again. + let restarted = + Engine::open_with_systemd_workload(world.engine.options.clone(), workload.clone())?; + assert!(restarted.fault_reports.lock().unwrap().is_empty()); + assert!(!restarted.supervise_one_admitted_generation()?); + assert_eq!(std::fs::read(incident_path(&world))?, written); + assert_eq!(incidents_of(&world)?, opened); assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); + assert_control_store_holds_no_incident(&world) + } + + #[test] + fn an_exhaustion_incident_closes_on_observed_health_not_on_the_window() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + workload.kill(); + spend_the_window(&world)?; + world.engine.supervise_one_admitted_generation()?; + let first = incidents_of(&world)?; + assert_eq!(first.len(), 1); + + // The window slides while the workload is still dead: exhaustion is + // over, the target is not healthy, and the incident stays open. + slide_the_window(&world)?; + let _ = world.engine.supervise_one_admitted_generation()?; + assert_eq!(incidents_of(&world)?, first); + + // Seen running, it closes as recovered. + workload.revive(); + assert!(!world.engine.supervise_one_admitted_generation()?); + let closed = incidents_of(&world)?; + assert_eq!(closed.len(), 1); + assert_eq!(closed[0].incident_key, first[0].incident_key); assert!( - world - .engine - .fault_reports - .lock() - .unwrap() - .contains_key("continuity:service"), - "exhaustion was not reported" + closed[0] + .closed_at_unix_millis + .is_some_and(|at| at >= closed[0].opened_at_unix_millis) + ); + assert_eq!(closed[0].close_reason, Some(CloseReason::Recovered)); + + // A later exhaustion is a new record with a new opening time. + std::thread::sleep(Duration::from_millis(5)); + workload.kill(); + spend_the_window(&world)?; + world.engine.supervise_one_admitted_generation()?; + let again = incidents_of(&world)?; + assert_eq!(again.len(), 2); + let reopened = again + .iter() + .find(|record| record.is_open()) + .context("no new incident")?; + assert!(reopened.opened_at_unix_millis > first[0].opened_at_unix_millis); + assert_ne!(reopened.incident_key, first[0].incident_key); + assert_control_store_holds_no_incident(&world) + } + + #[test] + fn an_incident_whose_target_is_no_longer_admitted_closes() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + workload.kill(); + spend_the_window(&world)?; + world.engine.supervise_one_admitted_generation()?; + assert!(incidents_of(&world)?[0].is_open()); + + let admitted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("no admitted generation")? + .envelope + .clone(); + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store) + .delete_batch_if_unchanged(&[admitted])? + ); + world.engine.supervise_one_admitted_generation()?; + let records = incidents_of(&world)?; + assert_eq!(records.len(), 1); + assert_eq!(records[0].close_reason, Some(CloseReason::NoLongerAdmitted)); + assert_control_store_holds_no_incident(&world) + } + + #[test] + fn a_tick_retires_incidents_closed_past_retention() -> Result<()> { + let world = EngineFixture::with_workload(SwitchWorkload::new())?; + let store = IncidentStore::new(&incident_path(&world)); + let condition = IncidentCondition::ContinuityExhausted; + let long_ago = now_millis()? - INCIDENT_RETENTION_MILLIS - 10_000; + assert!(store.open(condition, "gone", long_ago)?); + assert!(store.close(condition, "gone", CloseReason::Recovered, long_ago + 1)?); + assert!(store.open(condition, "recent", now_millis()?)?); + assert!(store.close(condition, "recent", CloseReason::Recovered, now_millis()?)?); + + world.engine.supervise_one_admitted_generation()?; + let live = store.read()?; + assert_eq!(live.len(), 1); + assert_eq!(live[0].subject, "recent"); + assert!(world.root.join("incident-history.cc").exists()); + Ok(()) + } + + /// The same exhaustion and recovery sequence, reduced to what continuity + /// decided: each tick's return, and the transactions and restart log it left. + fn continuity_decisions( + world: &EngineFixture, + engine: &Engine, + workload: &SwitchWorkload, + ) -> Result> { + let mut decided = Vec::new(); + let tick = |decided: &mut Vec<(bool, usize, usize)>| -> Result<()> { + let moved = engine.supervise_one_admitted_generation()?; + let snapshot = ControlSnapshot::read(&world.state_store)?; + decided.push(( + moved, + snapshot.transactions.len(), + snapshot.supervision_or_new("service").continuity_restarts.len(), + )); + Ok(()) + }; + workload.kill(); + spend_the_window(world)?; + tick(&mut decided)?; + tick(&mut decided)?; + slide_the_window(world)?; + tick(&mut decided)?; + workload.revive(); + tick(&mut decided)?; + workload.kill(); + spend_the_window(world)?; + tick(&mut decided)?; + Ok(decided) + } + + #[test] + fn an_unwritable_incident_store_changes_nothing_continuity_decides() -> Result<()> { + const CANARY: &str = "CANARY-6b1d-secret-segment"; + let healthy_workload = SwitchWorkload::new(); + let healthy = EngineFixture::with_workload(healthy_workload.clone())?; + admit_incumbent(&healthy)?; + let expected = continuity_decisions(&healthy, &healthy.engine, &healthy_workload)?; + assert!(incidents_of(&healthy)?.iter().any(|record| record.is_open())); + + // Not permission-dependent: a store path beneath a regular file cannot + // be created for any uid, root included. + let broken_workload = SwitchWorkload::new(); + let mut broken = EngineFixture::with_workload(broken_workload.clone())?; + admit_incumbent(&broken)?; + let blocker = broken.root.join(CANARY); + std::fs::write(&blocker, b"a file, not a directory")?; + let mut options = broken.engine.options.clone(); + options.incident_store = blocker.join("incidents.cc"); + broken.engine = Engine::open_with_systemd_workload(options, broken_workload.clone())?; + + let actual = continuity_decisions(&broken, &broken.engine, &broken_workload)?; + assert_eq!(actual, expected, "a broken incident store changed a continuity decision"); + assert!(!blocker.join("incidents.cc").exists()); + + // The fault is traced once, under its operation, and echoes no path. + let reports = broken.engine.fault_reports.lock().unwrap(); + let faults = reports + .iter() + .filter(|(key, _)| key.starts_with("incident-store:")) + .collect::>(); + assert_eq!( + faults.len(), + 1, + "{:?}", + faults.iter().map(|(key, _)| key).collect::>() + ); + let message = faults[0].1.last.lock().unwrap().clone().context("fault cleared")?; + assert!(message.contains("open an operator incident"), "{message}"); + assert!(!message.contains(CANARY), "{message}"); + drop(reports); + assert_control_store_holds_no_incident(&broken) + } + + #[test] + fn status_renders_incidents_and_survives_an_unreadable_store() -> Result<()> { + const CANARY: &str = "CANARY-90c2-secret-segment"; + let temp = TempDir::new()?; + let now = now_millis()?; + let mut snapshot = ControlSnapshot::default(); + let meters = TargetSupervision::new("service"); + snapshot.targets.push(Stored { + envelope: target_supervision_envelope(&meters, now)?, + value: meters, + }); + + let readable = temp.path().join("incidents.cc"); + IncidentStore::new(&readable).open(IncidentCondition::ContinuityExhausted, "service", now)?; + let lines = render_targets(&snapshot, &readable, now); + assert!(lines[0].starts_with("target service "), "{lines:?}"); + assert_eq!( + lines.last().map(String::as_str), + Some(format!(" incident continuity-exhausted opened-at {now}").as_str()) ); - // The oldest leaves the window: five remain, so one more is allowed. - set_meters( - &world, - &restarted_ago(&[3_700_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, - )?; - assert!(world.engine.supervise_one_admitted_generation()?); - assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 1); + let unreadable = temp.path().join(CANARY); + std::fs::write(&unreadable, b"not a cultcache file")?; + let lines = render_targets(&snapshot, &unreadable, now); + assert!(lines[0].starts_with("target service "), "{lines:?}"); + let last = lines.last().context("no lines")?; + assert!(last.starts_with("incidents unreadable: "), "{last}"); + assert!(!lines.iter().any(|line| line.contains(CANARY)), "{lines:?}"); Ok(()) } #[test] - fn an_exhausted_target_says_when_its_route_ceiling_reopens_only_if_it_is_full() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - workload.kill(); - let said = || -> String { - world - .engine - .fault_reports - .lock() + fn the_incident_store_is_named_by_serve_and_status() { + let Command::Serve(options) = + parse(["serve", "--incident-store", "/tmp/i.cc"].into_iter().map(str::to_owned)) .unwrap() - .get("continuity:service") - .and_then(|report| report.last.lock().unwrap().clone()) - .unwrap_or_default() + else { + panic!("expected serve") }; - let mut meters = - restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?; - set_meters(&world, &meters)?; - assert!(!world.engine.supervise_one_admitted_generation()?); - let open = said(); - assert!(open.contains("free for a deployment"), "{open}"); - assert!(!open.contains("route ceiling"), "{open}"); - - world.engine.clear_fault("continuity:service"); - let now = now_millis()?; - meters.route_actuations = (0..ROUTE_ACTUATION_CEILING as u64).map(|n| now - 1_000 + n).collect(); - set_meters(&world, &meters)?; - let reopens_at = meters.route_reopens_at(now).unwrap(); - assert!(!world.engine.supervise_one_admitted_generation()?); - let full = said(); - assert!(full.contains(&format!("reopens at unix ms {reopens_at}")), "{full}"); - Ok(()) + assert_eq!(options.incident_store, PathBuf::from("/tmp/i.cc")); + assert!(usage().contains("idunn status [--state-store PATH] [--incident-store PATH]")); + let Command::Status { incident_store, .. } = + parse(["status", "--incident-store", "/tmp/j.cc"].into_iter().map(str::to_owned)) + .unwrap() + else { + panic!("expected status") + }; + assert_eq!(incident_store, PathBuf::from("/tmp/j.cc")); } + /// A standing fault is traced once. A key is cleared only by an operation + /// that succeeded, so another site's short-circuit must not end it: a + /// target that is running but still exhausted closes nothing, and that + /// used to clear the reconcile pass's fault and print it again every pass. #[test] - fn continuity_restarts_are_spaced_by_a_doubling_wait_and_counted_in_the_mint() -> Result<()> { + fn a_failing_reconcile_is_traced_once_while_a_running_target_stays_exhausted() -> Result<()> { let workload = SwitchWorkload::new(); let world = EngineFixture::with_workload(workload.clone())?; admit_incumbent(&world)?; - workload.kill(); - let restarts = |world: &EngineFixture| -> Result { - let progressed = world.engine.supervise_one_admitted_generation()?; - let minted = !ControlSnapshot::read(&world.state_store)?.transactions.is_empty(); - assert_eq!(progressed, minted); - clear_live_transactions(world)?; - Ok(minted) - }; + std::fs::write(incident_path(&world), b"not an incident store")?; + spend_the_window(&world)?; - // The first restart is minted, and the same write counts it. - assert!(restarts(&world)?); - assert_eq!(meters_of(&world, "service")?.continuity_restarts.len(), 1); - // At once again: the 5 s wait has not passed. - assert!(!restarts(&world)?); - // The second wait is 10 s and the third 20 s: each is the last one doubled. - set_meters(&world, &restarted_ago(&[100_000, 6_000])?)?; - assert!(!restarts(&world)?); - set_meters(&world, &restarted_ago(&[100_000, 11_000])?)?; - assert!(restarts(&world)?); - assert_eq!(meters_of(&world, "service")?.continuity_restarts.len(), 3); - set_meters(&world, &restarted_ago(&[200_000, 100_000, 15_000])?)?; - assert!(!restarts(&world)?); - set_meters(&world, &restarted_ago(&[200_000, 100_000, 21_000])?)?; - assert!(restarts(&world)?); + for pass in 0..6 { + world.engine.supervise_one_admitted_generation()?; + let reports = world.engine.fault_reports.lock().unwrap(); + assert!( + reports.contains_key("incident-store:close:*"), + "pass {pass}: the standing reconcile fault was cleared, so it prints again" + ); + } Ok(()) } #[test] - fn a_deployment_yields_to_a_restart_only_while_the_target_has_restarts_left() -> Result<()> { + fn a_release_that_runs_one_pass_after_each_restart_keeps_one_incident() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + + // Each cycle: the restarts are spent, the release dies, is seen running + // for one pass (its last restart took), and dies again. + for _ in 0..4 { + workload.kill(); + spend_the_window(&world)?; + world.engine.supervise_one_admitted_generation()?; + workload.revive(); + world.engine.supervise_one_admitted_generation()?; + let records = incidents_of(&world)?; + assert_eq!(records.len(), 1, "a flapping release opened another incident"); + assert!(records[0].is_open(), "running while still exhausted closed the incident"); + } + + // Real recovery: running, and the window no longer holds the restarts. + slide_the_window(&world)?; + world.engine.supervise_one_admitted_generation()?; + let records = incidents_of(&world)?; + assert_eq!(records.len(), 1); + assert_eq!(records[0].close_reason, Some(CloseReason::Recovered)); + assert_control_store_holds_no_incident(&world) + } + + #[test] + fn a_target_leaving_admission_closes_only_its_own_incident() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let service = admit_incumbent(&world)?; + let store = IncidentStore::new(&incident_path(&world)); + let condition = IncidentCondition::ContinuityExhausted; + let now = now_millis()?; + for subject in ["service", "neighbour", "gone"] { + assert!(store.open(condition, subject, now)?); + } + + // Two targets are admitted: service, and a neighbour. Only the target + // that is not admitted has its incident closed. + let mut snapshot = ControlSnapshot::read(&world.state_store)?; + let envelope = snapshot + .admitted_for("service") + .context("no admitted generation")? + .envelope + .clone(); + snapshot.admitted.push(Stored { + envelope, + value: AdmittedGeneration { + target: "neighbour".into(), + ..service + }, + }); + world.engine.reconcile_incidents(&snapshot); + + let closed = |subject: &str| -> Result> { + Ok(incidents_of(&world)? + .into_iter() + .find(|record| record.subject == subject) + .context("no such incident")? + .close_reason) + }; + assert_eq!(closed("service")?, None); + assert_eq!(closed("neighbour")?, None); + assert_eq!(closed("gone")?, Some(CloseReason::NoLongerAdmitted)); + Ok(()) + } + + #[cfg(target_os = "linux")] + #[test] + fn a_held_incident_lock_changes_nothing_continuity_decides() -> Result<()> { + use crate::incident::tests::{STUCK_AFTER, hold_lock_of, run_while_held}; + + let healthy_workload = SwitchWorkload::new(); + let healthy = EngineFixture::with_workload(healthy_workload.clone())?; + admit_incumbent(&healthy)?; + let expected = continuity_decisions(&healthy, &healthy.engine, &healthy_workload)?; + + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + let path = incident_path(&world); + // Three ticks reach the open of an exhausted target, one exchange each. + let actual = run_while_held(hold_lock_of(&path), 3, || { + continuity_decisions(&world, &world.engine, &workload) + })?; + assert_eq!(actual, expected, "a held incident lock changed a continuity decision"); + assert!(!path.exists(), "a contended write landed"); + + // Every refused write is a traced fault naming no path. + { + let reports = world.engine.fault_reports.lock().unwrap(); + let faults = reports + .iter() + .filter(|(key, _)| key.starts_with("incident-store:")) + .map(|(_, report)| report.last.lock().unwrap().clone().unwrap_or_default()) + .collect::>(); + assert!(!faults.is_empty()); + let root = world.root.display().to_string(); + assert!(faults.iter().all(|fault| !fault.contains(&root)), "{faults:?}"); + } + + // The lock is free again: the next pass retries and the incident lands. + // (A child another test forked while it was held can keep it a moment.) + let deadline = std::time::Instant::now() + STUCK_AFTER; + while !incidents_of(&world)?.iter().any(|record| record.is_open()) { + assert!(std::time::Instant::now() < deadline, "the incident never landed"); + world.engine.supervise_one_admitted_generation()?; + std::thread::sleep(Duration::from_millis(20)); + } + assert_control_store_holds_no_incident(&world) + } + + /// A publish that could not land is not done, at the tick: the demotion + /// continuity needs before a restart is one exchange on the topology lock, + /// and another holder of it defers the restart instead of waiting for it. + #[cfg(target_os = "linux")] + #[test] + fn a_held_topology_lock_defers_continuity_and_mints_nothing() -> Result<()> { + use crate::incident::tests::{hold_lock_of, run_while_held}; + let workload = SwitchWorkload::new(); let world = EngineFixture::with_workload(workload.clone())?; let incumbent = admit_incumbent(&world)?; - let redeploy = + project_admitted(&world, &incumbent)?; + let store = world.engine.options.topology_store.clone(); + let projected = projected_under(&world, &incumbent.expected)?; + assert!(projected.contains(&IdunnRuntimeActivationRecord::TYPE.to_owned())); + let before = std::fs::read(&store)?; + workload.kill(); + + // One exchange: the demotion. The window is not spent, so continuity + // would restart if the projection could be demoted. + let moved = run_while_held(hold_lock_of(&store), 1, || { + world.engine.supervise_one_admitted_generation() + })?; + assert!(moved, "the deferral is a write the tick made"); + assert_eq!(std::fs::read(&store)?, before, "a contended publish changed the projection"); + let snapshot = ControlSnapshot::read(&world.state_store)?; + assert!(snapshot.transactions.is_empty(), "continuity minted over a projection it could not demote"); + let recorded = snapshot.supervision_for("service").context("no deferral recorded")?.value.clone(); + assert!(recorded.continuity_restarts.is_empty(), "a deferral spent an attempt"); + let reason = recorded.continuity_deferral_reason.clone().context("no deferral reason")?; + assert!(reason.contains("locked by another holder"), "{reason}"); + assert!(!reason.contains(&world.root.display().to_string()), "{reason}"); + + // The holder is gone and the deferral has run out: the demotion lands + // and the restart proceeds. + let mut ran_out = recorded; + ran_out.continuity_deferred_until = Some(1); + set_meters(&world, &ran_out)?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 1); + assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); + Ok(()) + } + + /// A deployment phase records a publication digest only after the publish + /// returned: the digest is the claim that it landed. + #[cfg(target_os = "linux")] + #[test] + fn a_contended_expected_publish_is_not_recorded_as_published() -> Result<()> { + use crate::incident::tests::{hold_lock_of, run_while_held}; + + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at(&world, DeploymentPhase::Warming)?; + let stored = resident(&world)?; + let mut starting = seeded.clone(); + starting.phase = DeploymentPhase::Starting; + starting.phase_deadline = None; + starting.expected_publication_sha256 = None; + starting.activation = None; + starting.workload = None; + starting.activation_publication_sha256 = None; + starting.warming = None; + replace_transaction(&world.state_store, &stored, &starting)?; + let store = world.engine.options.topology_store.clone(); + let stored = resident(&world)?; + + let error = run_while_held(hold_lock_of(&store), 1, || { + world.engine.advance_transaction(&stored) + }) + .expect_err("a contended publish must not advance the phase"); + assert!(format!("{error:#}").contains("locked by another holder"), "{error:#}"); + assert!(!store.exists(), "a contended publish wrote the projection"); + let after = resident(&world)?.value; + assert_eq!(after.phase, DeploymentPhase::Starting); + assert_eq!(after.expected_publication_sha256, None); + + // Free again: the same step publishes and records what it published. + world.engine.advance_transaction(&resident(&world)?)?; + let expected = seeded.expected.context("seeded transaction has no Expected")?; + assert_eq!( + resident(&world)?.value.expected_publication_sha256, + Some(expected.canonical_sha256()?) + ); + Ok(()) + } + + /// A Continuity transaction parked in Starting with nothing published yet, + /// plus the stored record the tick will see. + #[cfg(target_os = "linux")] + fn parked_in_starting(world: &EngineFixture) -> Result { + let seeded = transaction_at(world, DeploymentPhase::Warming)?; + let stored = resident(world)?; + let mut starting = seeded.clone(); + starting.phase = DeploymentPhase::Starting; + starting.phase_deadline = None; + starting.expected_publication_sha256 = None; + starting.activation = None; + starting.workload = None; + starting.activation_publication_sha256 = None; + starting.warming = None; + replace_transaction(&world.state_store, &stored, &starting)?; + Ok(seeded) + } + + /// A publish that lost to a held lock is transient: before the fence the + /// transaction stays in its phase with the error recorded, no abort is + /// begun, and the next tick after the release lands the publish. A + /// continuity restart is a Continuity transaction on this same path, so + /// its attempt is minted once and a contended publish spends no second one. + #[cfg(target_os = "linux")] + #[test] + fn a_contended_starting_publish_is_retried_on_a_later_tick_not_aborted() -> Result<()> { + use crate::incident::tests::{hold_lock_of, run_while_held}; + + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = parked_in_starting(&world)?; + let mut meters = TargetSupervision::new("service"); + meters.continuity_restarts = vec![now_millis()?]; + set_meters(&world, &meters)?; + let store = world.engine.options.topology_store.clone(); + + for tick in 0..2 { + run_while_held(hold_lock_of(&store), 1, || world.engine.resume_one_transaction())?; + world.engine.resume_backoff.lock().unwrap().clear(); + let held = resident(&world)?.value; + assert_eq!(held.phase, DeploymentPhase::Starting, "tick {tick}"); + assert!(held.pre_fencing_abort.is_none(), "tick {tick}: a held lock aborted the transaction"); + assert!(!held.is_terminal(), "tick {tick}"); + assert_eq!(held.expected_publication_sha256, None, "tick {tick}"); + let error = held.last_error.context("a contended publish left no last_error")?; + assert!(error.contains("locked by another holder"), "{error}"); + assert!(!store.exists(), "a contended publish wrote the projection"); + } + + // Released: the next tick lands the publish and records its digest. + world.engine.resume_one_transaction()?; + let landed = resident(&world)?.value; + assert_eq!(landed.phase, DeploymentPhase::Starting); + assert!(landed.pre_fencing_abort.is_none()); + assert_eq!( + landed.expected_publication_sha256, + Some(seeded.expected.context("seeded transaction has no Expected")?.canonical_sha256()?) + ); + assert_eq!(meters_of(&world, "service")?.continuity_restarts.len(), 1); + Ok(()) + } + + /// Only contention is transient: a publish that fails for any other reason + /// still ends the transaction before the fence, as it always did. + #[cfg(target_os = "linux")] + #[test] + fn a_non_contention_publish_error_still_aborts_before_the_fence() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + parked_in_starting(&world)?; + // A directory where the store should be: the snapshot cannot be read. + std::fs::create_dir_all(&world.engine.options.topology_store)?; + world.engine.resume_one_transaction()?; + let after = resident(&world)?.value; + assert!(after.pre_fencing_abort.is_some(), "a non-contention error did not abort"); + assert!( + !after.last_error.as_deref().unwrap_or_default().contains("locked by another holder"), + "{:?}", + after.last_error + ); + Ok(()) + } + + #[test] + fn status_prints_an_open_incident_under_its_target() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + workload.kill(); + spend_the_window(&world)?; + world.engine.supervise_one_admitted_generation()?; + let opened = incidents_of(&world)?[0].opened_at_unix_millis; + + let mut printed = Vec::new(); + status(&mut printed, &world.state_store, &incident_path(&world), None)?; + let printed = String::from_utf8(printed)?; + let target = printed.find("target service ").context("no target line")?; + let incident = printed + .find(&format!(" incident continuity-exhausted opened-at {opened}")) + .context("status printed no incident")?; + assert!(target < incident, "{printed}"); + Ok(()) + } + + #[test] + fn serve_ticks_until_told_to_stop() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + workload.kill(); + spend_the_window(&world)?; + let mut options = world.engine.options.clone(); + options.poll_millis = 1; + let engine = Engine::open_with_systemd_workload(options, workload.clone())?; + + let mut asked = 0; + serve_while(&engine, || { + asked += 1; + asked <= 3 + })?; + assert_eq!(asked, 4); + assert!(incidents_of(&world)?.iter().any(|record| record.is_open()), "no tick ran"); + Ok(()) + } + + #[test] + fn run_dispatches_serve_and_status() { + let temp = TempDir::new().unwrap(); + let args = |words: &[&str]| words.iter().map(|word| (*word).to_owned()).collect::>(); + // A control store beneath a regular file: serve's boot cannot make it, + // and a serve that returned at once would say Ok instead. + let file = temp.path().join("not-a-directory"); + std::fs::write(&file, b"a file").unwrap(); + let store = file.join("control.cc"); + assert!( + run(args(&["serve", "--state-store", store.to_str().unwrap()]).into_iter()).is_err() + ); + // A control store that is not one: status reads it and refuses. + let junk = temp.path().join("junk.cc"); + std::fs::write(&junk, b"not a cultcache file").unwrap(); + assert!( + run(args(&["status", "--state-store", junk.to_str().unwrap()]).into_iter()).is_err() + ); + } + + /// The incumbent was admitted this long ago, so an anchor written "in the + /// past" is not older than its generation. + fn age_incumbent(world: &EngineFixture, millis: u64) -> Result<()> { + edit_incumbent(world, |generation| generation.admitted_at_unix_millis -= millis)?; + Ok(()) + } + + /// The 2026-10-18 shape at the engine: a dead incumbent with a deployment + /// building over it. Continuity restarts the incumbent beside the build + /// and neither touches the other (handover-phase-split). + #[test] + fn a_dead_incumbent_is_restarted_alongside_a_deployment_that_is_building() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + let deploy = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; workload.kill(); - // The target's restarts are spent: nothing is left to yield to, and - // aborting the deployment would close the last door. + let stored_deploy = |world: &EngineFixture| -> Result> { + ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .find(|stored| stored.value.transaction_id == deploy.transaction_id) + .context("the deployment left the live set") + }; + let before = stored_deploy(&world)?; + + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 1, "no restart beside the building deployment"); + assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 2); + assert_eq!(stored_deploy(&world)?.envelope, before.envelope); + + for _ in 0..2 { + // Due at once again (streak 0), so only the live attempt holds it back. + set_meters(&world, &TargetSupervision::new("service"))?; + let _ = world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 1); + let after = stored_deploy(&world)?; + assert_eq!(after.envelope, before.envelope); + assert!(after.value.pre_fencing_abort.is_none()); + } + Ok(()) + } + + /// A `profile:` command over two targets, one of them crash-looping with a + /// live continuity, freezes at once: freeze refuses only a second live + /// deployment of a target (follow-up profile-deploy-joint-gap). + #[test] + fn a_profile_deploy_freezes_over_a_crash_looping_target() -> Result<()> { + use crate::deployment_plan::tests::BINDING; + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + workload.kill(); + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 1); + let continuity = ControlSnapshot::read(&world.state_store)?.transactions.remove(0); + + let (binding_head, binding_tail) = BINDING.split_once("[route]").context("route")?; + let binding = format!( + "{binding_head}[brakes]{}", + binding_tail.split_once("[brakes]").context("brakes")?.1 + ); + std::fs::create_dir_all(world.root.join("bindings"))?; + for target in ["service", "other"] { + let paired = binding.replacen( + "target = \"service\"\n", + &format!("target = \"{target}\"\nprofiles = [\"pair\"]\n"), + 1, + ); + std::fs::write(world.root.join(format!("bindings/{target}.toml")), paired)?; + } + queue_deploy_of(&world, "up-pair", "profile:pair")?; + assert!(world.engine.freeze_one_queued_command()?, "the profile command stayed queued"); + + let live = ControlSnapshot::read(&world.state_store)?.transactions; + let mut frozen = live + .iter() + .filter(|stored| stored.value.command_id == "up-pair") + .map(|stored| (stored.value.ordinal, stored.value.target.as_str())) + .collect::>(); + frozen.sort(); + assert_eq!(frozen, [(0, "other"), (1, "service")]); + let kept = live + .iter() + .find(|stored| stored.value.transaction_id == continuity.value.transaction_id) + .context("the continuity left the live set")?; + assert_eq!(kept.envelope, continuity.envelope); + + // A live deployment of `service` still refuses a second command for it. + queue_deploy_of(&world, "up-service-second", "service")?; + assert!(!world.engine.freeze_one_queued_command()?); + assert!( + every_transaction(&world)? + .iter() + .all(|transaction| transaction.command_id != "up-service-second") + ); + Ok(()) + } + + /// The live transaction `id`. + fn resident_id(world: &EngineFixture, id: &str) -> Result> { + ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .find(|stored| stored.value.transaction_id == id) + .context("no such live transaction") + } + + /// `stored` one phase on, as the seam would be asked to write it. + fn entering(stored: &Stored, phase: DeploymentPhase) -> Result { + let mut next = stored.value.clone(); + next.enter_phase(phase, now_millis()?); + Ok(next) + } + + /// Drop every live transaction but `keep`. + fn clear_other_live_transactions(world: &EngineFixture, keep: &str) -> Result<()> { + let others = ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .filter(|stored| stored.value.transaction_id != keep) + .map(|stored| stored.envelope) + .collect::>(); + assert!(SingleFileMessagePackBackingStore::new(&world.state_store).delete_batch_if_unchanged(&others)?); + Ok(()) + } + + /// The stored transaction `id`, moved into `phase` in place. + fn rephase(world: &EngineFixture, id: &str, phase: DeploymentPhase) -> Result<()> { + let stored = resident_id(world, id)?; + replace_transaction(&world.state_store, &stored, &entering(&stored, phase)?) + } + + /// A Deploy at `deploy_phase` and a Continuity at `continuity_phase`, both + /// over one incumbent of `service`, and that incumbent. + fn deploy_beside_continuity( + world: &EngineFixture, + deploy_phase: DeploymentPhase, + continuity_phase: DeploymentPhase, + ) -> Result<(DeploymentTransaction, DeploymentTransaction, AdmittedGeneration)> { + let incumbent = admit_incumbent(world)?; + let deploy = + seeded_transaction(world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + let continuity = seeded_transaction( + world, + DeploymentPhase::Warming, + CommandKind::Continuity, + Some(&incumbent), + )?; + for (seeded, phase) in [(&deploy, deploy_phase), (&continuity, continuity_phase)] { + if phase != DeploymentPhase::Warming { + rephase(world, &seeded.transaction_id, phase)?; + } + } + Ok((deploy, continuity, incumbent)) + } + + // BEGIN c2b-forever-wait harness + // The forever-waits of cut c2b, driven through the API that exists at + // 9ea6c71 (the base): each test ends on the waiter's own record, so at the + // base it fails at the waiter assertion (still waiting), never at setup. + // Lines ending in `head-only` read what only the cut adds (the incident + // store, `leaves_to_operator`); the base probe strips them. + + /// One scheduler pass over every live transaction, with no resume backoff + /// standing between it and the step. + fn resume_pass(world: &EngineFixture) -> Result { + world.engine.resume_backoff.lock().unwrap().clear(); + world.engine.resume_one_transaction() + } + + /// The runtime instance whose workload `id` runs. + fn runtime_instance_of(world: &EngineFixture, id: &str) -> Result { + match resident_id(world, id)?.value.workload { + Some(WorkloadObservation::Systemd(observed)) => Ok(observed.runtime_instance_id), + other => bail!("transaction {id} has no systemd workload: {other:?}"), + } + } + + /// The deadline of transaction `id` is already behind it. + fn pass_deadline_of(world: &EngineFixture, id: &str) -> Result<()> { + let current = resident_id(world, id)?; + let mut next = current.value.clone(); + let deadline = next.phase_deadline.as_mut().context("no deadline to pass")?; + deadline.entered_at_unix_millis = 1; + deadline.deadline_at_unix_millis = 2; + replace_transaction(&world.state_store, ¤t, &next) + } + + /// The deadline of `id`, when it has one, is already behind it. A record + /// that has none is left alone: at the base nothing times it, and the + /// scenario must still run to the waiter's assertion. + fn age_deadline_of(world: &EngineFixture, id: &str) -> Result<()> { + if resident_id(world, id)?.value.phase_deadline.is_some() { + pass_deadline_of(world, id)?; + } + Ok(()) + } + + /// The deployment `waiter_id` ended before fencing, with a cause naming + /// the owner it will not wait for. + fn assert_ended_for_owner(world: &EngineFixture, waiter_id: &str, owner_id: &str) -> Result<()> { + let ended = latest_id(world, waiter_id)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the waiting deployment did not end: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("owner-left-to-operator") && error.contains(owner_id), "{error}"); + assert!(ended.fencing.is_none()); + Ok(()) + } + + /// What the scheduler does for a deployment waiting at its Sealing tail: it + /// asks to enter Starting, and a refusal ends it before fencing. (A resume + /// pass would stop first at the deployment brake this world never releases.) + fn step_waiter(world: &EngineFixture, id: &str) -> Result<()> { + let waiting = resident_id(world, id)?; + if waiting.value.pre_fencing_abort.is_some() || waiting.value.phase >= DeploymentPhase::Starting { + return Ok(()); + } + if let Err(error) = world.engine.enter(&waiting, entering(&waiting, DeploymentPhase::Starting)?) { + world.engine.begin_pre_fencing_abort(&resident_id(world, id)?, error)?; + } + Ok(()) + } + + /// Passes of the scheduler over the owner, each followed by the waiter's + /// own attempt to enter Starting. + fn run_owner_and_waiter(world: &EngineFixture, waiter: &str, passes: usize) -> Result<()> { + for _ in 0..passes { + resume_pass(world)?; + if resident_id(world, waiter).is_ok() { + step_waiter(world, waiter)?; + } + } + Ok(()) + } + + fn operator_required(world: &EngineFixture) -> Result> { + world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired) // head-only + } + + /// NEGATIVE (a). A continuity that owns the target is aborting and its + /// abort step fails on every attempt. Its deadline passes: Idunn leaves it + /// to the operator and keeps it owned, and the deployment waiting behind it + /// ends itself before fencing instead of waiting forever. + #[test] + fn a_waiting_deployment_ends_when_its_owners_abort_is_left_to_the_operator() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; + let (deploy_id, owner_id) = (deploy.transaction_id.as_str(), continuity.transaction_id.as_str()); + workload.fail_stop_of(&runtime_instance_of(&world, owner_id)?); + world + .engine + .begin_post_fencing_abort(&resident_id(&world, owner_id)?, anyhow!("candidate refused"))?; + + // Before the deadline the deployment waits on a bounded owner, and no + // one is paged. + assert!(operator_required(&world)?.is_empty()); // head-only + for _ in 0..3 { + resume_pass(&world)?; + } + let waiting = resident_id(&world, deploy_id)?.value; + assert!(waiting.pre_fencing_abort.is_none()); + assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(owner_id))); + assert!(!resident_id(&world, owner_id)?.value.leaves_to_operator(now_millis()?)); // head-only + + age_deadline_of(&world, owner_id)?; + for _ in 0..20 { + resume_pass(&world)?; + } + let owner = resident_id(&world, owner_id)?.value; + assert!(!owner.is_terminal() && owner.post_fencing_abort.is_some(), "the owner was ended"); + assert!(owner.owns_target()); + // Exactly one incident, however many passes ran. + assert_eq!(operator_required(&world)?, [owner_id]); // head-only + assert_ended_for_owner(&world, deploy_id, owner_id) + } + + /// NEGATIVE (b). An admitted transaction drains its incumbent and the stop + /// keeps failing. A deployment waits behind it until the drain's deadline, + /// then ends itself; the drain stays and the target stays owned. + #[test] + fn a_waiting_deployment_ends_when_a_drain_whose_stop_keeps_failing_is_left_to_the_operator( + ) -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + let drainer = seeded_transaction(&world, DeploymentPhase::Fencing, CommandKind::Deploy, Some(&incumbent))?; + odin_reports_ready(&world, &drainer, seeded_sequence(Some(&incumbent)) + 1)?; + let drainer_id = drainer.transaction_id.as_str(); + drive_id(&world, drainer_id, |t| t.phase == DeploymentPhase::Complete)?; + let draining = resident_id(&world, drainer_id)?.value; + assert!(draining.drains_incumbent(), "the incumbent was already retired"); + assert!(draining.phase_deadline.is_some(), "a draining record has no deadline"); // head-only + let Some(PostCommitCleanup { incumbent: IncumbentCleanupEvidence::Pending { workload: old, .. }, .. }) = + &draining.post_commit_cleanup + else { + bail!("no pending incumbent: {:?}", draining.post_commit_cleanup); + }; + let WorkloadObservation::Systemd(old) = old else { bail!("incumbent is not a systemd unit") }; + workload.fail_stop_of(&old.runtime_instance_id); + + let admitted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("nothing admitted")? + .value + .clone(); + let waiter = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&admitted))?; + let waiter_id = waiter.transaction_id.as_str(); + for _ in 0..3 { + resume_pass(&world)?; + } + let waiting = resident_id(&world, waiter_id)?.value; + assert!(waiting.pre_fencing_abort.is_none()); + assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(drainer_id)), "{:?}", waiting.last_error); + assert_eq!(waiting.phase_deadline, None, "the waiter's deadline is armed under a bounded owner"); + assert!(operator_required(&world)?.is_empty()); // head-only + + age_deadline_of(&world, drainer_id)?; + for _ in 0..20 { + resume_pass(&world)?; + } + assert!(!resident_id(&world, drainer_id)?.value.is_terminal(), "the drain was ended"); + assert_eq!(operator_required(&world)?, [drainer_id]); // head-only + assert_ended_for_owner(&world, waiter_id, drainer_id) + } + + /// A stop-then-start transaction held in Warming with a started unit, as + /// the Starting step leaves it. + fn warm_stop_first(world: &EngineFixture, id: &str) -> Result<()> { + let stored = resident_id(world, id)?; + let mut next = stored.value.clone(); + let expected = next.expected.clone().context("no Expected")?; + let now = now_millis()?; + let launch = IdunnRuntimeActivationLaunch::issue( + &expected, + runtime_instance_id(&next.transaction_id)?, + now, + &world.engine.idunn_signer, + )?; + let mut seed = Vec::new(); + let activation = launch.write_credential(&mut seed)?; + let mut workload = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)? + .1 + .workload + .context("no recorded workload")?; + match &mut workload { + WorkloadObservation::Systemd(observed) => { + observed.runtime_instance_id = activation.runtime_instance_id.clone(); + observed.executable_sha256 = expected.artifact_sha256.clone(); + } + WorkloadObservation::Host(_) => bail!("recorded workload is not a systemd unit"), + } + next.expected_publication_sha256 = Some(expected.canonical_sha256()?); + next.activation_publication_sha256 = Some(activation.canonical_sha256()?); + next.activation = Some(activation); + next.workload = Some(workload); + next.enter_phase(DeploymentPhase::Warming, now); + replace_transaction(&world.state_store, &stored, &next) + } + + /// A stop-then-start deployment and continuity over one incumbent, on a + /// systemd unit. The continuity is held in Warming with a started unit and + /// the deployment, one `enter` later, waits at its Sealing tail on it. + fn warm_stop_first_owner( + workload: Arc, + ) -> Result<(EngineFixture, String, String)> { + let (world, deploy, continuity) = stop_first_world(workload.clone())?; + // This world serves no host actuator: the target's host workload runs + // through the same port as everything else here. + *world.engine.host_workload.lock().unwrap() = Some(workload); // seam + warm_stop_first(&world, &continuity)?; + let owner = resident_id(&world, &continuity)?; + assert_eq!(owner.value.phase, DeploymentPhase::Warming); + assert!(owner.value.owns_target()); + let waiting = resident_id(&world, &deploy)?; + world.engine.enter(&waiting, entering(&waiting, DeploymentPhase::Starting)?)?; + let held = resident_id(&world, &deploy)?.value; + assert_eq!(held.phase, DeploymentPhase::Sealing); + assert!(held.last_error.as_deref().is_some_and(|note| note.contains(&continuity))); + Ok((world, deploy, continuity)) + } + + /// NEGATIVE (c). A stop-then-start continuity takes the target at Starting + /// and waits in Warming for an Odin word that never comes: its step runs, + /// observes the unit and gate-waits. Its Warming deadline ends it, and the + /// deployment waiting at its Sealing tail takes the target. + #[test] + fn a_stop_first_owner_that_never_warms_ends_at_its_warming_deadline_and_the_waiter_takes_the_target( + ) -> Result<()> { + let (world, deploy, continuity) = warm_stop_first_owner(SwitchWorkload::new())?; + // The gate-wait is the Warming step itself, not an error of it. + run_owner_and_waiter(&world, &deploy, 3)?; + let waiting = resident_id(&world, &continuity)?.value; + assert_eq!(waiting.phase, DeploymentPhase::Warming); + assert!(waiting.pre_fencing_abort.is_none(), "the Warming step failed: {:?}", waiting.last_error); + assert!(waiting.last_error.is_some(), "the Warming step did not wait for Odin"); + assert!(resident_id(&world, &deploy)?.value.last_error.as_deref().is_some_and(|n| n.contains(&continuity))); + + age_deadline_of(&world, &continuity)?; + for _ in 0..12 { + if resident_id(&world, &continuity).is_err() { + break; + } + run_owner_and_waiter(&world, &deploy, 1)?; + } + let ended = latest_id(&world, &continuity)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the owner did not end: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("Warming phase passed its deadline"), "{error}"); + assert!(operator_required(&world)?.is_empty()); // head-only + + // The pass that retired the owner, or the next, moves the waiter to + // Starting; stop there, before its Starting step runs. + step_waiter(&world, &deploy)?; + let took = resident_id(&world, &deploy)?.value; + assert_eq!(took.phase, DeploymentPhase::Starting); + assert!(took.owns_target() && took.pre_fencing_abort.is_none()); + Ok(()) + } + + /// The wedged pre-fence abort of a stop-then-start owner is paged too: the + /// waiter behind it ends, and one incident opens. + #[test] + fn a_waiting_deployment_ends_when_a_stop_first_owners_prefence_abort_is_left_to_the_operator( + ) -> Result<()> { + let workload = SwitchWorkload::new(); + let (world, deploy, owner) = warm_stop_first_owner(workload.clone())?; + workload.fail_stop_of(&runtime_instance_of(&world, &owner)?); + world + .engine + .begin_pre_fencing_abort(&resident_id(&world, &owner)?, anyhow!("candidate refused"))?; + run_owner_and_waiter(&world, &deploy, 3)?; + // Before the abort's deadline: it waits, unpaged. + let aborting = resident_id(&world, &owner)?.value; + assert!(aborting.pre_fencing_abort.is_some() && !aborting.is_terminal()); + assert!(operator_required(&world)?.is_empty()); // head-only + assert!(resident_id(&world, &deploy)?.value.pre_fencing_abort.is_none()); + + age_deadline_of(&world, &owner)?; + run_owner_and_waiter(&world, &deploy, 20)?; + let aborting = resident_id(&world, &owner)?.value; + assert!(!aborting.is_terminal() && aborting.owns_target(), "the owner was ended"); + assert_eq!(operator_required(&world)?, [owner.as_str()]); // head-only + assert_ended_for_owner(&world, &deploy, &owner) + } + + /// A waiter whose own pre-fence abort is wedged, beside an owner it will + /// not outlast, is paged by its own deadline: the owner's standing does not + /// excuse it. + #[test] + fn a_waiters_wedged_prefence_abort_is_paged_beside_a_standing_owner() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; + let (deploy_id, owner_id) = (deploy.transaction_id.as_str(), continuity.transaction_id.as_str()); + workload.fail_stop_of(&runtime_instance_of(&world, deploy_id)?); + world + .engine + .begin_pre_fencing_abort(&resident_id(&world, deploy_id)?, anyhow!("candidate refused"))?; + for _ in 0..3 { + resume_pass(&world)?; + } + let aborting = resident_id(&world, deploy_id)?.value; + assert!(aborting.pre_fencing_abort.is_some() && !aborting.is_terminal()); + assert!(!operator_required(&world)?.iter().any(|subject| subject == deploy_id)); // head-only + + age_deadline_of(&world, deploy_id)?; + for _ in 0..3 { + resume_pass(&world)?; + } + assert!(!resident_id(&world, deploy_id)?.value.is_terminal()); + assert!(operator_required(&world)?.iter().any(|subject| subject == deploy_id), "{owner_id}"); // head-only + Ok(()) + } + // END c2b-forever-wait harness + + // BEGIN c2d continuity-yields + // A deployment waits at Warming beside a continuity that retries forward + // past its deadline (ruling continuity-yields-to-deploy). The pass order is + // the hostile one: the waiter always looks at the owner first. + + /// One pass over the waiter and then the owner, never clearing the resume + /// backoff. A record that left the live set is skipped. + fn waiter_then_owner(world: &EngineFixture, waiter: &str, owner: &str) -> Result<()> { + for id in [waiter, owner] { + if let Ok(stored) = resident_id(world, id) { + world.engine.resume_candidate(&stored)?; + } + } + Ok(()) + } + + /// A continuity at Fencing driven alone to AwaitingReady, where it + /// gate-waits for an Odin word that never comes, beside a Deploy waiting in + /// Warming. Returns the world, the workload, and the two ids. + fn waiter_beside_gate_waiting_continuity() + -> Result<(EngineFixture, Arc, DeploymentTransaction, String, String, AdmittedGeneration)> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let (deploy, continuity, incumbent) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; + let (deploy_id, owner_id) = (deploy.transaction_id.clone(), continuity.transaction_id.clone()); + drive_id(&world, &owner_id, |t| t.phase == DeploymentPhase::AwaitingReady)?; + Ok((world, workload, deploy, deploy_id, owner_id, incumbent)) + } + + /// NEGATIVE. The continuity's deadline has passed and it retries forward; + /// the deployment waiting on it must not end, the continuity must yield. + #[test] + fn a_waiting_deployment_lands_over_a_continuity_that_retries_forward_past_its_deadline() -> Result<()> { + let (world, workload, deploy, deploy_id, owner_id, incumbent) = waiter_beside_gate_waiting_continuity()?; + pass_deadline_of(&world, &owner_id)?; + let stops = workload.stops.load(std::sync::atomic::Ordering::SeqCst); + waiter_then_owner(&world, &deploy_id, &owner_id)?; + let waiter = latest_id(&world, &deploy_id)?; + assert!(waiter.completion.is_none(), "the waiter ended: {:?}", waiter.completion); + assert!(waiter.pre_fencing_abort.is_none()); + let owner = resident_id(&world, &owner_id)?.value; + let abort = owner.post_fencing_abort.as_ref().context("the continuity did not yield")?; + assert!(abort.error.starts_with("continuity-yields") && abort.error.contains(&deploy_id), "{}", abort.error); + // Nothing stopped the candidate before the yield began. + assert_eq!(workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); + + for _ in 0..8 { + waiter_then_owner(&world, &deploy_id, &owner_id)?; + world.engine.resume_backoff.lock().unwrap().clear(); + } + // The yield's abort retimed the record: its incident, if one opened, closed. + assert!(!operator_required(&world)?.contains(&owner_id)); + let ended = latest_id(&world, &owner_id)?; + assert!(matches!(ended.completion, Some(TransactionCompletion::FailedAfterFencing { .. })), "{:?}", ended.completion); + assert_eq!(workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops + 1); + + odin_reports_ready(&world, &deploy, seeded_sequence(Some(&incumbent)) + 10)?; + drive_id(&world, &deploy_id, |t| t.is_terminal())?; + let landed = latest_id(&world, &deploy_id)?; + assert!(matches!(landed.completion, Some(TransactionCompletion::Admitted { .. })), "{:?}", landed.completion); + let admitted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("nothing admitted")? + .value + .clone(); + assert_eq!(admitted.transaction_id, deploy_id); + assert_eq!(landed.fencing, Some(FencingEvidence::SkippedStateless)); + Ok(()) + } + + fn stops_of(workload: &SwitchWorkload) -> usize { + workload.stops.load(std::sync::atomic::Ordering::SeqCst) + } + + /// A Deploy at its Sealing tail, braked, beside a continuity: it awaits the + /// handover but cannot take it (the brake has not released it). + fn braked_sealing_deploy(world: &EngineFixture, deploy_id: &str) -> Result<()> { + make_stop_first(world, deploy_id)?; + let sealing = resident_id(world, deploy_id)?; + let mut tail = sealing.value.clone(); + tail.deployment_authorization = None; + replace_transaction(&world.state_store, &sealing, &tail)?; + let tail = resident_id(world, deploy_id)?.value; + assert!(tail.awaits_handover()); + assert!(!world.engine.deploy_can_move(&resident_id(world, deploy_id)?, now_millis()?)); + Ok(()) + } + + /// NEGATIVE. The boot lift yields to a Deploy only by the continuity's own + /// yield test: a braked Deploy awaiting the handover is no reason to keep a + /// pre-c2c abort running over the only copy of the body. + #[test] + fn the_boot_lift_returns_a_running_candidate_forward_beside_a_braked_waiting_deploy() -> Result<()> { + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + braked_sealing_deploy(&world, &deploy_id)?; + let owner = resident_id(&world, &owner_id)?; + world.engine.begin_post_fencing_abort( + &owner, + anyhow!("the AwaitingReady phase passed its deadline at unix ms 2"), + )?; + let returned = world.engine.return_wedged_continuity_aborts_forward()?; + assert_eq!(returned, vec![owner_id.clone()]); + for _ in 0..5 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + } + assert_eq!(stops_of(&workload), 0, "the running candidate was stopped"); + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + Ok(()) + } + + /// NEGATIVE. A boot lift that errs decided nothing: no continuity abort + /// runs a step until a later lift succeeds, and then the running candidate + /// is returned forward instead of stopped. + #[test] + fn a_boot_lift_that_errs_holds_every_continuity_abort_until_a_lift_succeeds() -> Result<()> { + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + braked_sealing_deploy(&world, &deploy_id)?; + let owner = resident_id(&world, &owner_id)?; + world.engine.begin_post_fencing_abort( + &owner, + anyhow!("the AwaitingReady phase passed its deadline at unix ms 2"), + )?; + // The lift's only fallible read fails: the store is unreadable at boot. + let intact = std::fs::read(&world.state_store)?; + std::fs::write(&world.state_store, b"not a control store")?; + world.engine.boot_lift(); + assert!(world.engine.lift_unsettled.load(std::sync::atomic::Ordering::SeqCst)); + std::fs::write(&world.state_store, &intact)?; + let stored = resident_id(&world, &owner_id)?; + assert!(stored.value.post_fencing_abort.is_some()); + // Still failing: the abort is held, no step runs. + std::fs::write(&world.state_store, b"not a control store")?; + assert!(!world.engine.resume_candidate(&stored)?, "a held abort moved"); + std::fs::write(&world.state_store, &intact)?; + assert_eq!(stops_of(&workload), 0); + // A later lift succeeds: the running candidate is returned forward. + for _ in 0..5 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + } + assert_eq!(stops_of(&workload), 0, "the running candidate was stopped"); + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + assert!(!world.engine.lift_unsettled.load(std::sync::atomic::Ordering::SeqCst)); + Ok(()) + } + + /// NEGATIVE. A held record (its candidate unobservable, not provably dead) + /// is not released to its abort because a braked Deploy starts awaiting the + /// handover. + #[test] + fn a_held_continuity_abort_is_not_released_by_a_braked_waiting_deploy() -> Result<()> { + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + rephase(&world, &deploy_id, DeploymentPhase::Sealing)?; + let owner = resident_id(&world, &owner_id)?; + world.engine.begin_post_fencing_abort( + &owner, + anyhow!("the AwaitingReady phase passed its deadline at unix ms 2"), + )?; + workload.silence(true); + let returned = world.engine.return_wedged_continuity_aborts_forward()?; + assert!(returned.is_empty()); + assert!(world.engine.lift_held.lock().unwrap().contains(&owner_id)); + for _ in 0..3 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + } + braked_sealing_deploy(&world, &deploy_id)?; + for _ in 0..5 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + } + assert_eq!(stops_of(&workload), 0, "the unobservable candidate was stopped"); + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_some()); + assert!(world.engine.lift_held.lock().unwrap().contains(&owner_id)); + Ok(()) + } + + /// The yield is a function of its inputs: the deadline, the continuity's + /// own record, and a deployment that awaits the handover. + #[test] + fn a_continuity_that_retries_forward_yields_only_past_its_deadline_and_only_to_a_deploy_awaiting_the_handover( + ) -> Result<()> { + // (i) Before the deadline nothing yields and the Deploy waits. + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + let stops = stops_of(&workload); + for _ in 0..5 { + waiter_then_owner(&world, &deploy_id, &owner_id)?; + let owner = latest_id(&world, &owner_id)?; + assert!(owner.completion.is_none(), "(i) {:?} {:?} {:?}", owner.completion, owner.phase, owner.last_error); + } + assert!(resident_id(&world, &owner_id).with_context(|| "(i)")?.value.post_fencing_abort.is_none()); + let waiting = resident_id(&world, &deploy_id)?.value; + assert!(waiting.pre_fencing_abort.is_none() && waiting.completion.is_none()); + assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(&owner_id))); + assert_eq!(stops_of(&workload), stops); + + // (ii) Past the deadline, a Deploy still building awaits nothing. + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + let stops = stops_of(&workload); + rephase(&world, &deploy_id, DeploymentPhase::Sealing)?; + pass_deadline_of(&world, &owner_id)?; + for _ in 0..4 { + waiter_then_owner(&world, "", &owner_id)?; + } + assert!(resident_id(&world, &owner_id).with_context(|| "(ii)")?.value.post_fencing_abort.is_none()); + assert_eq!(operator_required(&world)?, [owner_id.as_str()]); + assert_eq!(stops_of(&workload), stops); + + // (iii) A Deploy already ending before the fence awaits nothing. + let (world, _, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + world.engine.begin_pre_fencing_abort(&resident_id(&world, &deploy_id)?, anyhow!("refused"))?; + pass_deadline_of(&world, &owner_id)?; + waiter_then_owner(&world, "", &owner_id)?; + assert!(resident_id(&world, &owner_id).with_context(|| "(iii)")?.value.post_fencing_abort.is_none()); + + // (iv) Nor does a Deploy that has ended. + let (world, _, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + world.engine.begin_pre_fencing_abort(&resident_id(&world, &deploy_id)?, anyhow!("refused"))?; + for _ in 0..12 { + if latest_id(&world, &deploy_id)?.completion.is_some() { + break; + } + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, &deploy_id, "")?; + } + assert!(latest_id(&world, &deploy_id)?.completion.is_some()); + pass_deadline_of(&world, &owner_id)?; + waiter_then_owner(&world, "", &owner_id)?; + assert!(resident_id(&world, &owner_id).with_context(|| "(iv)")?.value.post_fencing_abort.is_none()); + Ok(()) + } + + /// `awaits_handover` and `yields_to_a_deploy` row by row. + #[test] + fn awaits_handover_and_yields_to_a_deploy_are_exact_functions_of_the_record() -> Result<()> { + let (world, _, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + let waiting = resident_id(&world, &deploy_id)?.value; + assert_eq!(waiting.phase, DeploymentPhase::Warming); + assert!(waiting.awaits_handover()); + for phase in [ + DeploymentPhase::Sealing, + DeploymentPhase::Starting, + DeploymentPhase::Fencing, + DeploymentPhase::Complete, + ] { + let mut moved = waiting.clone(); + moved.phase = phase; + assert!(!moved.awaits_handover(), "{phase:?}"); + } + let mut ended = waiting.clone(); + ended.completion = Some(TransactionCompletion::FailedBeforeFencing { error: "x".into() }); + assert!(!ended.awaits_handover()); + let mut continuity = resident_id(&world, &owner_id)?.value; + let mut as_continuity = waiting.clone(); + as_continuity.command_kind = CommandKind::Continuity; + assert!(!as_continuity.awaits_handover()); + world.engine.begin_pre_fencing_abort(&resident_id(&world, &deploy_id)?, anyhow!("x"))?; + assert!(!resident_id(&world, &deploy_id)?.value.awaits_handover()); + + // The stop-then-start handover is Starting: its Sealing tail awaits it. + let (stop_first, stop_deploy, _) = stop_first_world(Arc::new(StillWorkload))?; + let mut sealing = resident_id(&stop_first, &stop_deploy)?.value; + sealing.phase = DeploymentPhase::Sealing; + assert!(sealing.awaits_handover()); + sealing.phase = DeploymentPhase::Warming; + assert!(!sealing.awaits_handover()); + + // A continuity yields from Fencing until its route proof succeeds, and + // until aborting; a proven one keeps serving. + for phase in [ + DeploymentPhase::Fencing, + DeploymentPhase::Leasing, + DeploymentPhase::AwaitingReady, + DeploymentPhase::Routing, + ] { + continuity.phase = phase; + assert!(continuity.yields_to_a_deploy(), "{phase:?}"); + assert!(!continuity.serving_proven(), "{phase:?}"); + } + continuity.phase = DeploymentPhase::Committing; + assert!(continuity.serving_proven() && !continuity.yields_to_a_deploy()); + continuity.phase = DeploymentPhase::Routing; + continuity.routing = Some(RoutingEvidence::SkippedUnrouted); + assert!(continuity.serving_proven() && !continuity.yields_to_a_deploy()); + continuity.routing = None; + let mut deploy_record = waiting.clone(); + deploy_record.phase = DeploymentPhase::Fencing; + assert!(!deploy_record.yields_to_a_deploy()); + continuity.post_fencing_abort = Some(post_fencing_abort_intent(&continuity, "x")); + assert!(!continuity.yields_to_a_deploy()); + Ok(()) + } + + /// NEGATIVE. A continuity past its route proof is serving. Past its deadline + /// it keeps retrying and pages; a waiting Deploy does not stop it. + #[test] + fn a_committing_continuity_past_its_deadline_keeps_serving_and_pages() -> Result<()> { + let (world, workload, _, _, owner_id, incumbent) = waiter_beside_gate_waiting_continuity()?; + let owner = resident_id(&world, &owner_id)?.value; + // The continuity's incarnation already has a correlation (the + // incumbent's, same release): Odin re-stamps it Ready for this activation. + let expected = owner.expected.as_ref().context("no Expected")?; + let backing = SingleFileMessagePackBackingStore::new(&world.engine.options.odin_correlation_store); + let key = crate::drivers::incarnation_key_of(&expected.target, &expected.canonical_sha256()?); + let current = backing + .pull_all_read_only_snapshot()? + .into_iter() + .find(|entry| entry.key == key); + assert!(backing.compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + key: key.clone(), + current, + }], + &[CultCacheEnvelope { + key, + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + payload: signed_correlation(&world, &owner, seeded_sequence(Some(&incumbent)) + 10, true)?, + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some(cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into()), + }], + )?); + let phases = drive_id(&world, &owner_id, |t| t.phase == DeploymentPhase::Committing)?; + let at = latest_id(&world, &owner_id)?; + assert_eq!(at.phase, DeploymentPhase::Committing, "{phases:?} {:?} {:?}", at.completion, at.last_error); + // Its Committing step now fails on every attempt: the body is gone. + workload.kill(); + pass_deadline_of(&world, &owner_id)?; + let stops = stops_of(&workload); + for _ in 0..4 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + } + let owner = latest_id(&world, &owner_id)?; + assert!(owner.post_fencing_abort.is_none(), "a serving continuity yielded"); + assert_eq!(owner.phase, DeploymentPhase::Committing); + assert_eq!(operator_required(&world)?, [owner_id.as_str()]); + assert_eq!(stops_of(&workload), stops); + Ok(()) + } + + /// NEGATIVE. A Deploy held at its Sealing tail by the deployment brake + /// cannot take the handover, so a continuity does not stop its candidate + /// for it; once the brake releases, the continuity yields. The yield closes + /// the continuity's operator-required incident Recovered. + #[test] + fn a_continuity_does_not_yield_to_a_braked_deploy_and_yields_once_the_brake_releases() -> Result<()> { + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + make_stop_first(&world, &deploy_id).context("A make_stop_first")?; + let sealing = resident_id(&world, &deploy_id).context("B sealing")?; + let mut tail = sealing.value.clone(); + tail.deployment_authorization = None; + replace_transaction(&world.state_store, &sealing, &tail)?; + assert!(resident_id(&world, &deploy_id).context("C awaits")?.value.awaits_handover()); + pass_deadline_of(&world, &owner_id)?; + let stops = stops_of(&workload); + for _ in 0..4 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + } + let held = latest_id(&world, &owner_id)?; + assert!(held.post_fencing_abort.is_none(), "the continuity yielded to a braked Deploy"); + assert_eq!(operator_required(&world)?, [owner_id.as_str()]); + assert_eq!(stops_of(&workload), stops); + + // The operator releases the brake for this exact Deploy. + let deploy = resident_id(&world, &deploy_id).context("E deploy")?.value; + let authorization = deployment_authorization( + &world, + deploy.expected.as_ref().context("no Expected")?, + &deploy_id, + now_millis()?, + )?; + let store = deploy.plan.as_ref().context("no plan")?.parsed_inputs()?.1.brakes.deployment_store; + let record: IdunnDeploymentBrakeRecord = rmp_serde::from_slice(&authorization.canonical_brake_bytes)?; + assert!(SingleFileMessagePackBackingStore::new(&store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: IdunnDeploymentBrakeRecord::TYPE.into(), + key: record.brake_id.clone(), + current: None, + }], + &[CultCacheEnvelope { + key: record.brake_id.clone(), + r#type: IdunnDeploymentBrakeRecord::TYPE.into(), + payload: authorization.canonical_brake_bytes.clone(), + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some(IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into()), + }], + )?); + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + let yielded = resident_id(&world, &owner_id).context("F yielded")?.value; + let abort = yielded.post_fencing_abort.as_ref().context("the continuity did not yield")?; + assert!(abort.error.starts_with("continuity-yields") && abort.error.contains(&deploy_id), "{}", abort.error); + // The yield retimed the record: the next pass closes its incident + // Recovered, and it stays closed. + for _ in 0..3 { + world.engine.resume_backoff.lock().unwrap().clear(); + world.engine.resume_one_transaction()?; + } + assert!(operator_required(&world)?.is_empty()); + let records = world.engine.incidents().read()?; + assert_eq!(records.len(), 1); + assert_eq!(records[0].subject, owner_id); + assert_eq!(records[0].close_reason, Some(CloseReason::Recovered)); + Ok(()) + } + + /// The c2b waiter still ends on an owner that will not yield: the yield's + /// own abort wedged past its deadline. + #[test] + fn the_c2b_waiter_still_ends_on_an_owner_that_will_not_yield() -> Result<()> { + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + workload.fail_stop_of(&runtime_instance_of(&world, &owner_id)?); + pass_deadline_of(&world, &owner_id)?; + waiter_then_owner(&world, &deploy_id, &owner_id)?; + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_some()); + pass_deadline_of(&world, &owner_id)?; + for _ in 0..20 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, &deploy_id, &owner_id)?; + } + assert!(!resident_id(&world, &owner_id)?.value.is_terminal()); + assert_ended_for_owner(&world, &deploy_id, &owner_id) + } + + /// The yield's abort is not held behind the backoff the forward retry + /// armed: movement spends it. + #[test] + fn a_yield_abort_is_not_held_behind_the_forward_retry_backoff() -> Result<()> { + let (world, _, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + // The Deploy is still building, so the pass past the deadline retries + // forward and arms the continuity backoff. + rephase(&world, &deploy_id, DeploymentPhase::Sealing)?; + pass_deadline_of(&world, &owner_id)?; + waiter_then_owner(&world, "", &owner_id)?; + let armed = *world + .engine + .resume_backoff + .lock() + .unwrap() + .get(&owner_id) + .context("the unmoved step set no backoff")?; + assert_eq!(armed.ceiling_millis, CONTINUITY_BACKOFF_CEILING_MILLIS); + // It reaches the handover: the next pass yields, and the abort runs + // with no clearing and no fake time. + rephase(&world, &deploy_id, DeploymentPhase::Warming)?; + waiter_then_owner(&world, "", &owner_id)?; + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_some()); + for _ in 0..3 { + waiter_then_owner(&world, "", &owner_id)?; + } + let done = latest_id(&world, &owner_id)?; + let cleaned = done.post_fencing_abort.as_ref().map(|abort| abort.candidate_cleanup); + assert_eq!(cleaned, Some(CleanupEvidence::Complete), "{:?}", done.last_error); + Ok(()) + } + // END c2d continuity-yields + + #[test] + fn an_operator_required_incident_closes_when_its_transaction_ends() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; + let id = seeded.transaction_id.as_str(); + workload.fail_stop_of(&runtime_instance_of(&world, id)?); + world.engine.begin_post_fencing_abort(&resident_id(&world, id)?, anyhow!("candidate refused"))?; + pass_deadline_of(&world, id)?; + let open = || world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired); + for _ in 0..4 { + resume_pass(&world)?; + } + assert_eq!(open()?, [id]); + let records = world.engine.incidents().read()?; + assert_eq!(records.len(), 1); + + // An incident whose transaction is gone closes even while another is + // still left to the operator. + world.engine.incidents().open(IncidentCondition::OperatorRequired, "tx-vanished", now_millis()?)?; + resume_pass(&world)?; + assert_eq!(open()?, [id]); + let records = world.engine.incidents().read()?; + + // Still left to the operator: still open, and not written again. + resume_pass(&world)?; + assert_eq!(world.engine.incidents().read()?, records); + + // The step succeeds and the transaction ends; the next pass closes it. + workload.mend_stops(); + for _ in 0..12 { + if resident_id(&world, id).is_err() { + break; + } + resume_pass(&world)?; + } + assert!(latest_id(&world, id)?.is_terminal()); + resume_pass(&world)?; + assert!(open()?.is_empty()); + let closed = world.engine.incidents().read()?; + assert!(closed.iter().all(|record| record.close_reason == Some(CloseReason::Recovered))); + assert_eq!(closed.iter().filter(|record| record.subject == id).count(), 1); + resume_pass(&world)?; + assert_eq!(world.engine.incidents().read()?, closed, "a spent incident was written again"); + Ok(()) + } + + #[test] + fn a_waiters_warming_deadline_is_disarmed_while_an_owner_holds_the_target_and_rearmed_when_it_ends( + ) -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; + let (deploy_id, owner_id) = (deploy.transaction_id.as_str(), continuity.transaction_id.as_str()); + rephase(&world, deploy_id, DeploymentPhase::Warming)?; + assert!(resident_id(&world, deploy_id)?.value.phase_deadline.is_some()); + + resume_pass(&world)?; + let waiting = resident_id(&world, deploy_id)?.value; + assert_eq!(waiting.phase_deadline, None, "the owner does not bound the waiter"); + assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(owner_id))); + // Held for as long as the owner stands: a deadline that fell due + // meanwhile would otherwise abort it. + for _ in 0..3 { + resume_pass(&world)?; + } + assert!(resident_id(&world, deploy_id)?.value.pre_fencing_abort.is_none()); + + // The owner retires without moving the generation. The waiter's own + // entry is long past, but it is timed from now. + clear_other_live_transactions(&world, deploy_id)?; + let stored = resident_id(&world, deploy_id)?; + let mut aged = stored.value.clone(); + aged.created_at_unix_millis = 1; + aged.updated_at_unix_millis = 1; + replace_transaction(&world.state_store, &stored, &aged)?; + let retired = now_millis()?; + resume_pass(&world)?; + let rearmed = resident_id(&world, deploy_id)?.value; + let deadline = rearmed.phase_deadline.context("the waiter was not re-armed")?; + assert_eq!(deadline.phase, DeploymentPhase::Warming); + assert!(deadline.entered_at_unix_millis >= retired); + assert!(rearmed.pre_fencing_abort.is_none()); + Ok(()) + } + + #[test] + fn exactly_one_transaction_takes_a_target_from_one_view() -> Result<()> { + let take = |snapshot: &ControlSnapshot, world: &EngineFixture, id: &str| -> Result { + let stored = resident_id(world, id)?; + world.engine.take_target(snapshot, &stored, &entering(&stored, DeploymentPhase::Fencing)?) + }; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + let view = ControlSnapshot::read(&world.state_store)?; + + assert_eq!(take(&view, &world, &continuity.transaction_id)?, Handover::Took); + // The second claimant read the same world: its CAS loses. + assert_eq!(take(&view, &world, &deploy.transaction_id)?, Handover::Lost); + let after = ControlSnapshot::read(&world.state_store)?; + assert_eq!(after.transactions.iter().filter(|stored| stored.value.owns_target()).count(), 1); + assert_eq!(resident_id(&world, &deploy.transaction_id)?.value.phase, DeploymentPhase::Warming); + // A fresh view names the owner instead of racing it. + assert_eq!( + take(&after, &world, &deploy.transaction_id)?, + Handover::OwnedBy(continuity.transaction_id.clone()) + ); + + // The reverse world: the deployment wins and the continuity loses. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + let view = ControlSnapshot::read(&world.state_store)?; + assert_eq!(take(&view, &world, &deploy.transaction_id)?, Handover::Took); + assert_eq!(take(&view, &world, &continuity.transaction_id)?, Handover::Lost); + Ok(()) + } + + #[test] + fn the_handover_loses_when_the_admitted_generation_moved_after_it_was_read() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, _, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + // Only the continuity is a rival here; take it out of the guards' way. + clear_other_live_transactions(&world, &deploy.transaction_id)?; + let view = ControlSnapshot::read(&world.state_store)?; + let stored = resident_id(&world, &deploy.transaction_id)?; + + // Same generation, new envelope: a refresh landed after the read. + edit_incumbent(&world, |generation| generation.admitted_at_unix_millis += 1)?; + assert_eq!( + world.engine.take_target(&view, &stored, &entering(&stored, DeploymentPhase::Fencing)?)?, + Handover::Lost + ); + // A view that already shows another generation is Moved, not raced. + let mut other = stored.clone(); + other.value.incumbent_generation_id = Some("generation-elsewhere".into()); + assert_eq!( + world.engine.take_target( + &ControlSnapshot::read(&world.state_store)?, + &other, + &entering(&other, DeploymentPhase::Fencing)? + )?, + Handover::Moved + ); + Ok(()) + } + + #[test] + fn the_seam_makes_a_deployment_wait_and_a_continuity_step_aside() -> Result<()> { + // A deployment ready to fence while a continuity owns the target waits + // where it is, with the owner named and no abort. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; + let stored = resident_id(&world, &deploy.transaction_id)?; + world.engine.enter(&stored, entering(&stored, DeploymentPhase::Fencing)?)?; + let waiting = resident_id(&world, &deploy.transaction_id)?.value; + assert_eq!(waiting.phase, DeploymentPhase::Warming); + assert!(waiting.pre_fencing_abort.is_none()); + assert!( + waiting.last_error.as_deref().is_some_and(|note| note.contains(&continuity.transaction_id)), + "{:?}", + waiting.last_error + ); + // A deployment whose generation moved is left for its rebase. + let world_two = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy_two, _, _) = + deploy_beside_continuity(&world_two, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + clear_other_live_transactions(&world_two, &deploy_two.transaction_id)?; + let mut stored_two = resident_id(&world_two, &deploy_two.transaction_id)?; + stored_two.value.incumbent_generation_id = Some("generation-elsewhere".into()); + let before = ControlSnapshot::read(&world_two.state_store)?.transactions[0].envelope.clone(); + world_two.engine.enter(&stored_two, entering(&stored_two, DeploymentPhase::Fencing)?)?; + assert_eq!(ControlSnapshot::read(&world_two.state_store)?.transactions[0].envelope, before); + + // A continuity in the same position is superseded, by either cause. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Fencing, DeploymentPhase::Warming)?; + let stored = resident_id(&world, &continuity.transaction_id)?; + let error = world + .engine + .enter(&stored, entering(&stored, DeploymentPhase::Fencing)?) + .expect_err("a continuity entered a target another transaction owns"); + assert!(format!("{error:#}").contains("continuity superseded"), "{error:#}"); + assert!(format!("{error:#}").contains(&deploy.transaction_id), "{error:#}"); + clear_other_live_transactions(&world, &continuity.transaction_id)?; + let mut stored = resident_id(&world, &continuity.transaction_id)?; + stored.value.incumbent_generation_id = Some("generation-elsewhere".into()); + let error = world + .engine + .enter(&stored, entering(&stored, DeploymentPhase::Fencing)?) + .expect_err("a continuity entered over a generation that moved"); + assert!(format!("{error:#}").contains("continuity superseded"), "{error:#}"); + // Phases are entered one at a time. + let stored = resident_id(&world, &continuity.transaction_id)?; + assert!(world.engine.enter(&stored, entering(&stored, DeploymentPhase::Routing)?).is_err()); + Ok(()) + } + + #[test] + fn the_handover_phase_is_starting_only_for_stop_then_start() -> Result<()> { + assert_eq!(handover_phase(true), DeploymentPhase::Starting); + assert_eq!(handover_phase(false), DeploymentPhase::Fencing); + + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let mut seeded = transaction_at(&world, DeploymentPhase::Warming)?; + assert_eq!(seeded.takes_target_at(), DeploymentPhase::Fencing); + // The target is owned from the handover to Complete, and not after; + // it is live until Complete. + for (phase, owns) in [ + (DeploymentPhase::Sealing, false), + (DeploymentPhase::Starting, false), + (DeploymentPhase::Warming, false), + (DeploymentPhase::Fencing, true), + (DeploymentPhase::Leasing, true), + (DeploymentPhase::AwaitingReady, true), + (DeploymentPhase::Routing, true), + (DeploymentPhase::Committing, true), + (DeploymentPhase::Complete, false), + ] { + seeded.phase = phase; + assert_eq!(seeded.owns_target(), owns, "{phase:?}"); + assert_eq!(seeded.is_live(), phase != DeploymentPhase::Complete, "{phase:?}"); + } + Ok(()) + } + + /// A route-less host-actuator target that rolls out stop-then-start: the + /// seeded systemd binding with its runner, workload and rollout swapped. + fn stop_first_binding(world: &EngineFixture) -> Result { + use crate::deployment_plan::tests::BINDING; + const HOST: &str = r#"[runners.rust] +driver = "host-native" +affordances = ["source-read", "artifact-write"] +allowed_programs = ["cargo"] + +[workload] +driver = "host-actuator" +host = "yggdrasil" +host_trust_anchor_store = "/etc/gamecult/trust/host-actuator.cc" +release_root = "/srv/service/releases" +runtime_root = "/run/gamecult/idunn/service" +state_root = "/var/lib/gamecult/service" + +[workload.argument_bindings] +state_root = "/var/lib/gamecult/service" + +[workload.secret_files] +GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-identity.cc" + +"#; + let (head, rest) = BINDING.split_once("[runners.rust]").context("no runner")?; + let (_, rest) = rest.split_once("[runtime_identity]").context("no identity")?; + let (identity, rest) = rest.split_once("[route]").context("no route")?; + let (_, brakes) = rest.split_once("[brakes]").context("no brakes")?; + bind_provider( + world, + &format!("{head}{HOST}[runtime_identity]{identity}[brakes]{brakes}") + .replace("candidate-then-promote", "stop-then-start") + .replace( + "/var/lib/gamecult/idunn/service-deployment-brake.cc", + &world.root.join("service-deployment-brake.cc").display().to_string(), + ), + ) + } + + /// The seeded transaction `id`, rewritten as the Sealing transaction of a + /// stop-then-start target: the host-actuator plan, release and Expected, + /// and none of the evidence that follows Sealing. + fn make_stop_first(world: &EngineFixture, id: &str) -> Result<()> { + use crate::deployment_plan::tests::{ + RECIPE, artifact_receipt, source, + }; + let stored = resident_id(world, id)?; + let mut next = stored.value.clone(); + let binding = stop_first_binding(world)?; + let recipe = RECIPE + .split("[[dependencies]]") + .next() + .context("recipe is empty")? + .replace("route_required = true", "route_required = false") + .replace( + r#"["GAMECULT_IDUNN_CANDIDATE_BIND", "GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, + r#"["GAMECULT_IDUNN_ACTIVATION_CREDENTIAL", "GAMECULT_IDUNN_RUNTIME_BUNDLE", "GAMECULT_RUNTIME_PRESENCE_IDENTITY"]"#, + ); + // A host-native runner has no network profile to fetch an input with. + let (before_inputs, inputs) = recipe + .split_once("[[external_inputs]]") + .context("recipe has no external inputs")?; + let recipe = format!( + "{before_inputs}[[artifacts]]{}", + inputs.split_once("[[artifacts]]").context("recipe has no artifacts")?.1 + ); + let plan = compile_deployment_plan( + recipe.as_bytes(), + binding.as_bytes(), + source(&recipe), + &format!("service-{id}"), + None, + 110, + &[], + )?; + let release = SealedRelease::new( + &plan, + vec![artifact_receipt()], + Vec::new(), + 120, + )?; + let mut expected = release.expected_projection(&plan)?; + declare_odin(&mut expected); + let now = now_millis()?; + if next.command_kind == CommandKind::Deploy { + next.frozen_source = Some(FrozenSourceReceipt { + transaction_id: id.into(), + plan_id: plan.plan_id.clone(), + snapshot_sha256: sha256_id(b"frozen source snapshot"), + }); + next.deployment_authorization = + Some(deployment_authorization(world, &expected, id, now)?); + } + next.installed_release = Some(crate::drivers::InstalledReleaseObservation { + sealed_release_id: release.sealed_release_id.clone(), + root: PathBuf::from("/srv/service/releases/test"), + }); + next.sealed_release = Some(release); + next.expected = Some(expected); + next.plan = Some(plan); + next.expected_publication_sha256 = None; + next.activation_publication_sha256 = None; + next.activation = None; + next.workload = None; + next.warming = None; + next.isolation = None; + next.enter_phase(DeploymentPhase::Sealing, now); + replace_transaction(&world.state_store, &stored, &next) + } + + /// An admitted incumbent of `service`, and beside it a Deploy and a + /// Continuity (their ids, in that order), both stop-then-start at Sealing. + fn stop_first_world( + workload: Arc, + ) -> Result<(EngineFixture, String, String)> { + let world = EngineFixture::with_workload(workload)?; + let incumbent = admit_incumbent(&world)?; + let mut ids = Vec::new(); + for kind in [CommandKind::Deploy, CommandKind::Continuity] { + let seeded = + seeded_transaction(&world, DeploymentPhase::Warming, kind, Some(&incumbent))?; + make_stop_first(&world, &seeded.transaction_id)?; + ids.push(seeded.transaction_id); + } + let continuity = ids.pop().context("no continuity")?; + let deploy = ids.pop().context("no deploy")?; + Ok((world, deploy, continuity)) + } + + /// Stop-then-start takes the target at Starting, by the same CAS as + /// Fencing: of two such transactions at Sealing, whichever enters Starting + /// first owns the target, and the other neither enters nor aborts. + #[test] + fn two_stop_then_start_transactions_never_both_enter_starting() -> Result<()> { + use DeploymentPhase::{Sealing, Starting}; + for deploy_first in [true, false] { + let (world, deploy, continuity) = stop_first_world(Arc::new(StillWorkload))?; + let (first, second) = if deploy_first { + (deploy.clone(), continuity.clone()) + } else { + (continuity.clone(), deploy.clone()) + }; + let stored = resident_id(&world, &first)?; + assert_eq!(stored.value.takes_target_at(), Starting); + assert!(!stored.value.owns_target()); + world.engine.enter(&stored, entering(&stored, Starting)?)?; + let owner = resident_id(&world, &first)?.value; + assert_eq!(owner.phase, Starting); + assert!(owner.owns_target(), "a stop-first transaction at Starting does not own the target"); + + let rival = resident_id(&world, &second)?; + let outcome = world.engine.enter(&rival, entering(&rival, Starting)?); + if deploy_first { + let error = outcome.expect_err("a continuity entered Starting over an owner"); + assert!(format!("{error:#}").contains("continuity superseded"), "{error:#}"); + assert!(format!("{error:#}").contains(&deploy), "{error:#}"); + } else { + outcome?; + let waiting = resident_id(&world, &second)?.value; + assert!( + waiting.last_error.as_deref().is_some_and(|note| note.contains(&continuity)), + "{:?}", + waiting.last_error + ); + } + // The store still reads: one owner, and the rival held at Sealing. + let after = ControlSnapshot::read(&world.state_store)?; + assert_eq!(after.transactions.iter().filter(|stored| stored.value.owns_target()).count(), 1); + let held = resident_id(&world, &second)?.value; + assert_eq!(held.phase, Sealing); + assert!(held.pre_fencing_abort.is_none()); + } + Ok(()) + } + + /// A stop-first deployment that has entered Starting has stopped the + /// incumbent by design; supervision must read it as the owner and not + /// restart what it stopped. + #[test] + fn supervision_does_not_restart_the_incumbent_a_stop_first_owner_stopped() -> Result<()> { + let workload = SwitchWorkload::new(); + let (world, deploy, _) = stop_first_world(workload.clone())?; + clear_other_live_transactions(&world, &deploy)?; + age_incumbent(&world, 500_000)?; + let stored = resident_id(&world, &deploy)?; + world.engine.enter(&stored, entering(&stored, DeploymentPhase::Starting)?)?; + workload.kill(); + + let before = incumbent_envelope(&world)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 0, "supervision restarted what the owner stopped"); + assert_eq!(incumbent_envelope(&world)?, before); + + // Released, the same dead incumbent is restarted. + clear_other_live_transactions(&world, "")?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 1); + Ok(()) + } + + /// An owner fencing the incumbent is the only writer of it: supervision + /// skips the generation, so it neither refreshes it from Odin nor repairs + /// or retires around it. + #[test] + fn supervision_leaves_a_generation_its_owner_is_fencing_alone() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (incumbent, admitted) = admit_seeded_incumbent(&world)?; + let owner = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + rephase(&world, &owner.transaction_id, DeploymentPhase::Fencing)?; + // Odin has a newer reading of the incumbent than the one it holds. + odin_republishes(&world, &admitted, seeded_sequence(None) + 5)?; + + let before = incumbent_envelope(&world)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(incumbent_envelope(&world)?, before, "supervision refreshed a generation an owner is fencing"); + + // The same world with no owner takes the reading, so the skip alone held it. + clear_other_live_transactions(&world, "")?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_ne!(incumbent_envelope(&world)?, before); + Ok(()) + } + + /// The step that ends a transaction archives it inside its own write. It + /// left the live set because it finished: progress, not a fault. + #[test] + fn the_step_that_ends_a_transaction_reports_progress_and_no_fault() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let ended = transaction_at(&world, DeploymentPhase::Warming)?; + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + let id = ended.transaction_id; + let mut last = None; + for _ in 0..20 { + let Ok(current) = resident_id(&world, &id) else { + break; + }; + let moved = world.engine.resume_candidate(¤t)?; + if resident_id(&world, &id).is_err() { + last = Some(moved); + break; + } + } + assert_eq!(last, Some(true), "the ending step reported no progress"); + assert!(latest_id(&world, &id)?.is_terminal()); + assert!(world.engine.fault_reports.lock().unwrap().is_empty()); + + // Through the tick, the way the scheduler runs it: no fault is kept. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let ended = transaction_at(&world, DeploymentPhase::Warming)?; + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + for _ in 0..20 { + if resident_id(&world, &ended.transaction_id).is_err() { + break; + } + world.engine.resume_one_transaction()?; + } + assert!(latest_id(&world, &ended.transaction_id)?.is_terminal()); + assert!(world.engine.fault_reports.lock().unwrap().is_empty()); + Ok(()) + } + + #[test] + fn the_store_holds_one_deploy_one_continuity_and_one_owner_per_target() -> Result<()> { + let seed = |first: (CommandKind, DeploymentPhase), second: (CommandKind, DeploymentPhase)| { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let incumbent = admit_incumbent(&world)?; + for (kind, phase) in [first, second] { + let seeded = + seeded_transaction(&world, DeploymentPhase::Warming, kind, Some(&incumbent))?; + if phase != DeploymentPhase::Warming { + rephase(&world, &seeded.transaction_id, phase)?; + } + } + Ok::<_, anyhow::Error>(world) + }; + use CommandKind::{Continuity, Deploy}; + use DeploymentPhase::{Fencing, Warming}; + + // A pre-handover deployment beside an owning continuity, and the + // reverse: one owner, one of each kind. + for (first, second) in [((Deploy, Warming), (Continuity, Fencing)), ((Continuity, Warming), (Deploy, Fencing))] { + let world = seed(first, second)?; + assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 2); + } + let refused = |first, second| -> Result { + let world = seed(first, second)?; + Ok(format!("{:#}", ControlSnapshot::read(&world.state_store).err().context("two were accepted")?)) + }; + assert!(refused((Deploy, Fencing), (Continuity, Fencing))?.contains("own the incumbent slot")); + assert!(refused((Deploy, Warming), (Deploy, Warming))?.contains("multiple live deployments")); + assert!(refused((Continuity, Warming), (Continuity, Warming))?.contains("multiple live continuity transactions")); + Ok(()) + } + + /// The continuity transaction minted over `service`, whatever its state. + fn the_continuity(world: &EngineFixture) -> Result> { + ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .find(|stored| stored.value.command_kind == CommandKind::Continuity) + .context("no live continuity") + } + + #[test] + fn a_deployment_rebases_onto_the_generation_continuity_admitted_mid_build_and_fences_it() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + let deploy = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + let deploy_id = deploy.transaction_id.as_str(); + // The build recorded a route preflight against the incumbent it saw. + let building = resident_id(&world, deploy_id)?; + let mut preflighted = building.value.clone(); + preflighted.route_preflight = Some(crate::drivers::RoutePreflightReceipt { + route_id: "service".into(), + candidate_runtime_instance_id: building + .value + .activation + .as_ref() + .context("no activation")? + .runtime_instance_id + .clone(), + candidate_membership_sha256: sha256_id(b"membership"), + incumbent_runtime_instance_id: None, + incumbent_membership_sha256: None, + incumbent_configuration: None, + }); + replace_transaction(&world.state_store, &building, &preflighted)?; + workload.launching(); + workload.kill(); + assert!(world.engine.supervise_one_admitted_generation()?); + let continuity_id = the_continuity(&world)?.value.transaction_id; + + // Continuity restarts the incumbent beside the build, to Complete. + workload.revive(); + drive_id(&world, &continuity_id, |t| t.phase == DeploymentPhase::Warming)?; + odin_republishes(&world, &latest_id(&world, &continuity_id)?, seeded_sequence(Some(&incumbent)) + 10)?; + drive_id(&world, &continuity_id, |t| t.is_terminal())?; + let restarted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("the restart admitted nothing")? + .value + .clone(); + assert_ne!(restarted.generation_id, incumbent.generation_id); + assert_eq!(restarted.transaction_id, continuity_id); + let untouched = resident_id(&world, deploy_id)?; + assert_eq!(untouched.value.incumbent_generation_id, Some(incumbent.generation_id.clone())); + + // The deployment's next step is the rebase and nothing else. + assert!(untouched.value.route_preflight.is_some() && untouched.value.warming.is_some()); + world.engine.advance_transaction(&untouched)?; + let rebased = resident_id(&world, deploy_id)?.value; + assert_eq!(rebased.incumbent_generation_id, Some(restarted.generation_id.clone())); + assert_eq!(rebased.phase, DeploymentPhase::Warming); + assert!(rebased.warming.is_none() && rebased.route_preflight.is_none() && rebased.isolation.is_none()); + assert!(rebased.pre_fencing_abort.is_none()); + assert!(rebased.odin_publisher_sequence_cursor >= restarted.odin_publisher_sequence_cursor); + assert_eq!(rebased.expected, untouched.value.expected); + + // It then fences the generation continuity admitted, and replaces it. + odin_reports_ready(&world, &deploy, restarted.odin_publisher_sequence_cursor + 10)?; + drive_id(&world, deploy_id, |t| t.is_terminal())?; + let landed = latest_id(&world, deploy_id)?; + assert!(matches!(landed.completion, Some(TransactionCompletion::Admitted { .. })), "{landed:?}"); + assert_eq!(landed.fencing, Some(FencingEvidence::SkippedStateless)); + assert!(matches!( + landed.post_commit_cleanup.map(|cleanup| cleanup.incumbent), + Some( + IncumbentCleanupEvidence::Pending { generation_id, .. } + | IncumbentCleanupEvidence::Complete { generation_id } + ) if generation_id == restarted.generation_id + )); + let now_admitted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("the deployment admitted nothing")? + .value + .transaction_id + .clone(); + assert_eq!(now_admitted, deploy_id); + Ok(()) + } + + /// Kill the Warming step with the workload: if the gate did not hold the + /// deployment, running the step would be a pre-fencing error and an abort. + #[test] + fn a_deployment_waits_in_warming_while_a_continuity_owns_the_target() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; + workload.kill(); + let deploy_id = deploy.transaction_id.as_str(); + let before = resident_id(&world, deploy_id)?; + + for _ in 0..3 { + let stored = resident_id(&world, deploy_id)?; + world.engine.advance_transaction(&stored)?; + } + let waiting = resident_id(&world, deploy_id)?; + assert_eq!(waiting.value.phase, DeploymentPhase::Warming); + assert!(waiting.value.pre_fencing_abort.is_none()); + assert_eq!(waiting.value.warming, before.value.warming); + assert_eq!(waiting.value.isolation, before.value.isolation); + assert!( + waiting.value.last_error.as_deref().is_some_and(|note| note.contains(&continuity.transaction_id)), + "{:?}", + waiting.value.last_error + ); + + // The build overlaps: a deployment still in Sealing is not gated. + rephase(&world, deploy_id, DeploymentPhase::Sealing)?; + let sealing = resident_id(&world, deploy_id)?; + world.engine.advance_transaction(&sealing)?; + let after = resident_id(&world, deploy_id)?.value; + assert!( + after.last_error.as_deref().is_some_and(|note| !note.contains(&continuity.transaction_id)), + "Sealing was held by the owner: {:?}", + after.last_error + ); + + // The owner completes having moved the generation: the next step is the + // rebase, and with the target free the deployment goes on. + rephase(&world, deploy_id, DeploymentPhase::Warming)?; + workload.revive(); + clear_other_live_transactions(&world, deploy_id)?; + edit_incumbent(&world, |generation| generation.generation_id = "generation-restarted".into())?; + world.engine.advance_transaction(&resident_id(&world, deploy_id)?)?; + let rebased = resident_id(&world, deploy_id)?.value; + assert_eq!(rebased.incumbent_generation_id.as_deref(), Some("generation-restarted")); + assert!(rebased.warming.is_none()); + Ok(()) + } + + #[test] + fn a_continuity_whose_target_was_taken_or_generation_replaced_aborts_itself_and_is_not_rebased() -> Result<()> { + let resume = |world: &EngineFixture, id: &str| -> Result<()> { + for _ in 0..12 { + let stored = resident_id(world, id); + let Ok(stored) = stored else { break }; + // The last step archives the record, which the scheduler's own + // progress read reports as an error; the record is what counts. + let _ = world.engine.resume_candidate(&stored); + world.engine.resume_backoff.lock().unwrap().clear(); + } + Ok(()) + }; + // Taken: the deployment owns the target; the continuity ends itself and + // the deployment goes on, untouched. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Fencing, DeploymentPhase::Warming)?; + let owner = resident_id(&world, &deploy.transaction_id)?; + resume(&world, &continuity.transaction_id)?; + let ended = latest_id(&world, &continuity.transaction_id)?; + assert!( + matches!(&ended.completion, Some(TransactionCompletion::FailedBeforeFencing { error }) if error.contains("continuity superseded")), + "{:?}", + ended.completion + ); + assert_eq!(resident_id(&world, &deploy.transaction_id)?.envelope, owner.envelope); + + // Replaced: the generation it restarts is gone. It is not rebased. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, incumbent) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + clear_other_live_transactions(&world, &continuity.transaction_id)?; + edit_incumbent(&world, |generation| generation.generation_id = "generation-elsewhere".into())?; + resume(&world, &continuity.transaction_id)?; + let ended = latest_id(&world, &continuity.transaction_id)?; + assert!( + matches!(&ended.completion, Some(TransactionCompletion::FailedBeforeFencing { error }) if error.contains("continuity superseded")), + "{:?}", + ended.completion + ); + assert_eq!(ended.incumbent_generation_id, Some(incumbent.generation_id)); + let _ = deploy; + Ok(()) + } + + #[test] + fn ownership_is_read_per_target_and_names_only_a_transaction_that_holds_it() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; + let mut view = ControlSnapshot::read(&world.state_store)?; + assert_eq!(view.on_target("service").count(), 2); + assert_eq!(view.on_target("elsewhere").count(), 0); + // The pre-handover deployment holds nothing; the continuity holds the slot. + let owner = view.target_owner("service", None).context("no owner")?; + assert_eq!(owner.value.transaction_id, continuity.transaction_id); + assert!(view.target_owner("service", Some(&continuity.transaction_id)).is_none()); + assert!(view.target_owner("elsewhere", None).is_none()); + // The same records filed under another target are that target's. + for stored in &mut view.transactions { + stored.value.target = "elsewhere".into(); + } + assert_eq!(view.on_target("service").count(), 0); + assert!(view.target_owner("service", None).is_none()); + assert_eq!(view.on_target("elsewhere").count(), 2); + assert!(view.target_owner("elsewhere", None).is_some()); + let _ = deploy; + Ok(()) + } + + /// A deployment that has committed still drains the old incumbent; the + /// target is not restarted over that cleanup, and is once it ends. + #[test] + fn a_target_draining_its_old_incumbent_is_not_restarted_until_the_drain_ends() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + let deploy = + seeded_transaction(&world, DeploymentPhase::Fencing, CommandKind::Deploy, Some(&incumbent))?; + odin_reports_ready(&world, &deploy, seeded_sequence(Some(&incumbent)) + 1)?; + drive_id(&world, &deploy.transaction_id, |t| t.phase == DeploymentPhase::Complete)?; + let draining = resident_id(&world, &deploy.transaction_id)?.value; + assert!(draining.drains_incumbent() && !draining.owns_target() && !draining.is_live()); + + age_incumbent(&world, 500_000)?; + workload.kill(); + for _ in 0..3 { + let _ = world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 0, "restarted over a draining incumbent"); + } + drive_id(&world, &deploy.transaction_id, |t| t.is_terminal())?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 1); + Ok(()) + } + + /// A deployment that has not taken the target no longer suppresses the + /// repair of the admitted Expected (only the Expected, only when inexact). + #[test] + fn the_admitted_expected_is_repaired_beside_a_deployment_that_is_building() -> Result<()> { + let world = EngineFixture::with_workload(SwitchWorkload::new())?; + let incumbent = admit_incumbent(&world)?; + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + let store = world.engine.options.topology_store.clone(); + if store.exists() { + std::fs::remove_file(&store)?; + } + assert_eq!(projected_under(&world, &incumbent.expected)?, Vec::::new()); + + let _ = world.engine.supervise_one_admitted_generation()?; + assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); + Ok(()) + } + + #[test] + fn a_candidate_port_is_held_by_the_admitted_a_live_and_a_draining_transaction_only() -> Result<()> { + use crate::deployment_plan::tests::BINDING; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + let mut view = ControlSnapshot::read(&world.state_store)?; + assert!(view.candidate_ports_in_use().is_empty()); + + // Ports are set in memory: the fixture plans are unrouted. + view.admitted[0].value.plan.candidate_port = Some(18000); + for stored in &mut view.transactions { + let port = if stored.value.transaction_id == deploy.transaction_id { 18001 } else { 18002 }; + stored.value.plan.as_mut().context("no plan")?.candidate_port = Some(port); + } + assert_eq!(view.candidate_ports_in_use(), BTreeSet::from([18000, 18001, 18002])); + // Complete with cleanup still pending drains: its port stays held. + let finished = view + .transactions + .iter_mut() + .find(|stored| stored.value.transaction_id == continuity.transaction_id) + .context("no continuity")?; + finished.value.phase = DeploymentPhase::Complete; + finished.value.post_commit_cleanup = Some(PostCommitCleanup { + incumbent: IncumbentCleanupEvidence::SkippedNoIncumbent, + source: SourceCleanupEvidence::Pending, + }); + assert_eq!(view.candidate_ports_in_use(), BTreeSet::from([18000, 18001, 18002])); + let finished = view + .transactions + .iter_mut() + .find(|stored| stored.value.transaction_id == continuity.transaction_id) + .context("no continuity")?; + finished.value.post_commit_cleanup = Some(PostCommitCleanup { + incumbent: IncumbentCleanupEvidence::SkippedNoIncumbent, + source: SourceCleanupEvidence::Complete, + }); + assert_eq!(view.candidate_ports_in_use(), BTreeSet::from([18000, 18001])); + + // A routed binding gets the first free port of its range; an unrouted one none. + let routed = OperatorBinding::parse(BINDING)?; + assert_eq!(world.engine.select_candidate_port(&routed)?, Some(routed.route.as_ref().context("route")?.private_port_start)); + let (head, tail) = BINDING.split_once("[route]").context("route")?; + let unrouted = OperatorBinding::parse(&format!("{head}[brakes]{}", tail.split_once("[brakes]").context("brakes")?.1))?; + assert_eq!(world.engine.select_candidate_port(&unrouted)?, None); + Ok(()) + } + + #[test] + fn only_a_deployment_before_its_handover_is_rebased() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + // Another generation is admitted under both. + edit_incumbent(&world, |generation| generation.generation_id = "generation-restarted".into())?; + let rebase = |id: &str| -> Result { + world.engine.rebase_onto_admitted(&resident_id(&world, id)?) + }; + // A continuity restarts one generation and no other. + assert!(!rebase(&continuity.transaction_id)?); + assert_ne!( + resident_id(&world, &continuity.transaction_id)?.value.incumbent_generation_id.as_deref(), + Some("generation-restarted") + ); + // A deployment already ending is left to end. + let ending = resident_id(&world, &deploy.transaction_id)?; + let mut aborting = ending.value.clone(); + aborting.pre_fencing_abort = Some(pre_fencing_abort_intent(&aborting, "cancelled by operator")); + replace_transaction(&world.state_store, &ending, &aborting)?; + assert!(!rebase(&deploy.transaction_id)?); + let ending = resident_id(&world, &deploy.transaction_id)?; + let mut live = ending.value.clone(); + live.pre_fencing_abort = None; + replace_transaction(&world.state_store, &ending, &live)?; + // Past its handover the incumbent is pinned, whatever is admitted. + rephase(&world, &deploy.transaction_id, DeploymentPhase::Fencing)?; + assert!(!rebase(&deploy.transaction_id)?); + Ok(()) + } + + #[test] + fn the_handover_does_not_wait_on_a_finished_transaction_being_retired() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + // A committed transaction left resident, finished and not yet retired. + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; + odin_reports_ready(&world, &seeded, seeded_sequence(None) + 1)?; + drive(&world, |transaction| transaction.phase == DeploymentPhase::Complete)?; + let incumbent = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("commit wrote no admitted generation")? + .value + .clone(); + let deploy = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + let view = ControlSnapshot::read(&world.state_store)?; + assert!(view.transactions.iter().any(|stored| stored.value.is_terminal())); + + // It is retired between the read and the handover; that is no race. + assert!(world.engine.retire_one_terminal_transaction()?); + let stored = resident_id(&world, &deploy.transaction_id)?; + assert_eq!( + world.engine.take_target(&view, &stored, &entering(&stored, DeploymentPhase::Fencing)?)?, + Handover::Took + ); + Ok(()) + } + + /// A dead incumbent with a deployment building beside it (its candidate + /// alive) whose Warming evidence is complete, and that deployment. + fn dead_incumbent_with_a_deployment_ready_to_fence( + workload: &SwitchWorkload, + world: &EngineFixture, + ) -> Result { + let incumbent = admit_incumbent(world)?; + age_incumbent(world, 500_000)?; + let deploy = + seeded_transaction(world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + let candidate = deploy.activation.as_ref().context("no activation")?; + workload.spare(&candidate.runtime_instance_id); + workload.kill(); + Ok(deploy) + } + + #[test] + fn a_continuity_minted_as_a_deployment_takes_the_target_is_resolved_next_pass() -> Result<()> { + // (a) The deployment fences first in the tick, so supervision reads its + // ownership and mints nothing. + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let deploy = dead_incumbent_with_a_deployment_ready_to_fence(&workload, &world)?; + assert!(world.engine.run_scheduler_tick()?); + assert_eq!(resident_id(&world, &deploy.transaction_id)?.value.phase, DeploymentPhase::Fencing); + assert_eq!(continuity_mints(&world)?, 0); + let _ = world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 0); + + // (b) Supervision minted from the older view: the continuity starts at + // Sealing and owns nothing, the deployment takes the target, and the + // continuity is superseded at its next step, never the deployment. + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let deploy = dead_incumbent_with_a_deployment_ready_to_fence(&workload, &world)?; + assert!(world.engine.supervise_one_admitted_generation()?); + let continuity_id = the_continuity(&world)?.value.transaction_id; + assert_eq!(resident_id(&world, &continuity_id)?.value.phase, DeploymentPhase::Sealing); + world.engine.advance_transaction(&resident_id(&world, &deploy.transaction_id)?)?; + assert_eq!(resident_id(&world, &deploy.transaction_id)?.value.phase, DeploymentPhase::Fencing); + for _ in 0..8 { + let _ = world.engine.run_scheduler_tick(); + world.engine.resume_backoff.lock().unwrap().clear(); + let snapshot = ControlSnapshot::read(&world.state_store)?; + assert!(snapshot.transactions.iter().filter(|stored| stored.value.owns_target()).count() <= 1); + } + let ended = latest_id(&world, &continuity_id)?; + assert!( + matches!(&ended.completion, Some(TransactionCompletion::FailedBeforeFencing { error }) if error.contains("continuity superseded")), + "{:?}", + ended.completion + ); + assert!(latest_id(&world, &deploy.transaction_id)?.pre_fencing_abort.is_none()); + Ok(()) + } + + /// Put the target's backoff anchor `ago` milliseconds back. + fn anchor_ago(world: &EngineFixture, ago: u64) -> Result<()> { + let mut meters = meters_of(world, "service")?; + meters.continuity_backoff_from_unix_millis = Some(now_millis()? - ago); + set_meters(world, &meters) + } + + /// Continuity transactions minted over an incumbent (the one that admitted + /// it was seeded over nothing). + fn continuity_mints(world: &EngineFixture) -> Result { + Ok(every_transaction(world)? + .iter() + .filter(|transaction| { + transaction.command_kind == CommandKind::Continuity + && transaction.incumbent_generation_id.is_some() + }) + .count()) + } + + #[test] + fn continuity_keeps_restarting_past_the_paging_threshold() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + workload.kill(); + // Six charged attempts in the hour: the incident pages, nothing stops. set_meters( &world, &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, )?; - world.engine.supervise_one_admitted_generation()?; - let after = resident(&world)?.value; - assert_eq!(after.transaction_id, redeploy.transaction_id); - assert!(after.pre_fencing_abort.is_none(), "yielded with no restarts left"); + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + assert!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis.is_some()); + let opened = incidents_of(&world)?; + assert_eq!(opened.len(), 1); + assert!(opened[0].is_open()); - set_meters(&world, &restarted_ago(&[200_000])?)?; - world.engine.supervise_one_admitted_generation()?; - assert!(resident(&world)?.value.pre_fencing_abort.is_some()); + // The streak-6 wait is 160 s. + anchor_ago(&world, 159_000)?; + assert!(!world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + anchor_ago(&world, 160_001)?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 1); + let after = meters_of(&world, "service")?; + assert_eq!(after.continuity_restarts.len(), 6); + assert!(after.continuity_restarts.last().is_some_and(|&last| now_millis().unwrap() - last < 60_000)); + assert_eq!(after.continuity_attempt_streak, 7); + assert_eq!(after.continuity_backoff_from_unix_millis, None); + assert_eq!(incidents_of(&world)?, opened); Ok(()) } + #[test] + fn continuity_backoff_is_a_capped_doubling_of_the_attempt_streak() { + let from = 10_000_000_u64; + for (streak, seconds) in [ + (1_u32, 5_u64), + (2, 10), + (3, 20), + (4, 40), + (5, 80), + (6, 160), + (7, 300), + (8, 300), + (9, 300), + ] { + let mut meters = TargetSupervision::new("service"); + meters.continuity_attempt_streak = streak; + meters.continuity_backoff_from_unix_millis = Some(from); + // The paging log disagrees with the streak on purpose: the wait + // is a function of the streak, not of the window's count. + meters.continuity_restarts = vec![from - 10; if streak == 1 { 6 } else { 2 }]; + let due = from + seconds * 1_000; + assert_eq!(meters.backoff_due(0), Some(due), "streak {streak}"); + assert_eq!( + meters.continuity_backoff(due - 1, 0), + ContinuityBackoff::Waiting, + "streak {streak}" + ); + assert_eq!( + meters.continuity_backoff(due, 0), + ContinuityBackoff::Due { streak }, + "streak {streak}" + ); + } + // No streak, no wait. + let mut fresh = TargetSupervision::new("service"); + fresh.continuity_backoff_from_unix_millis = Some(from); + assert_eq!(fresh.continuity_backoff(from, 0), ContinuityBackoff::Due { streak: 0 }); + } + + #[test] + fn a_sustained_run_resets_the_streak_and_only_a_sustained_run() { + let now = 100_000_000_u64; + let mut meters = TargetSupervision::new("service"); + meters.continuity_attempt_streak = 7; + meters.continuity_restarts = vec![1_000]; + // Lived the sustained time, and no attempt since admission. + assert_eq!( + meters.continuity_backoff(now, now - CONTINUITY_SUSTAINED_RUN_MILLIS), + ContinuityBackoff::Due { streak: 0 } + ); + // A moment short of it, down to the millisecond. + for short in [10_000, 1_000, 1] { + assert_eq!( + meters.continuity_backoff(now, now - CONTINUITY_SUSTAINED_RUN_MILLIS + short), + ContinuityBackoff::Begin, + "{short} ms short" + ); + } + // The newest attempt decides, not the oldest: one before admission and + // one after it, on a generation that has lived long enough. + meters.continuity_restarts = vec![now - 800_000, now - 100_000]; + assert_eq!(meters.continuity_backoff(now, now - 700_000), ContinuityBackoff::Begin); + meters.continuity_restarts = vec![1_000]; + // Attempts after admission failed on this generation: it has not run well. + meters.continuity_restarts = vec![now - 100_000]; + assert_eq!(meters.continuity_backoff(now, now - 700_000), ContinuityBackoff::Begin); + // Never attempted over this log, long lived: reset. + meters.continuity_restarts = Vec::new(); + assert_eq!( + meters.continuity_backoff(now, now - 700_000), + ContinuityBackoff::Due { streak: 0 } + ); + assert_eq!(meters.continuity_backoff(now, now - 590_000), ContinuityBackoff::Begin); + // An anchor older than its generation decides nothing. + meters.continuity_backoff_from_unix_millis = Some(now - 800_000); + assert_eq!(meters.backoff_due(now - 700_000), None); + assert_eq!( + meters.continuity_backoff(now, now - 700_000), + ContinuityBackoff::Due { streak: 0 } + ); + assert_eq!(meters.continuity_backoff(now, now - 590_000), ContinuityBackoff::Begin); + // Streak zero with no history is simply due. + let fresh = TargetSupervision::new("service"); + assert_eq!(fresh.continuity_backoff(now, now - 1_000), ContinuityBackoff::Due { streak: 0 }); + } + + #[test] + fn a_wait_that_crosses_the_sustained_run_ends_early_and_only_by_less_than_the_cap() { + let admitted = 50_000_000_u64; + let mut meters = TargetSupervision::new("service"); + meters.continuity_attempt_streak = 7; + meters.continuity_restarts = vec![admitted - 1_000]; + // The wait began when the generation was 400 s old and runs 300 s. + meters.continuity_backoff_from_unix_millis = Some(admitted + 400_000); + assert_eq!(meters.backoff_due(admitted), Some(admitted + 700_000)); + assert_eq!( + meters.continuity_backoff(admitted + CONTINUITY_SUSTAINED_RUN_MILLIS - 1, admitted), + ContinuityBackoff::Waiting + ); + assert_eq!( + meters.continuity_backoff(admitted + CONTINUITY_SUSTAINED_RUN_MILLIS, admitted), + ContinuityBackoff::Due { streak: 0 } + ); + assert!( + CONTINUITY_SUSTAINED_RUN_MILLIS - 400_000 < CONTINUITY_BACKOFF_CEILING_MILLIS, + "the shortening is under the cap" + ); + } + + #[test] + fn a_continuity_wait_runs_from_when_the_target_was_found_dead_and_free() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + workload.kill(); + let mut meters = restarted_ago(&[1_000_000])?; + meters.continuity_attempt_streak = 3; + set_meters(&world, &meters)?; + + // The last attempt was long ago, but the wait starts now: a wait + // anchored on the mint would already be over. + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + assert!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis.is_some()); + assert!(!world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + + // Streak 3 waits 20 s. + anchor_ago(&world, 19_000)?; + assert!(!world.engine.supervise_one_admitted_generation()?); + anchor_ago(&world, 20_001)?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 1); + assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, 4); + Ok(()) + } + + #[test] + fn a_queued_deployment_freezes_in_the_gap_after_a_continuity_attempt() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + workload.kill(); + let mut meters = restarted_ago(&[3_000_000])?; + meters.continuity_attempt_streak = 2; + set_meters(&world, &meters)?; + queue_declaring_redeploy(&world, "up-service-queued")?; + + for _ in 0..4 { + let _ = world.engine.run_scheduler_tick(); + } + let all = every_transaction(&world)?; + assert!( + all.iter().any(|transaction| transaction.command_id == "up-service-queued"), + "the queued deployment never froze" + ); + // Freeze needs no gap (a deployment before its handover owns nothing), + // and continuity may restart the incumbent beside it after its wait. + for _ in 0..3 { + let _ = world.engine.run_scheduler_tick(); + let aborts = every_transaction(&world)? + .into_iter() + .filter_map(|transaction| transaction.pre_fencing_abort) + .collect::>(); + // The deployment may fail on its own account (a root test runner + // cannot resolve its source); continuity must never be the reason. + assert!( + aborts.iter().all(|abort| !abort.error.contains("continuity")), + "the deployment was yielded: {aborts:?}" + ); + } + Ok(()) + } + + #[test] + fn a_unit_lost_to_a_host_reboot_is_rematerialized_once_with_no_backoff_step() -> Result<()> { + let fresh_world = |workload: &Arc| -> Result { + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + let mut meters = restarted_ago(&[1_000_000])?; + meters.continuity_attempt_streak = 3; + meters.continuity_backoff_from_unix_millis = Some(now_millis()?); + set_meters(&world, &meters)?; + Ok(world) + }; + let workload = SwitchWorkload::new(); + let world = fresh_world(&workload)?; + let before = meters_of(&world, "service")?; + + // Control: dead but not rebooted, the ordinary 20 s wait holds. + workload.kill(); + assert!(!world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + + // The host rebooted under the unit: one mint, charged to nothing. + workload.reboot(); + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 1); + let after = meters_of(&world, "service")?; + assert_eq!(after.continuity_restarts, before.continuity_restarts); + assert_eq!(after.continuity_attempt_streak, 3); + assert!(after.continuity_rematerialized_at_unix_millis.is_some()); + assert_eq!(after.continuity_backoff_from_unix_millis, None); + + // The bound is spent: the next death of this generation waits like any other. + clear_live_transactions(&world)?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + assert!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis.is_some()); + assert!(!world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + + // Control: a deferred projection spaces a re-materialization too. + let workload = SwitchWorkload::new(); + let world = fresh_world(&workload)?; + let mut deferred = meters_of(&world, "service")?; + deferred.defer_continuity(now_millis()?, "demotion failed"); + set_meters(&world, &deferred)?; + workload.reboot(); + assert!(!world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + Ok(()) + } + + fn supervision_envelope_raw(payload: Vec, schema: &str) -> Result { + Ok(CultCacheEnvelope { + key: "service".into(), + r#type: TargetSupervision::TYPE.into(), + payload, + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some(schema.into()), + }) + } + + #[test] + fn a_v1_target_supervision_record_is_lifted_and_migrated_once() -> Result<()> { + let world = EngineFixture::new()?; + let meters = restarted_ago(&[200_000])?; + // A record as every one on a running host was written: seven keys, v1. + let payload = rmp_serde::to_vec(&( + TARGET_SUPERVISION_SCHEMA_V1, + &meters.target, + &meters.route_actuations, + &meters.continuity_restarts, + &meters.continuity_deferred_until, + &meters.continuity_deferral_reason, + &meters.route_survivals, + ))?; + SingleFileMessagePackBackingStore::new(&world.state_store).insert_entry_if_absent( + supervision_envelope_raw(payload, TARGET_SUPERVISION_SCHEMA_V1)?, + )?; + let read = meters_of(&world, "service")?; + assert_eq!(read.continuity_restarts, meters.continuity_restarts); + assert_eq!(read.continuity_attempt_streak, 0); + assert_eq!(read.continuity_backoff_from_unix_millis, None); + assert_eq!(read.continuity_rematerialized_at_unix_millis, None); + + assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); + assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 0); + let stored = ControlSnapshot::read(&world.state_store)?; + let envelope = &stored.supervision_for("service").unwrap().envelope; + assert_eq!(envelope.schema_id.as_deref(), Some(TARGET_SUPERVISION_SCHEMA)); + assert_eq!(meters_of(&world, "service")?.continuity_restarts, meters.continuity_restarts); + + let mut anchored = TargetSupervision::new("service"); + anchored.continuity_backoff_from_unix_millis = Some(0); + assert!(anchored.validate().is_err()); + anchored.continuity_backoff_from_unix_millis = None; + anchored.continuity_rematerialized_at_unix_millis = Some(0); + assert!(anchored.validate().is_err()); + Ok(()) + } + + #[test] + fn one_target_supervision_schema_id_names_one_shape() -> Result<()> { + let meters = restarted_ago(&[200_000])?; + let ten = rmp_serde::to_vec(&meters)?; + let seven = rmp_serde::to_vec(&( + TARGET_SUPERVISION_SCHEMA, + &meters.target, + &meters.route_actuations, + &meters.continuity_restarts, + &meters.continuity_deferred_until, + &meters.continuity_deferral_reason, + &meters.route_survivals, + ))?; + // The current id refuses the old shape and the old id refuses the new one. + assert!( + read_supervision_record(&supervision_envelope_raw(seven, TARGET_SUPERVISION_SCHEMA)?) + .is_err() + ); + assert!( + read_supervision_record(&supervision_envelope_raw( + ten.clone(), + TARGET_SUPERVISION_SCHEMA_V1 + )?) + .is_err() + ); + assert_eq!( + read_supervision_record(&supervision_envelope_raw(ten, TARGET_SUPERVISION_SCHEMA)?)?, + meters + ); + Ok(()) + } + + #[test] + fn continuity_keeps_minting_at_every_streak_and_log_length() -> Result<()> { + // No stop at any streak, however long, with the paging log full or not. + let full = [3_000_000_u64, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000]; + for streak in 1..=12_u32 { + for log in [&[1_000_000_u64][..], &full[..]] { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + workload.kill(); + let mut meters = restarted_ago(log)?; + meters.continuity_attempt_streak = streak; + set_meters(&world, &meters)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 0, "streak {streak}: the wait begins"); + anchor_ago(&world, 300_001)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 1, "streak {streak} log {} stopped", log.len()); + assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, streak + 1); + } + } + Ok(()) + } + + #[test] + fn a_mint_after_a_sustained_run_stores_the_reset_streak() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 700_000)?; + workload.kill(); + let mut meters = restarted_ago(&[3_000_000])?; + meters.continuity_attempt_streak = 7; + set_meters(&world, &meters)?; + // Ran 700 s since the last attempt: no wait, and the streak restarts at 1. + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 1); + assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, 1); + Ok(()) + } + + #[test] + fn a_re_materialization_counts_as_the_last_attempt_for_the_sustained_reset() { + let admitted_at = 10_000_000_u64; + let now = admitted_at + 700_000; + let mut meters = TargetSupervision::new("service"); + meters.continuity_attempt_streak = 5; + meters.continuity_restarts = vec![admitted_at - 1_000]; + // Before admission: a sustained run, reset. + meters.continuity_rematerialized_at_unix_millis = Some(admitted_at - 1); + assert_eq!( + meters.continuity_backoff(now, admitted_at), + ContinuityBackoff::Due { streak: 0 } + ); + // After admission: that attempt failed on this generation. + meters.continuity_rematerialized_at_unix_millis = Some(admitted_at + 1); + assert_eq!(meters.continuity_backoff(now, admitted_at), ContinuityBackoff::Begin); + // Even with no streak to step: the re-materialization is an attempt, + // and the next one waits the base step from its anchor. + meters.continuity_attempt_streak = 0; + meters.continuity_restarts = Vec::new(); + assert_eq!(meters.continuity_backoff(now, admitted_at), ContinuityBackoff::Begin); + meters.continuity_backoff_from_unix_millis = Some(now); + assert_eq!(meters.backoff_due(admitted_at), Some(now + CONTINUITY_RESTART_BACKOFF_MILLIS)); + assert_eq!(meters.continuity_backoff(now, admitted_at), ContinuityBackoff::Waiting); + assert_eq!( + meters.continuity_backoff(now + CONTINUITY_RESTART_BACKOFF_MILLIS, admitted_at), + ContinuityBackoff::Due { streak: 0 } + ); + meters.continuity_rematerialized_at_unix_millis = None; + assert_eq!(meters.backoff_due(admitted_at), Some(now)); + } + + #[test] + fn continuity_mints_at_the_largest_streaks_and_stores_them_saturating() -> Result<()> { + // One test at u32::MAX kills every monotone stop gate (`streak >= N`). + let full = [3_000_000_u64, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000]; + for streak in [13_u32, 64, u32::MAX - 1, u32::MAX] { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + workload.kill(); + let mut meters = restarted_ago(&full)?; + meters.continuity_attempt_streak = streak; + set_meters(&world, &meters)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 0, "streak {streak}: the wait begins"); + anchor_ago(&world, 299_000)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 0, "streak {streak}: the cap is 300 s"); + anchor_ago(&world, 300_001)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 1, "streak {streak} stopped"); + assert_eq!( + meters_of(&world, "service")?.continuity_attempt_streak, + streak.saturating_add(1) + ); + } + Ok(()) + } + + #[test] + fn a_brief_run_does_not_reset_the_streak() -> Result<()> { + // A crash loop seen running for one poll after each start still waits + // by streak; only a sustained run resets. + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 100_000)?; + workload.kill(); + let mut meters = restarted_ago(&[1_000_000, 900_000, 800_000])?; + meters.continuity_attempt_streak = 3; + set_meters(&world, &meters)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 0); + workload.revive(); + world.engine.supervise_one_admitted_generation()?; + workload.kill(); + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 0, "a brief run reset the wait to nothing"); + anchor_ago(&world, 19_000)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 0, "streak 3 waits 20 s"); + anchor_ago(&world, 20_001)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 1); + assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, 4); + Ok(()) + } + + #[test] + fn a_sustained_run_no_pass_observed_still_resets_past_a_standing_anchor() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 100_000)?; + workload.kill(); + let mut meters = restarted_ago(&[1_500_000, 1_400_000, 1_300_000])?; + meters.continuity_attempt_streak = 3; + set_meters(&world, &meters)?; + world.engine.supervise_one_admitted_generation()?; + let anchor = meters_of(&world, "service")?.continuity_backoff_from_unix_millis; + assert!(anchor.is_some()); + // It comes back on its own and runs 700 s while every pass is silence. + workload.revive(); + workload.silence(true); + age_incumbent(&world, 700_000)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!( + meters_of(&world, "service")?.continuity_backoff_from_unix_millis, + anchor, + "a silent pass wrote the record" + ); + // Then it dies: the old anchor has no say, no wait, the streak restarts. + workload.kill(); + workload.silence(false); + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 1); + assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, 1); + Ok(()) + } + + #[test] + fn owed_steps_counts_the_re_materialization_only_for_its_own_generation() { + let admitted_at = 10_000_000_u64; + let mut meters = TargetSupervision::new("service"); + assert_eq!(meters.owed_steps(admitted_at), None); + meters.continuity_rematerialized_at_unix_millis = Some(admitted_at - 1); + assert_eq!(meters.owed_steps(admitted_at), None, "an earlier generation's remat is spent"); + meters.continuity_rematerialized_at_unix_millis = Some(admitted_at); + assert_eq!(meters.owed_steps(admitted_at), Some(0)); + meters.continuity_attempt_streak = 3; + assert_eq!(meters.owed_steps(admitted_at), Some(2)); + } + + #[test] + fn a_v1_supervision_record_whose_inner_schema_differs_is_refused() -> Result<()> { + let meters = restarted_ago(&[200_000])?; + let payload = rmp_serde::to_vec(&( + "idunn.some_other_schema", + &meters.target, + &meters.route_actuations, + &meters.continuity_restarts, + &meters.continuity_deferred_until, + &meters.continuity_deferral_reason, + &meters.route_survivals, + ))?; + assert!( + read_supervision_record(&supervision_envelope_raw(payload, TARGET_SUPERVISION_SCHEMA_V1)?) + .is_err() + ); + Ok(()) + } + + mod after_a_reboot { + use super::*; + + fn resident_continuity(world: &EngineFixture) -> Result { + Ok(ControlSnapshot::read(&world.state_store)? + .transactions + .iter() + .filter(|s| s.value.command_kind == CommandKind::Continuity) + .count()) + } + + /// The attempt failed and was retired: drop continuity transactions + /// and their commands only. + fn drop_continuity(world: &EngineFixture) -> Result<()> { + let snapshot = ControlSnapshot::read(&world.state_store)?; + let ids = snapshot + .transactions + .iter() + .filter(|s| s.value.command_kind == CommandKind::Continuity) + .map(|s| s.value.command_id.clone()) + .collect::>(); + let envelopes = snapshot + .transactions + .iter() + .filter(|s| s.value.command_kind == CommandKind::Continuity) + .map(|s| s.envelope.clone()) + .chain( + snapshot + .commands + .iter() + .filter(|s| ids.contains(&s.value.command_id)) + .map(|s| s.envelope.clone()), + ) + .collect::>(); + if !envelopes.is_empty() { + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store) + .delete_batch_if_unchanged(&envelopes)? + ); + } + Ok(()) + } + + /// (attempts minted on tick 1, on tick 2, whether the queued deploy froze by then). + fn gap_run(reboot: bool) -> Result<(usize, usize, bool)> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 700_000)?; + if reboot { + workload.reboot(); + } else { + workload.kill(); + } + queue_declaring_redeploy(&world, "up-gap")?; + // One tick, in the tick's order: supervise, then freeze. + world.engine.supervise_one_admitted_generation()?; + let first = resident_continuity(&world)?; + world.engine.freeze_one_queued_command()?; + // That attempt failed and was retired. + drop_continuity(&world)?; + world.engine.supervise_one_admitted_generation()?; + let second = resident_continuity(&world)?; + world.engine.freeze_one_queued_command()?; + let frozen = every_transaction(&world)?.iter().any(|t| t.command_id == "up-gap"); + Ok((first, second, frozen)) + } + + #[test] + fn a_failed_re_materialization_is_followed_by_a_gap_a_queued_deployment_can_take( + ) -> Result<()> { + // Control: an ordinary attempt after a sustained run leaves a gap. + assert_eq!(gap_run(false)?, (1, 0, true)); + // The re-materialization is an attempt too: the next pass waits. + assert_eq!(gap_run(true)?, (1, 0, true)); + Ok(()) + } + + #[test] + fn a_running_pass_ends_the_wait_so_a_later_death_after_a_sustained_run_resets( + ) -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + workload.kill(); + let mut meters = restarted_ago(&[1_000_000])?; + meters.continuity_attempt_streak = 3; + set_meters(&world, &meters)?; + world.engine.supervise_one_admitted_generation()?; + assert!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis.is_some()); + workload.revive(); + world.engine.supervise_one_admitted_generation()?; + assert_eq!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis, None); + + // It runs on for days, then dies: no wait, and the streak restarts. + age_incumbent(&world, 200_000)?; + workload.kill(); + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 1); + assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, 1); + Ok(()) + } + + #[test] + fn a_running_pass_keeps_the_re_materialization_so_the_next_death_still_waits() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 700_000)?; + workload.reboot(); + world.engine.supervise_one_admitted_generation()?; + assert_eq!(resident_continuity(&world)?, 1); + drop_continuity(&world)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(resident_continuity(&world)?, 0, "the gap"); + // Seen running, then dead again: the failed attempt still counts. + workload.revive(); + world.engine.supervise_one_admitted_generation()?; + workload.kill(); + world.engine.supervise_one_admitted_generation()?; + assert_eq!(resident_continuity(&world)?, 0, "no gap after the failed attempt"); + anchor_ago(&world, 5_001)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(resident_continuity(&world)?, 1); + Ok(()) + } + } + /// A source that resolves one fixed recipe and never freezes anything. struct FixedSource { recipe: String, @@ -15213,6 +20284,97 @@ mod tests { Ok(()) } + /// A source whose resolve step fails with a fixed message, as a build or + /// resolve step does when its output is a long log. + struct FailingSource { + message: String, + } + + impl SourcePort for FailingSource { + fn resolve( + &self, + _: &OperatorBinding, + _: &str, + _: u64, + ) -> Result { + bail!("{}", self.message) + } + + fn freeze(&self, _: &str, _: &CompiledDeploymentPlan) -> Result { + bail!("this source freezes nothing") + } + + fn observe_frozen( + &self, + _: &CompiledDeploymentPlan, + _: &FrozenSourceReceipt, + ) -> Result { + bail!("this source freezes nothing") + } + + fn cleanup(&self, _: &str, _: Option<&FrozenSourceReceipt>) -> Result<()> { + Ok(()) + } + } + + #[test] + fn bounded_detail_output_always_satisfies_require_detail() -> Result<()> { + for unit in ["a", "\u{e9}", "\u{2500}", "\u{1f600}"] { + for bytes in [0, 1, 511, 512, 513, 2047, 2048, 2049, 2600, 9000] { + // A prefix of one byte offsets every multibyte char against the cuts. + for prefix in ["", "x", "xx", "xxx"] { + let input = format!("{prefix}{}", unit.repeat(bytes / unit.len())); + let output = bounded_detail(&input); + require_detail(&output, "detail")?; + assert!(output.len() <= 2048, "{} bytes", output.len()); + } + } + } + let nul = bounded_detail("a\0b\0"); + require_detail(&nul, "detail")?; + assert_eq!(nul, "ab"); + assert_eq!(bounded_detail("\0"), bounded_detail("")); + require_detail(&bounded_detail(""), "detail")?; + let exact = "a".repeat(2048); + assert_eq!(bounded_detail(&exact), exact); + Ok(()) + } + + #[test] + fn bounded_detail_keeps_the_tail_of_a_long_error() { + let input = format!("first line\n{}\nthe last line", "\u{2500}".repeat(5000)); + let output = bounded_detail(&input); + assert!(output.starts_with("first line")); + assert!(output.ends_with("the last line")); + assert!(output.contains(DETAIL_ELISION)); + assert!(output.len() > 1024, "the budget was not used: {}", output.len()); + } + + #[test] + fn a_sealing_failure_with_a_long_multibyte_error_aborts_and_frees_the_target() -> Result<()> { + // 3000 chars of 3-byte box drawing: 9000 bytes, the shape of a build log. + let log = format!("{}\nthe real error is here", "\u{2500}".repeat(3000)); + let mut world = sealing_world(&unrouted_recipe())?; + world.engine.source = Arc::new(FailingSource { message: log }); + for _ in 0..8 { + world.engine.resume_backoff.lock().unwrap().clear(); + let _ = world.engine.run_scheduler_tick(); + } + assert!( + resident(&world).is_err(), + "the failed transaction still owns the target" + ); + let done = latest(&world)?; + let TransactionCompletion::FailedBeforeFencing { error } = + done.completion.context("the aborted transaction did not complete")? + else { + bail!("the transaction did not fail before fencing"); + }; + assert!(error.len() <= 2048, "{}", error.len()); + assert!(error.ends_with("the real error is here"), "the tail was lost"); + Ok(()) + } + #[test] fn admission_takes_a_recipe_that_declares_its_proof() -> Result<()> { let recipe = unrouted_recipe().replace( @@ -15270,11 +20432,16 @@ mod tests { ); std::fs::create_dir_all(world.root.join("bindings"))?; std::fs::write(world.root.join("bindings/service.toml"), binding)?; + queue_deploy_of(world, command_id, "service") + } + + /// Queue a Deploy command for `selector`; the bindings are the caller's. + fn queue_deploy_of(world: &EngineFixture, command_id: &str, selector: &str) -> Result<()> { let command = DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), command_id: command_id.into(), kind: CommandKind::Deploy, - selector: "service".into(), + selector: selector.into(), requested_by: "operator".into(), requested_at_unix_millis: now_millis()?, }; @@ -15375,25 +20542,6 @@ mod tests { Ok(()) } - /// The control for the test above: the same redeploy over a dead incumbent - /// that is not held is yielded to continuity. - #[test] - fn a_declaring_redeploy_still_yields_to_continuity_when_the_incumbent_is_not_held() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - let incumbent = admit_incumbent(&world)?; - assert!(incumbent.readiness().is_ok()); - let redeploy = - seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; - workload.kill(); - world.engine.supervise_one_admitted_generation()?; - let after = resident(&world)?.value; - assert_eq!(after.transaction_id, redeploy.transaction_id); - let abort = after.pre_fencing_abort.context("the redeploy was not yielded")?; - assert!(abort.error.contains("yielded to continuity"), "{}", abort.error); - Ok(()) - } - /// `seeded` with its own Expected undeclared, so Idunn holds it. fn undeclared(world: &EngineFixture, seeded: &DeploymentTransaction) -> Result { let mut bare = seeded.clone(); @@ -15440,10 +20588,11 @@ mod tests { for _ in 0..4 { let _ = world.engine.supervise_one_admitted_generation()?; } - let after = resident(&world)?.value; - assert_eq!(after.transaction_id, held.transaction_id); + let after = resident_id(&world, &held.transaction_id)?.value; assert!(after.pre_fencing_abort.is_none(), "Idunn aborted a held record"); assert!(after.completion.is_none()); + // Continuity restarts the declared incumbent beside it, once. + assert_eq!(continuity_mints(&world)?, 1); Ok(()) } @@ -15475,7 +20624,7 @@ mod tests { )); assert!(finished.is_terminal()); // The target is released and nothing of the incumbent was touched. - assert!(!finished.blocks_new_target_mutation()); + assert!(!finished.is_live() && !finished.drains_incumbent()); assert_eq!(incumbent_envelope(&world)?, before); assert!( every_transaction(&world)? @@ -15562,24 +20711,31 @@ mod tests { let now = t - 86_400_000; let mut meters = TargetSupervision::new("service"); meters.route_actuations = vec![t; ROUTE_ACTUATION_CEILING]; - meters.continuity_restarts = vec![t; CONTINUITY_RESTART_ATTEMPTS]; + meters.continuity_restarts = vec![t; CONTINUITY_PAGING_RESTARTS]; meters.continuity_deferred_until = Some(t); meters.continuity_deferral_reason = Some("demotion failed".into()); meters.validate()?; // Entries from the future count as now: the meters hold for one window. let read_only = meters.clone(); - assert!(read_only.restarts_exhausted(now)); + assert!(read_only.restarts_page(now)); let refused = meters.charge_route(now, RouteActuation::Forward).unwrap_err(); assert!(refused.reopens_at_unix_millis <= now + ROUTE_ACTUATION_WINDOW_MILLIS); // The clamp is written, so the window runs from the step and no longer. assert_eq!(meters.route_actuations, vec![now; ROUTE_ACTUATION_CEILING]); - assert_eq!(meters.continuity_restarts, vec![now; CONTINUITY_RESTART_ATTEMPTS]); - assert!(!meters.restarts_exhausted(now + CONTINUITY_RESTART_WINDOW_MILLIS)); + assert_eq!(meters.continuity_restarts, vec![now; CONTINUITY_PAGING_RESTARTS]); + assert!(!meters.restarts_page(now + CONTINUITY_RESTART_WINDOW_MILLIS)); meters.charge_route(now + ROUTE_ACTUATION_WINDOW_MILLIS, RouteActuation::Forward)?; // Waits set for the future are due, not stalled for the length of the step. - assert!(!read_only.continuity_is_waiting(now)); + assert!(!read_only.continuity_deferred(now)); + let mut anchored = TargetSupervision::new("service"); + anchored.continuity_attempt_streak = 7; + anchored.continuity_backoff_from_unix_millis = Some(t); + assert_eq!(anchored.continuity_backoff(now, now), ContinuityBackoff::Due { streak: 7 }); + anchored.continuity_attempt_streak = 1; + anchored.continuity_backoff_from_unix_millis = Some(now + 10_000); + assert_eq!(anchored.continuity_backoff(now, now), ContinuityBackoff::Waiting); let route = RouteSupervisionState { next_challenge_at_unix_millis: Some(t), ..RouteSupervisionState::default() @@ -15763,16 +20919,30 @@ mod tests { meters.continuity_restarts = vec![now - 9_000, now - 4_000]; meters.continuity_deferred_until = Some(now + 500); meters.continuity_deferral_reason = Some("projection would not demote".into()); + meters.continuity_attempt_streak = 2; + meters.continuity_backoff_from_unix_millis = Some(now - 4_000); let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; generation.last_error = Some("held".into()); + generation.admitted_at_unix_millis = 1; let lines = render_supervision("service", Some(&generation), Some(&meters), now).join("\n"); assert!(lines.contains(&format!("target service {}", generation.generation_id)), "{lines}"); assert!(lines.contains("held: held"), "{lines}"); assert!(lines.contains(&format!( - "continuity restarts 2/6 next-restart-at {}", + "continuity attempts-in-hour 2 pages-at 6 streak 2 next-attempt-at {}", now - 4_000 + 10_000 )), "{lines}"); + assert!(!lines.contains("rematerialized-at"), "{lines}"); + meters.continuity_rematerialized_at_unix_millis = Some(now - 1_000); + let lines = render_supervision("service", Some(&generation), Some(&meters), now).join(" +"); + assert!(lines.contains(&format!("continuity rematerialized-at {}", now - 1_000)), "{lines}"); + // An anchor older than the generation is not shown as a pending wait. + generation.admitted_at_unix_millis = now; + let lines = render_supervision("service", Some(&generation), Some(&meters), now).join(" +"); + assert!(lines.contains("streak 2 next-attempt-at none"), "{lines}"); + generation.admitted_at_unix_millis = 1; assert!(lines.contains("continuity deferred until 10000500: projection would not demote"), "{lines}"); assert!(lines.contains(&format!( "route actuations 12/12 in window, reopens-at {}", @@ -18180,7 +23350,7 @@ mod tests { // it first: the write ends the pass, and nothing is minted from // entries that only look recent because the clock stepped back. let mut restarts = TargetSupervision::new("service"); - restarts.continuity_restarts = vec![ahead; CONTINUITY_RESTART_ATTEMPTS]; + restarts.continuity_restarts = vec![ahead; CONTINUITY_PAGING_RESTARTS]; set_meters(&routed.world, &restarts)?; routed.workload.kill(); assert!(routed.tick()?); @@ -18355,12 +23525,12 @@ mod tests { &routed.world, &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, )?; - for _ in 0..2 * CONTINUITY_RESTART_ATTEMPTS { + for _ in 0..2 * CONTINUITY_PAGING_RESTARTS { routed.world.engine.charge_route_actuation("service", RouteActuation::Survival)?; } let spent = meters_of(&routed.world, "service")?; - assert!(spent.restarts_exhausted(now_millis()?)); - assert_eq!(spent.route_survivals.len(), 2 * CONTINUITY_RESTART_ATTEMPTS); + assert!(spent.restarts_page(now_millis()?)); + assert_eq!(spent.route_survivals.len(), 2 * CONTINUITY_PAGING_RESTARTS); // Warming's preflight and Routing's install are both admitted. routed.step()?; @@ -18387,6 +23557,18 @@ mod tests { Ok(()) } + /// Scheduler passes until `done`, at most twelve. + #[cfg(unix)] + fn drive_until_resuming(&self, done: impl Fn(&DeploymentTransaction) -> bool) -> Result<()> { + for _ in 0..12 { + if done(&self.transaction()?) { + return Ok(()); + } + self.resume()?; + } + bail!("the transaction never reached the awaited state: {:?}", self.transaction()?.last_error) + } + fn resume_until_terminal(&self) -> Result { for _ in 0..12 { if latest(&self.world)?.completion.is_some() { @@ -18412,11 +23594,12 @@ mod tests { /// Soul's F1 probe, committed. A continuity's route changes are /// Survival, so no ceiling refuses its install and rollback while its - /// route proof keeps failing. The Routing deadline ends it: no attempt - /// runs after the deadline. + /// route proof keeps failing. Its Routing deadline does not end it: the + /// continuity pages the operator and retries forward, never stopping its + /// candidate (ruling post-fence-forward-retry). #[cfg(unix)] #[test] - fn a_continuity_whose_route_proof_fails_ends_at_its_routing_deadline() -> Result<()> { + fn a_continuity_whose_route_proof_fails_retries_forward_past_its_routing_deadline_and_pages() -> Result<()> { let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; routed.stub.set_state("active"); routed.run_to_routing()?; @@ -18434,36 +23617,46 @@ mod tests { assert!(waiting.last_error.is_some()); assert_eq!(waiting.phase_deadline, Some(deadline), "a resume moved the deadline"); + let stops = routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst); pass_deadline(&routed.world)?; - routed.resume()?; - let abort = routed - .transaction()? - .post_fencing_abort - .context("the passed deadline did not end the phase")?; - assert!(abort.error.contains("Routing phase passed its deadline"), "{}", abort.error); - assert!(!abort.error.contains("lease-not-adopted"), "{}", abort.error); - let finished = routed.resume_until_terminal()?; - assert!( - matches!( - finished.completion, - Some(TransactionCompletion::FailedAfterFencing { - recovery: TerminalRecovery::RestoreIncumbent, - .. - }) - ), - "{:?}", - finished.completion + for _ in 0..5 { + routed.resume()?; + } + // Past the deadline the continuity is not aborted: aborting it + // would stop the only running copy of the admitted body. It is left + // to the operator, and its Routing step keeps running. + let left = routed.transaction()?; + assert!(left.post_fencing_abort.is_none(), "the passed deadline aborted a continuity"); + assert_eq!(left.phase, DeploymentPhase::Routing); + assert!(left.completion.is_none()); + assert_eq!( + routed.world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?, + [left.transaction_id.clone()] ); - // After the deadline no attempt runs; the withdrawal writes and - // reloads once, whatever the rolled-back disk shows. - assert_eq!(routed.reloads(), 7, "an attempt ran after the deadline"); + assert_eq!(routed.world.engine.incidents().read()?.len(), 1, "one page per episode"); + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops, "the candidate was stopped"); + assert!(routed.reloads() > 6, "no attempt ran after the deadline"); + + // The proof passes: the phase moves on, and the first pass after + // closes the incident. + routed.stub.hang_up.store(false, Ordering::SeqCst); + routed.drive_until_resuming(|t| t.phase == DeploymentPhase::Committing)?; + routed.resume()?; + assert!( + routed.world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?.is_empty(), + "the incident outlived the episode" + ); + let records = routed.world.engine.incidents().read()?; + assert_eq!(records.len(), 1); + assert_eq!(records[0].close_reason, Some(CloseReason::Recovered)); + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); Ok(()) } /// The live shape of F1. A continuity reuses the admitted Expected, so /// its membership is the admitted fragment already on disk: while its /// proof fails, the install and the rollback write nothing and reload - /// nothing, and neither does the withdrawal after its deadline. + /// nothing, and neither does its forward retry after its deadline. #[cfg(unix)] #[test] fn a_continuity_over_its_admitted_route_reloads_nothing_while_its_proof_fails() @@ -18492,8 +23685,11 @@ mod tests { assert_eq!(routed.world.route_stubs.count(""), 0, "a failing continuity ran a host program"); pass_deadline(&routed.world)?; - routed.resume_until_terminal()?; - assert_eq!(routed.world.route_stubs.count(""), 0, "its withdrawal ran a host program"); + for _ in 0..5 { + routed.resume()?; + } + assert!(routed.transaction()?.post_fencing_abort.is_none(), "a passed deadline aborted it"); + assert_eq!(routed.world.route_stubs.count(""), 0, "its retry ran a host program"); assert_eq!(std::fs::read(routed.fragment_path())?, admitted, "the admitted route moved"); Ok(()) } @@ -18573,19 +23769,56 @@ mod tests { Ok(()) } + /// A candidate no one ever answers for is ended by its Warming + /// deadline, or by the operator's expiry: it owns only itself. + #[test] + fn a_candidate_that_never_correlates_aborts_before_fencing_at_its_warming_deadline() -> Result<()> { + for by_operator in [false, true] { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + routed.stub.hang_up.store(true, Ordering::SeqCst); + for _ in 0..3 { + routed.resume()?; + } + let waiting = routed.transaction()?; + assert!(waiting.pre_fencing_abort.is_none(), "it ended before its deadline"); + assert!(waiting.phase_deadline.is_some()); + assert_eq!(waiting.phase, DeploymentPhase::Warming); + + let cause = if by_operator { + expire(&routed.world.state_store, "continuity-service", "operator")?; + "expired by the operator" + } else { + pass_deadline(&routed.world)?; + "Warming phase passed its deadline" + }; + routed.resume()?; + let abort = routed.transaction()?.pre_fencing_abort.context("it was not aborted")?; + assert!(abort.error.contains(cause), "{}", abort.error); + let ended = routed.resume_until_terminal()?; + assert!(matches!(ended.completion, Some(TransactionCompletion::FailedBeforeFencing { .. }))); + } + Ok(()) + } + /// `idunn expire` writes a request and nothing else; the resolver ends /// the phase from it, and the spent request is deleted. #[test] fn an_operator_expiry_ends_the_phase_through_the_resolver() -> Result<()> { let early = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + // Warming is a deadline phase: the operator can expire it, and + // the pre-fencing abort then ends it. Once aborting it cannot be + // expired again. + expire(&early.world.state_store, "continuity-service", "operator")?; + early.resume()?; + assert!(early.transaction()?.pre_fencing_abort.is_some(), "the expiry did not end Warming"); let refused = expire(&early.world.state_store, "continuity-service", "operator") - .expect_err("a transaction before the fence was expired"); + .expect_err("an aborting transaction was expired"); assert!(format!("{refused:#}").contains("idunn cancel"), "{refused:#}"); - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; let untouched = resident(&routed.world)?.envelope; - expire(&routed.world.state_store, "continuity-service", "operator")?; - assert!(expire(&routed.world.state_store, "continuity-service", "operator").is_err()); + expire(&routed.world.state_store, "up-service", "operator")?; + assert!(expire(&routed.world.state_store, "up-service", "operator").is_err()); let snapshot = ControlSnapshot::read(&routed.world.state_store)?; assert_eq!(snapshot.expiry_requests.len(), 1); assert_eq!(resident(&routed.world)?.envelope, untouched, "the CLI wrote the transaction"); @@ -18601,6 +23834,36 @@ mod tests { ControlSnapshot::read(&routed.world.state_store)?.expiry_requests.is_empty(), "the spent request stayed" ); + + // A continuity past Fencing retries forward: expiring it would + // abort the only running copy of its body. The CLI refuses, and a + // request a pre-c2c binary stored for one is retired as spent. + let continuity = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let refused = expire(&continuity.world.state_store, "continuity-service", "operator") + .expect_err("a continuity past Fencing was expired"); + assert!(format!("{refused:#}").contains("retries forward"), "{refused:#}"); + assert!(ControlSnapshot::read(&continuity.world.state_store)?.expiry_requests.is_empty()); + let now = now_millis()?; + let stored = ExpiryRequest { + schema_version: EXPIRY_REQUEST_SCHEMA.into(), + command_id: "continuity-service".into(), + requested_by: "operator".into(), + requested_at_unix_millis: now, + }; + assert!(SingleFileMessagePackBackingStore::new(&continuity.world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: ExpiryRequest::TYPE.into(), + key: stored.command_id.clone(), + current: None, + }], + &[expiry_request_envelope(&stored, now)?], + )?); + continuity.resume()?; + assert!( + ControlSnapshot::read(&continuity.world.state_store)?.expiry_requests.is_empty(), + "the request for a continuity past Fencing stayed" + ); + assert!(continuity.transaction()?.post_fencing_abort.is_none(), "the request aborted a continuity"); Ok(()) } @@ -18632,7 +23895,7 @@ mod tests { assert!(format!("{refused:#}").contains("no live transaction"), "{refused:#}"); // Another target's deployment in flight is not this command's. - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; let now = now_millis()?; let other = DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), @@ -18658,10 +23921,10 @@ mod tests { ], &[command_envelope(&other, now)?, transaction_envelope(&sealing, now)?], )?); - expire(&routed.world.state_store, "continuity-service", "operator")?; + expire(&routed.world.state_store, "up-service", "operator")?; let requests = ControlSnapshot::read(&routed.world.state_store)?.expiry_requests; assert_eq!(requests.len(), 1); - assert_eq!(requests[0].value.command_id, "continuity-service"); + assert_eq!(requests[0].value.command_id, "up-service"); Ok(()) } @@ -18669,7 +23932,7 @@ mod tests { /// live transactions, a request for the other leaves this one running. #[test] fn an_expiry_request_ends_only_its_own_commands_phase() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; let now = now_millis()?; let other = DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), @@ -18682,8 +23945,8 @@ mod tests { let sealing = DeploymentTransaction::new(&other, "other".into(), 0, None, now)?; // The request names a command whose id has this transaction's // length and first bytes, and differs only in its last. - let near = "continuity-servicx"; - assert_eq!(near.len(), "continuity-service".len()); + let near = "up-servicx"; + assert_eq!(near.len(), "up-service".len()); let request = ExpiryRequest { schema_version: EXPIRY_REQUEST_SCHEMA.into(), command_id: near.into(), @@ -18693,11 +23956,11 @@ mod tests { // And commands whose ids are a strict prefix of this one's, and // extend it: a match is the whole id, in neither direction a part. let prefix = ExpiryRequest { - command_id: "continuity-servic".into(), + command_id: "up-servic".into(), ..request.clone() }; let extension = ExpiryRequest { - command_id: "continuity-service-2".into(), + command_id: "up-service-2".into(), ..request.clone() }; let store = SingleFileMessagePackBackingStore::new(&routed.world.state_store); @@ -18728,24 +23991,26 @@ mod tests { let mine = snapshot .transactions .iter() - .find(|t| t.value.command_id == "continuity-service") + .find(|t| t.value.command_id == "up-service") .context("no continuity")?; assert_eq!(routed.world.engine.resolve_phase_deadline(mine)?, None); let after = ControlSnapshot::read(&routed.world.state_store)?; let unchanged = after .transactions .iter() - .find(|t| t.value.command_id == "continuity-service") + .find(|t| t.value.command_id == "up-service") .context("no continuity")?; assert_eq!(unchanged.envelope, mine.envelope, "another command's request ended this phase"); Ok(()) } - /// Q3 at the deadline, through the Engine. A continuity whose candidate - /// was granted a lease and never proved it held it fails as - /// `lease-not-adopted`; one whose Ready proved it restarts the admitted - /// release. A deployment whose candidate holds a lease is left to the - /// operator: nothing is aborted and its step still runs. + /// The deadline of a stateful target, through the Engine. A deployment + /// whose candidate holds a lease is left to the operator: nothing is + /// aborted and its step still runs. So is a continuity, adopted or not, + /// whose candidate is the only running copy of the admitted body. An + /// abort begun on a continuity directly still names its recovery: it + /// restarts the admitted release when its Ready proved the lease, and + /// restores the incumbent when it never did. #[test] fn a_stateful_deadline_resolves_by_lease_adoption() -> Result<()> { let granted = |routed: &RoutedWorld| -> Result { @@ -18759,13 +24024,29 @@ mod tests { .to_owned()) }; + // Past its deadline a continuity is not aborted, adopted or not: + // its candidate is the only running copy of the admitted body. It + // is left to the operator and its step runs. + let open = |routed: &RoutedWorld| -> Result> { + routed.world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired) + }; let unadopted = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; granted(&unadopted)?; pass_deadline(&unadopted.world)?; - unadopted.resume()?; - let abort = unadopted.transaction()?.post_fencing_abort.context("not ended")?; - assert!(abort.error.starts_with("lease-not-adopted: "), "{}", abort.error); - assert!(abort.error.contains("AwaitingReady phase passed its deadline"), "{}", abort.error); + for _ in 0..3 { + unadopted.resume()?; + } + let held = unadopted.transaction()?; + assert!(held.post_fencing_abort.is_none(), "an unadopted continuity was aborted"); + assert_eq!(held.phase, DeploymentPhase::AwaitingReady); + assert_eq!(held.last_error.as_deref(), Some("candidate is still warming"), "its step did not run"); + assert_eq!(open(&unadopted)?, [held.transaction_id.clone()]); + // An abort begun on it directly (a candidate found dead) still + // restores the incumbent when the lease was never adopted. + unadopted + .world + .engine + .begin_post_fencing_abort(&resident(&unadopted.world)?, anyhow!("test abort"))?; let finished = unadopted.resume_until_terminal()?; assert!(matches!( finished.completion, @@ -18789,10 +24070,19 @@ mod tests { assert_eq!(adoption.signed_presence_sha256, evidence.canonical_sha256); assert_eq!(adoption.source, AdoptionSource::Direct); adopted.drive_until(|t| t.phase == DeploymentPhase::Routing)?; + adopted.stub.hang_up.store(true, Ordering::SeqCst); pass_deadline(&adopted.world)?; - adopted.resume()?; - let abort = adopted.transaction()?.post_fencing_abort.context("not ended")?; - assert!(!abort.error.contains("lease-not-adopted"), "{}", abort.error); + for _ in 0..3 { + adopted.resume()?; + } + let held = adopted.transaction()?; + assert!(held.post_fencing_abort.is_none(), "an adopted continuity was aborted"); + assert_eq!(held.phase, DeploymentPhase::Routing); + assert_eq!(open(&adopted)?, [held.transaction_id.clone()]); + adopted + .world + .engine + .begin_post_fencing_abort(&resident(&adopted.world)?, anyhow!("test abort"))?; let finished = adopted.resume_until_terminal()?; assert!( matches!( @@ -18820,8 +24110,71 @@ mod tests { Ok(()) } + /// The incident of a stateful deployment left to the operator is one + /// per episode. An operator's expiry before the deadline reports the + /// hold and pages no one; the deadline opens one incident; and the + /// incident closes as soon as the record is no longer left, while the + /// transaction goes on. + #[test] + fn a_stateful_deploys_incident_is_one_per_episode_and_follows_its_deadline() -> Result<()> { + let deploy = + build_routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false, true, CommandKind::Deploy)?; + deploy.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let held = deploy.transaction()?; + let id = held.transaction_id.as_str(); + let open = || deploy.world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired); + let now = now_millis()?; + let request = ExpiryRequest { + schema_version: EXPIRY_REQUEST_SCHEMA.into(), + command_id: held.command_id.clone(), + requested_by: "operator".into(), + requested_at_unix_millis: now, + }; + assert!(SingleFileMessagePackBackingStore::new(&deploy.world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: ExpiryRequest::TYPE.into(), + key: request.command_id.clone(), + current: None, + }], + &[expiry_request_envelope(&request, now)?], + )?); + + // The operator asked, the deadline has not passed: nobody is paged, + // however many passes see the request. + for _ in 0..4 { + deploy.resume()?; + } + assert!(open()?.is_empty(), "an expiry before the deadline paged the operator"); + assert!(deploy.world.engine.incidents().read()?.is_empty()); + assert!(deploy.transaction()?.post_fencing_abort.is_none(), "a leased deployment was aborted"); + + // The deadline passes: one incident, not one per pass. + pass_deadline(&deploy.world)?; + for _ in 0..4 { + deploy.resume()?; + } + assert_eq!(open()?, [id]); + assert_eq!(deploy.world.engine.incidents().read()?.len(), 1); + + // The record goes on with a deadline in the future (its step + // advanced): the incident closes while the transaction lives. + let current = resident(&deploy.world)?; + let mut next = current.value.clone(); + let due = next.phase_deadline.as_mut().context("no deadline")?; + due.entered_at_unix_millis = now_millis()?; + due.deadline_at_unix_millis = due.entered_at_unix_millis + 3_600_000; + replace_transaction(&deploy.world.state_store, ¤t, &next)?; + deploy.resume()?; + assert!(!deploy.transaction()?.is_terminal()); + assert!(open()?.is_empty(), "the incident outlived the episode"); + let records = deploy.world.engine.incidents().read()?; + assert_eq!(records.len(), 1); + assert_eq!(records[0].close_reason, Some(CloseReason::Recovered)); + Ok(()) + } + /// B5 Soul A. The physical grant lands before `Granted` is durable. A - /// continuity that dies in that window and passes its deadline is + /// continuity that dies in that window and whose abort begins is /// aborted from `Prepared`; the abort still withdraws the lease on /// disk, so the next fencing of the target is not refused. #[test] @@ -18841,9 +24194,12 @@ mod tests { ); assert!(driver.observe_exact(&lease)?, "the grant is not on disk"); - pass_deadline(&routed.world)?; + routed + .world + .engine + .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("test abort"))?; routed.resume()?; - assert!(routed.transaction()?.post_fencing_abort.is_some(), "the deadline did not end it"); + assert!(routed.transaction()?.post_fencing_abort.is_some(), "the abort did not begin"); routed.resume_until_terminal()?; assert!(driver.observe_empty()?, "the abort left the granted lease on disk"); // What the next fencing does to a candidate's lease path. @@ -18851,6 +24207,486 @@ mod tests { Ok(()) } + /// ghostlight-abort-lease-before-stop. A candidate that holds its + /// lifetime write lease keeps the lease's shared lock until it is + /// stopped, so the abort stops it before revoking; revoking first + /// fails on the lock on every attempt and the abort never ends. + #[cfg(unix)] + #[test] + fn an_abort_stops_a_candidate_that_holds_its_lease_before_revoking_it() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| matches!(t.leasing, Some(LeasingEvidence::Granted { .. })))?; + let lease_path = routed.world.root.join("lease/process-write-lease.cc"); + routed.workload.hold_lease_lock(&lease_path); + let driver = CultCacheWriteLeaseDriver::new("service", &lease_path); + + routed + .world + .engine + .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("test abort"))?; + routed.resume()?; + assert!(routed.transaction()?.post_fencing_abort.is_some(), "the abort did not begin"); + let stops = routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst); + let finished = routed.resume_until_terminal()?; + assert!( + matches!(finished.completion, Some(TransactionCompletion::FailedAfterFencing { .. })), + "{:?}", + finished.last_error + ); + assert!(driver.observe_empty()?, "the abort left the lease on disk"); + assert!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst) > stops, "the candidate was never stopped"); + Ok(()) + } + + /// The deadline's abort is a continuity's no longer, and a deployment's + /// still: a deployment's candidate is a new release and its incumbent + /// the admitted body, which an abort restores. + #[cfg(unix)] + #[test] + fn a_deploy_with_a_live_incumbent_still_aborts_to_it_at_its_deadline() -> Result<()> { + let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.stub.hang_up.store(true, Ordering::SeqCst); + routed.resume()?; + assert!(routed.transaction()?.post_fencing_abort.is_none(), "aborted before its deadline"); + + pass_deadline(&routed.world)?; + routed.resume()?; + let abort = routed + .transaction()? + .post_fencing_abort + .context("the passed deadline did not end a deployment")?; + assert!(abort.error.contains("Routing phase passed its deadline"), "{}", abort.error); + assert!( + routed.world.engine.return_wedged_continuity_aborts_forward()?.is_empty(), + "the boot lift took a deployment's abort" + ); + assert!(routed.transaction()?.post_fencing_abort.is_some()); + let finished = routed.resume_until_terminal()?; + assert!( + matches!( + finished.completion, + Some(TransactionCompletion::FailedAfterFencing { + recovery: TerminalRecovery::RestoreIncumbent, + .. + }) + ), + "{:?}", + finished.completion + ); + Ok(()) + } + + /// A record left to the operator whose step returns Ok without moving it + /// (a gate-wait on a silent candidate) is backed off like a failed + /// attempt, to the continuity cap: its candidate is challenged once per + /// backoff, not once per pass. Before its deadline every pass challenges. + #[test] + fn a_continuity_left_to_the_operator_challenges_a_silent_candidate_once_per_backoff() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let engine = &routed.world.engine; + let id = routed.transaction()?.transaction_id; + let hits = || routed.stub.candidate_hits.load(Ordering::SeqCst); + + let before = hits(); + for _ in 0..3 { + engine.resume_one_transaction()?; + } + assert!(hits() >= before + 3, "a pass inside the deadline did not challenge"); + assert!(engine.resume_backoff.lock().unwrap().get(&id).is_none()); + + pass_deadline(&routed.world)?; + engine.resume_one_transaction()?; + let first = hits(); + assert!(first > before + 3, "the pass past the deadline did not challenge"); + let armed = *engine.resume_backoff.lock().unwrap().get(&id).context("the unmoved step set no backoff")?; + assert_eq!(armed.ceiling_millis, CONTINUITY_BACKOFF_CEILING_MILLIS); + for _ in 0..5 { + engine.resume_one_transaction()?; + } + assert_eq!(hits(), first, "a wedged candidate was challenged inside its backoff"); + assert!(routed.transaction()?.post_fencing_abort.is_none()); + + // The skip check reads the ceiling the backoff was armed with: a + // wait longer than the short cap is still a wait. + engine.resume_backoff.lock().unwrap().get_mut(&id).unwrap().not_before_unix_millis = + now_millis()? + 200_000; + engine.resume_one_transaction()?; + assert_eq!(hits(), first, "the skip check read the short cap"); + Ok(()) + } + + /// What a pre-c2c Idunn left when a continuity's phase passed its + /// deadline: the abort begun, route restoration done, the lease + /// withdrawal and candidate cleanup still owed, and the deadline stale. + #[cfg(unix)] + fn wedge_abort(routed: &RoutedWorld, error: &str) -> Result<()> { + routed + .world + .engine + .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("{error}"))?; + routed.resume()?; + let aborting = routed.transaction()?.post_fencing_abort.context("no abort")?; + assert_eq!(aborting.route_restoration, CleanupEvidence::Complete); + assert_eq!(aborting.candidate_cleanup, CleanupEvidence::Pending); + pass_deadline(&routed.world) + } + + #[cfg(unix)] + fn lease_of(routed: &RoutedWorld) -> Result<(IdunnProcessWriteLeaseRecord, String)> { + let leasing = routed.transaction()?.leasing.context("no leasing evidence")?; + Ok(( + leasing.issued_lease().context("no issued lease")?.clone(), + leasing.lease_sha256().context("no granted lease")?.to_owned(), + )) + } + + #[cfg(unix)] + fn lease_driver(routed: &RoutedWorld) -> (CultCacheWriteLeaseDriver, PathBuf) { + let path = routed.world.root.join("lease/process-write-lease.cc"); + (CultCacheWriteLeaseDriver::new("service", &path), path) + } + + /// The live shape of tx-4eaed669 (Odin): Routing, lease granted and + /// adopted, the proof failing, the abort begun at the Routing deadline. + /// A pre-c2c Idunn went on to stop the candidate and the target lost + /// its only running copy. The boot lift returns the record to forward + /// retry; nothing is stopped, and the proof, once it passes, commits. + #[cfg(unix)] + #[test] + fn a_wedged_odin_shaped_continuity_abort_is_returned_to_forward_retry_at_boot() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let (lease, lease_sha256) = lease_of(&routed)?; + routed.stub.set_state("active"); + *routed.stub.lease.lock().unwrap() = Some(lease_sha256); + routed.drive_until(|t| t.phase == DeploymentPhase::Routing)?; + assert!(routed.transaction()?.lease_adoption.is_some()); + let (driver, path) = lease_driver(&routed); + routed.workload.hold_lease_lock(&path); + routed.stub.hang_up.store(true, Ordering::SeqCst); + wedge_abort( + &routed, + "the Routing phase passed its deadline at unix ms 1792324133638", + )?; + let stops = routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst); + + let lifted_at = now_millis()?; + let returned = routed.world.engine.return_wedged_continuity_aborts_forward()?; + for _ in 0..10 { + routed.resume()?; + } + assert_eq!( + routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), + stops, + "the candidate was stopped" + ); + let retrying = routed.transaction()?; + assert_eq!(returned, [retrying.transaction_id.clone()]); + assert!(driver.observe_exact(&lease)?, "the lease left the disk"); + assert!(retrying.post_fencing_abort.is_none(), "the abort came back"); + assert!(retrying.completion.is_none()); + assert_eq!(retrying.phase, DeploymentPhase::Routing); + let deadline = retrying.phase_deadline.context("no deadline")?; + assert!(deadline.entered_at_unix_millis >= lifted_at, "the stale deadline stayed"); + assert!(routed.world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?.is_empty()); + + routed.stub.hang_up.store(false, Ordering::SeqCst); + routed.drive_until_resuming(|t| t.phase == DeploymentPhase::Committing)?; + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); + assert!(routed.fragment_path().exists(), "the route membership was not re-installed"); + Ok(()) + } + + /// The live shape of tx-039b91e9 (ghostlight): AwaitingReady with the + /// lease granted and not adopted, aborted at the readiness deadline. + /// Returned to forward retry, the candidate that later answers is + /// adopted and the record enters Routing. + #[cfg(unix)] + #[test] + fn a_wedged_ghostlight_shaped_continuity_abort_is_returned_to_forward_retry_at_boot() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let (lease, lease_sha256) = lease_of(&routed)?; + let (driver, path) = lease_driver(&routed); + routed.workload.hold_lease_lock(&path); + wedge_abort( + &routed, + "lease-not-adopted: the AwaitingReady phase passed its deadline at unix ms 1792324133638", + )?; + let stops = routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst); + + let returned = routed.world.engine.return_wedged_continuity_aborts_forward()?; + for _ in 0..5 { + routed.resume()?; + } + let waiting = routed.transaction()?; + assert_eq!(returned, [waiting.transaction_id.clone()]); + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops, "the candidate was stopped"); + assert!(driver.observe_exact(&lease)?, "the lease left the disk"); + assert!(waiting.post_fencing_abort.is_none() && waiting.completion.is_none()); + assert_eq!(waiting.phase, DeploymentPhase::AwaitingReady); + assert!(waiting.lease_adoption.is_none()); + + routed.stub.set_state("active"); + *routed.stub.lease.lock().unwrap() = Some(lease_sha256.clone()); + routed.drive_until_resuming(|t| t.phase == DeploymentPhase::Routing)?; + let adopted = routed.transaction()?; + assert_eq!(adopted.lease_adoption.context("no adoption recorded")?.write_lease_sha256, lease_sha256); + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); + Ok(()) + } + + /// NEGATIVE. The boot lift runs before anything it needs may answer: a + /// systemctl error, a host-actuator hub no actuator has reached. The + /// abort of a candidate it could not observe is held, not run (its first + /// step stops the candidate), and the lift is retried on later passes. + /// Once observation answers Ok the record is lifted; once it answers + /// dead the abort proceeds. + #[cfg(unix)] + #[test] + fn a_boot_lift_that_cannot_observe_holds_the_abort_and_retries() -> Result<()> { + for hub_unattached in [false, true] { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let (lease, lease_sha256) = lease_of(&routed)?; + routed.stub.set_state("active"); + *routed.stub.lease.lock().unwrap() = Some(lease_sha256); + routed.drive_until(|t| t.phase == DeploymentPhase::Routing)?; + let (driver, path) = lease_driver(&routed); + routed.workload.hold_lease_lock(&path); + routed.stub.hang_up.store(true, Ordering::SeqCst); + wedge_abort(&routed, "the Routing phase passed its deadline at unix ms 1792324133638")?; + let stops = routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst); + let observation_fails = || { + if hub_unattached { + routed.workload.silence(true); + } else { + routed.workload.kill(); + } + }; + + // Boot: observation errs. Nothing is lifted and nothing stops. + observation_fails(); + let returned = routed.world.engine.return_wedged_continuity_aborts_forward()?; + assert!(returned.is_empty(), "lifted {returned:?} it could not observe"); + for _ in 0..5 { + routed.resume()?; + assert_eq!( + routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), + stops, + "an undecided abort stopped the candidate (hub_unattached {hub_unattached})" + ); + } + let held = routed.transaction()?; + let abort = held.post_fencing_abort.context("the abort was dropped")?; + assert_eq!(abort.lease_withdrawal, CleanupEvidence::Pending); + assert!(driver.observe_exact(&lease)?, "the lease left the disk"); + + // Observation answers Ok: the next pass lifts the record. + routed.workload.silence(false); + routed.workload.revive(); + routed.resume()?; + let lifted = routed.transaction()?; + assert!(lifted.post_fencing_abort.is_none(), "a later pass did not lift the record"); + assert_eq!(lifted.phase, DeploymentPhase::Routing); + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); + } + + // Observation answers dead: the abort proceeds to its end. + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + wedge_abort(&routed, "the AwaitingReady phase passed its deadline")?; + routed.workload.silence(true); + assert!(routed.world.engine.return_wedged_continuity_aborts_forward()?.is_empty()); + routed.resume()?; + assert!(routed.transaction()?.post_fencing_abort.is_some_and(|a| a.candidate_cleanup == CleanupEvidence::Pending)); + routed.workload.silence(false); + routed.workload.bury(); + let finished = routed.resume_until_terminal()?; + assert!( + matches!(finished.completion, Some(TransactionCompletion::FailedAfterFencing { .. })), + "{:?}", + finished.completion + ); + Ok(()) + } + + /// NEGATIVE. The live shape of tx-4eaed669 (Odin) left to the operator + /// at its Routing deadline, with a Deploy now awaiting the handover of + /// its target. The continuity yields through its own abort: the lease + /// leaves the disk only after the stop, the target is free, and the + /// Deploy is unended. (The Fencing half is not asserted here: the seeded + /// Deploy plans a stateless target, so its fence skips the lease path.) + #[cfg(unix)] + #[test] + fn an_odin_shaped_continuity_left_to_the_operator_yields_its_lease_to_a_waiting_deploy() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let (lease, lease_sha256) = lease_of(&routed)?; + routed.stub.set_state("active"); + *routed.stub.lease.lock().unwrap() = Some(lease_sha256); + routed.drive_until(|t| t.phase == DeploymentPhase::Routing)?; + assert!(routed.transaction()?.lease_adoption.is_some()); + let (driver, path) = lease_driver(&routed); + routed.workload.hold_lease_lock(&path); + routed.stub.hang_up.store(true, Ordering::SeqCst); + let owner_id = routed.transaction()?.transaction_id; + pass_deadline(&routed.world)?; + routed.resume()?; + let left = routed.transaction()?; + assert!(left.post_fencing_abort.is_none(), "the continuity aborted with nothing awaiting the target"); + assert_eq!(routed.world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?, [owner_id.as_str()]); + assert!(driver.observe_exact(&lease)?); + + let deploy = seeded_transaction(&routed.world, DeploymentPhase::Warming, CommandKind::Deploy, None)?; + let stops = routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst); + for _ in 0..12 { + routed.resume()?; + let owner = latest_id(&routed.world, &owner_id)?; + if owner.post_fencing_abort.is_none() { + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); + assert!(driver.observe_exact(&lease)?, "the lease left before the yield"); + } + if owner.completion.is_some() { + break; + } + } + let ended = latest_id(&routed.world, &owner_id)?; + assert!( + matches!( + ended.completion, + Some(TransactionCompletion::FailedAfterFencing { recovery: TerminalRecovery::RestartAdmitted, .. }) + ), + "{:?}", + ended.completion + ); + assert!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst) > stops); + assert!(!driver.observe_exact(&lease)?, "the lease stayed on disk"); + assert!(ControlSnapshot::read(&routed.world.state_store)?.target_owner("service", None).is_none()); + let waiting = resident_id(&routed.world, &deploy.transaction_id)?.value; + assert!(waiting.completion.is_none() && waiting.pre_fencing_abort.is_none()); + Ok(()) + } + + /// The lift is a function of its inputs: each case lacks one thing the + /// lift requires, and its abort stays and runs to its end. + #[cfg(unix)] + #[test] + fn the_boot_lift_leaves_every_abort_it_cannot_prove_harmless() -> Result<()> { + #[derive(Clone, Copy, Debug)] + enum Case { + CandidateNotRunning, + LeaseWithdrawalRecordedComplete, + LeaseWithdrawn, + CandidateCleaned, + LeaseRevokedUnderIt, + DeployWithRunningCandidate, + CompletedRecord, + DeployAwaitsTheTarget, + } + for case in [ + Case::CandidateNotRunning, + Case::LeaseWithdrawalRecordedComplete, + Case::LeaseWithdrawn, + Case::CandidateCleaned, + Case::LeaseRevokedUnderIt, + Case::DeployWithRunningCandidate, + Case::CompletedRecord, + Case::DeployAwaitsTheTarget, + ] { + let deploy = matches!(case, Case::DeployWithRunningCandidate); + // The cleanup comes after the lease withdrawal, so a case with + // cleanup done and the withdrawal not is a stateless target's. + let stateful = !matches!(case, Case::CandidateCleaned); + let routed = build_routed_world( + Odin::Unreachable, + 1, + DeploymentPhase::Fencing, + false, + stateful, + if deploy { CommandKind::Deploy } else { CommandKind::Continuity }, + )?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + routed.world.engine.begin_post_fencing_abort( + &resident(&routed.world)?, + anyhow!("the AwaitingReady phase passed its deadline"), + )?; + // Route restoration; then each case takes the abort one real + // step, or one input, away from the lift's predicate. + routed.resume()?; + match case { + Case::CandidateNotRunning => routed.workload.bury(), + Case::LeaseWithdrawn | Case::CandidateCleaned => routed.resume()?, + Case::LeaseRevokedUnderIt => { + let (lease, _) = lease_of(&routed)?; + lease_driver(&routed).0.revoke_exact(Some(&lease))?; + } + Case::DeployWithRunningCandidate => {} + // The record says the lease was withdrawn while the candidate + // still runs and the lease is still exact on disk: only the + // record's own evidence keeps the lift off it. + Case::LeaseWithdrawalRecordedComplete => { + let stored = resident(&routed.world)?; + let mut next = stored.value.clone(); + next.post_fencing_abort + .as_mut() + .context("no abort")? + .lease_withdrawal = CleanupEvidence::Complete; + replace_transaction(&routed.world.state_store, &stored, &next)?; + } + // The abort of the Odin-shaped negative, begun to yield: a + // Deploy awaits the handover of its target. + Case::DeployAwaitsTheTarget => { + seeded_transaction(&routed.world, DeploymentPhase::Warming, CommandKind::Deploy, None)?; + } + Case::CompletedRecord => { + routed.resume_until_terminal()?; + } + } + let owner_id = the_continuity(&routed.world).ok().map(|c| c.value.transaction_id); + let abort = the_continuity(&routed.world).ok().and_then(|c| c.value.post_fencing_abort); + match case { + Case::LeaseWithdrawn => assert_eq!( + abort.as_ref().map(|a| (a.lease_withdrawal, a.candidate_cleanup)), + Some((CleanupEvidence::Complete, CleanupEvidence::Pending)) + ), + Case::CandidateCleaned => assert_eq!( + abort.as_ref().map(|a| (a.lease_withdrawal, a.candidate_cleanup)), + Some((CleanupEvidence::Skipped, CleanupEvidence::Complete)) + ), + _ => {} + } + let before = ControlSnapshot::read(&routed.world.state_store)?; + let returned = routed.world.engine.return_wedged_continuity_aborts_forward()?; + assert!(returned.is_empty(), "lifted {returned:?}"); + let after = ControlSnapshot::read(&routed.world.state_store)?; + assert_eq!( + before.transactions.iter().map(|t| &t.envelope).collect::>(), + after.transactions.iter().map(|t| &t.envelope).collect::>(), + "the lift wrote a record it left" + ); + if let (Case::DeployAwaitsTheTarget, Some(owner)) = (case, &owner_id) { + assert!(abort.is_some(), "the lift withdrew the yield"); + clear_other_live_transactions(&routed.world, owner)?; + } + if matches!(case, Case::LeaseWithdrawalRecordedComplete) { + // An edited record, not a reachable one: the lift's refusal + // is the whole claim, and its abort is not run to the end. + continue; + } + let finished = routed.resume_until_terminal().with_context(|| format!("{case:?}"))?; + assert!( + matches!(finished.completion, Some(TransactionCompletion::FailedAfterFencing { .. })), + "{:?}", + finished.completion + ); + } + Ok(()) + } + /// B5 Soul F. A stateful candidate Odin reports on: its Ready word /// carries the granted lease, and that Ready records the adoption, /// sourced from Odin's topology and naming the presence Odin observed. @@ -18981,4 +24817,435 @@ mod tests { Ok(()) } } + + // BEGIN runner-failure-evidence tests + // A failed runner step is named, its output tail is kept in + // runner-evidence.cc, and no output reaches an error, control.cc or the + // journal line. The runner is the real DockerRunnerDriver driving a fake + // docker program, through the engine's Sealing path. + + const RUNNER_CANARY: &str = "CANARY-7c41e0d9"; + + /// A source that freezes a fixed tree, so a deployment reaches the runner. + struct TreeSource { + recipe: String, + root: PathBuf, + } + + impl SourcePort for TreeSource { + fn resolve( + &self, + _: &OperatorBinding, + _: &str, + _: u64, + ) -> Result { + Ok(crate::drivers::ResolvedSource { + facts: crate::deployment_plan::tests::source(&self.recipe), + recipe_bytes: self.recipe.clone().into_bytes(), + }) + } + + fn freeze(&self, transaction_id: &str, plan: &CompiledDeploymentPlan) -> Result { + Ok(FrozenSourceReceipt { + transaction_id: transaction_id.to_owned(), + plan_id: plan.plan_id.clone(), + snapshot_sha256: format!("sha256-{}", "a".repeat(64)), + }) + } + + fn observe_frozen( + &self, + plan: &CompiledDeploymentPlan, + receipt: &FrozenSourceReceipt, + ) -> Result { + Ok(crate::drivers::FrozenSource::fixture( + receipt.clone(), + plan.source.clone(), + plan.recipe_blob.clone(), + self.root.clone(), + )) + } + + fn cleanup(&self, _: &str, _: Option<&FrozenSourceReceipt>) -> Result<()> { + Ok(()) + } + } + + /// The unrouted recipe with two steps ahead of its build, and no external + /// input: the second step, `test-daemon`, is the one the fake docker fails. + fn two_step_recipe() -> String { + let recipe = unrouted_recipe().replace( + "capability = \"service.runtime\"", + &format!("capability = \"{ODIN_RENDEZVOUS_CAPABILITY}\""), + ); + let (head, rest) = recipe + .split_once("[[external_inputs]]") + .expect("the recipe declares an external input"); + let (_, tail) = rest.split_once("[[artifacts]]").expect("the recipe declares artifacts"); + let head = head.replacen( + "[[steps]]\nid = \"build\"", + "[[steps]]\nid = \"install\"\nphase = \"prepare\"\nrunner = \"rust\"\nargv = [\"cargo\", \"fetch\"]\n\n\ + [[steps]]\nid = \"test-daemon\"\nphase = \"test\"\nrunner = \"rust\"\nargv = [\"cargo\", \"test\"]\n\n\ + [[steps]]\nid = \"build\"", + 1, + ); + format!("{head}[[artifacts]]{tail}") + } + + /// A Sealing world whose runner is the real driver over a fake docker: the + /// first call succeeds, the second prints the canary and an escape + /// sequence on both streams and exits 3. + #[cfg(unix)] + fn failing_runner_world() -> Result<(EngineFixture, TempDir)> { + use std::os::unix::fs::PermissionsExt; + let tools = TempDir::new()?; + let docker = tools.path().join("docker"); + std::fs::write( + &docker, + format!( + "#!/bin/sh\nn=$(cat \"$0.count\" 2>/dev/null || echo 0)\nn=$((n+1))\necho $n > \"$0.count\"\n\ + [ \"$n\" -lt 2 ] && exit 0\n\ + printf 'out {RUNNER_CANARY} \\033[2J\\n'\nprintf 'err {RUNNER_CANARY}\\n' >&2\nexit 3\n" + ), + )?; + std::fs::set_permissions(&docker, std::fs::Permissions::from_mode(0o755))?; + let recipe = two_step_recipe(); + let mut world = sealing_world(&recipe)?; + let tree = tools.path().join("tree"); + std::fs::create_dir_all(&tree)?; + std::fs::write(tree.join("README"), b"source")?; + world.engine.source = Arc::new(TreeSource { recipe, root: tree }); + world.engine.docker_runner = DockerRunnerDriver { docker_program: docker }; + Ok((world, tools)) + } + + /// Scheduler ticks until the deployment ends (or the budget is spent). + fn tick_until_ended(world: &EngineFixture) -> Result { + for _ in 0..24 { + world.engine.resume_backoff.lock().unwrap().clear(); + let _ = world.engine.run_scheduler_tick(); + if latest(world)?.completion.is_some() { + break; + } + } + latest(world) + } + + fn printed_status(world: &EngineFixture) -> Result { + let mut printed = Vec::new(); + status(&mut printed, &world.state_store, &world.engine.options.incident_store, Some("up-service"))?; + Ok(String::from_utf8(printed)?) + } + + fn canary_failure(stdout: &[u8]) -> RunnerStepFailure { + RunnerStepFailure { + subject: crate::drivers::RunnerSubject::Step("test-daemon".into()), + runner: "rust".into(), + exit_code: Some(3), + signal: None, + stdout_tail: stdout.to_vec(), + stdout_total_bytes: stdout.len() as u64, + stderr_tail: stdout.to_vec(), + stderr_total_bytes: stdout.len() as u64, + } + } + + #[cfg(unix)] + #[test] + fn a_failed_runner_step_is_named_kept_and_never_echoed() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + let (world, _tools) = failing_runner_world()?; + let ended = tick_until_ended(&world)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the deployment did not fail before fencing: {:?} {:?}", ended.phase, ended.last_error); + }; + // Named: the second step, on its runner, with the exit status. + assert!( + error.contains("runner step test-daemon on runner rust exited with exit status 3"), + "{error}" + ); + // Never echoed: no error text, no control.cc byte, carries the output. + let abort = ended.pre_fencing_abort.as_ref().context("no pre-fencing abort")?; + for text in [error.as_str(), abort.error.as_str(), &format!("{ended:?}")] { + assert!(!text.contains(RUNNER_CANARY), "runner output reached a record: {text}"); + } + // The terminal transaction has moved to history.cc: both stores. + for store in [world.state_store.clone(), history_store_path(&world.state_store)] { + let bytes = std::fs::read(&store)?; + assert!(!bytes.is_empty(), "{} is empty", store.display()); + assert!( + !bytes.windows(RUNNER_CANARY.len()).any(|window| window == RUNNER_CANARY.as_bytes()), + "runner output reached {}", + store.display() + ); + } + // Kept: in the private store, tails of both streams, and on status. + let evidence = runner_evidence_store_path(&world.state_store); + assert_eq!(std::fs::metadata(&evidence)?.permissions().mode() & 0o777, 0o600); + let record = runner_evidence::for_transaction(&evidence, &ended.transaction_id)? + .context("no evidence for the failed transaction")?; + assert_eq!(record.subject_id, "test-daemon"); + assert_eq!(record.exit_code, Some(3)); + assert!(String::from_utf8_lossy(&record.stdout_tail).contains(RUNNER_CANARY)); + assert!(String::from_utf8_lossy(&record.stderr_tail).contains(RUNNER_CANARY)); + let printed = printed_status(&world)?; + assert!(printed.contains("failed step test-daemon"), "{printed}"); + assert!(printed.contains(RUNNER_CANARY), "{printed}"); + assert!(!printed.contains('\u{1b}'), "status printed a raw escape byte"); + Ok(()) + } + + #[test] + fn the_journal_line_names_the_store_and_never_the_output() -> Result<()> { + let world = EngineFixture::new()?; + let (transaction, _) = terminal_transaction_with_command("ghostlight")?; + let failure = canary_failure(RUNNER_CANARY.as_bytes()); + let line = world + .engine + .keep_runner_failure(&transaction, &anyhow::Error::new(failure.clone()).context("sealing")) + .context("a runner failure produced no journal line")?; + assert!(!line.contains(RUNNER_CANARY), "{line}"); + for needle in [ + transaction.transaction_id.as_str(), + "up-ghostlight", + "ghostlight", + "runner step test-daemon on runner rust exited with exit status 3", + "runner-evidence.cc", + "idunn status --command up-ghostlight", + ] { + assert!(line.contains(needle), "{needle} missing from {line}"); + } + // Not a runner failure: nothing is kept and nothing is said. + assert!( + world + .engine + .keep_runner_failure(&transaction, &anyhow::anyhow!("a different failure")) + .is_none() + ); + // The line a failed write produces says so, with no output. + let unwritable = runner_failure_journal_line( + &transaction, + &failure, + Path::new("/var/lib/gamecult/idunn/runner-evidence.cc"), + &Err(anyhow::anyhow!("disk full")), + ); + assert!(unwritable.contains("evidence was not kept: disk full"), "{unwritable}"); + assert!(!unwritable.contains(RUNNER_CANARY)); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_failed_evidence_write_does_not_stop_the_abort() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + // A directory where the store should be: every write to it fails. + std::fs::create_dir(runner_evidence_store_path(&world.state_store))?; + let ended = tick_until_ended(&world)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the unwritable store stopped the abort: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("runner step test-daemon on runner rust"), "{error}"); + let line = world + .engine + .keep_runner_failure(&ended, &anyhow::Error::new(canary_failure(b""))) + .context("no journal line")?; + assert!(line.contains("evidence was not kept"), "{line}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn an_undecodable_evidence_store_does_not_stop_the_next_failure_keeping_its_evidence() -> Result<()> { + let damage = format!("\u{c1} not a store {RUNNER_CANARY}-in-the-damage"); + let (world, _tools) = failing_runner_world()?; + let evidence = runner_evidence_store_path(&world.state_store); + std::fs::write(&evidence, damage.as_bytes())?; + let ended = tick_until_ended(&world)?; + // The transaction's own outcome is the runner failure, not the store. + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the damaged store changed the outcome: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("runner step test-daemon on runner rust"), "{error}"); + assert!(!error.contains("damage"), "{error}"); + // The damage is kept aside; the failure's evidence is in a fresh store. + assert_eq!(std::fs::read(unreadable_store_path(&evidence))?, damage.as_bytes()); + let record = runner_evidence::for_transaction(&evidence, &ended.transaction_id)? + .context("the failure kept no evidence beside a damaged store")?; + assert_eq!(record.subject_id, "test-daemon"); + assert!(printed_status(&world)?.contains("failed step test-daemon")); + // The journal says it, naming the file and none of its content. + std::fs::write(&evidence, damage.as_bytes())?; + let line = world + .engine + .keep_runner_failure(&ended, &anyhow::Error::new(canary_failure(b"x"))) + .context("no journal line")?; + assert!(line.contains("an undecodable evidence store was set aside as"), "{line}"); + assert!(line.contains("runner-evidence.unreadable.cc"), "{line}"); + assert!(!line.contains("damage") && !line.contains("not a store"), "{line}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn an_unreadable_evidence_store_costs_only_its_lines() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + let ended = tick_until_ended(&world)?; + std::fs::write(runner_evidence_store_path(&world.state_store), b"\xc1not a store")?; + let printed = printed_status(&world)?; + assert!(printed.contains(&format!("transaction {}", ended.transaction_id)), "{printed}"); + assert!(printed.contains("target service phase"), "{printed}"); + assert!(printed.contains("pre-fencing abort"), "{printed}"); + assert_eq!(printed.matches("runner evidence unreadable").count(), 1, "{printed}"); + Ok(()) + } + + /// Everything status prints from the evidence store goes through one + /// allowlist, so a hostile build log reaches the terminal as visible + /// escapes whatever its class. + #[cfg(unix)] + #[test] + fn status_prints_hostile_runner_output_as_visible_escapes_only() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + let ended = tick_until_ended(&world)?; + let hostile: Vec<(&str, &str)> = vec![ + ("\u{1b}[2J", "\\u{1b}[2J"), + ("\u{7}", "\\u{7}"), + ("\r", "\\u{d}"), + ("\u{202e}", "\\u{202e}"), + ("\u{2066}", "\\u{2066}"), + ("\u{85}", "\\u{85}"), + ("\u{e0041}", "\\u{e0041}"), + ("\u{200b}", "\\u{200b}"), + ("\u{feff}", "\\u{feff}"), + ]; + let mut tail: Vec = Vec::new(); + for &(raw, _) in &hostile { + tail.extend_from_slice(b"a"); + tail.extend_from_slice(raw.as_bytes()); + tail.extend_from_slice(b"b\n"); + } + tail.extend_from_slice(b"bad \xff\xc1 bytes\n"); + runner_evidence::record( + &runner_evidence_store_path(&world.state_store), + &RunnerFailureRecord::from_failure( + &ended.transaction_id, + &ended.command_id, + &ended.target, + &canary_failure(&tail), + 9, + ), + )?; + let printed = printed_status(&world)?; + for &(raw, escaped) in &hostile { + assert!(!printed.contains(raw), "{escaped} reached the terminal raw"); + assert!(printed.contains(&format!("a{escaped}b")), "{escaped} missing: {printed}"); + } + assert!(printed.contains("bad \\xff\\xc1 bytes"), "{printed}"); + let section = printed.find(" failed step").context("no evidence section")?; + assert!(printed[section..].is_ascii(), "non-ASCII reached the terminal: {printed}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn status_prints_the_unreadable_store_line_escaped() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + tick_until_ended(&world)?; + let evidence = runner_evidence_store_path(&world.state_store); + // The decode error of this store quotes the stored subject kind. + let mut bytes = std::fs::read(&evidence)?; + let at = bytes + .windows(4) + .position(|window| window == b"step") + .context("no step marker in the evidence store")?; + bytes[at..at + 4].copy_from_slice(b"\x1b[2J"); + std::fs::write(&evidence, bytes)?; + let printed = printed_status(&world)?; + assert_eq!(printed.matches("runner evidence unreadable").count(), 1, "{printed}"); + assert!(!printed.contains('\u{1b}'), "an escape byte reached the terminal"); + assert!(printed.contains("\\u{1b}[2J"), "{printed}"); + Ok(()) + } + + #[test] + fn the_journal_line_names_a_displaced_set_aside_store_by_time_only() -> Result<()> { + let (transaction, _) = terminal_transaction_with_command("ghostlight")?; + let failure = canary_failure(RUNNER_CANARY.as_bytes()); + let path = Path::new("/var/lib/gamecult/idunn/runner-evidence.cc"); + let first = runner_failure_journal_line( + &transaction, + &failure, + path, + &Ok(Kept::SetAside { replaced: None }), + ); + assert!(first.contains("was set aside as"), "{first}"); + assert!(!first.contains("replacing"), "{first}"); + let second = runner_failure_journal_line( + &transaction, + &failure, + path, + &Ok(Kept::SetAside { replaced: Some(1234) }), + ); + assert!( + second.contains("replacing an earlier set-aside store last written at unix millis 1234"), + "{second}" + ); + assert!(!second.contains(RUNNER_CANARY)); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn control_store_types_are_unchanged_by_a_runner_failure() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + tick_until_ended(&world)?; + // The store a rolled-back Idunn reads: it refuses any foreign type. + ControlSnapshot::read(&world.state_store)?; + let mut transactions = 0; + for store in [world.state_store.clone(), history_store_path(&world.state_store)] { + for envelope in SingleFileMessagePackBackingStore::new(&store).pull_all_read_only_snapshot()? { + assert!(!envelope.r#type.contains("runner_failure"), "{}", envelope.r#type); + if envelope.r#type == DeploymentTransaction::TYPE { + transactions += 1; + assert_eq!(envelope.schema_id.as_deref(), Some("idunn.deployment_transaction.v4")); + } + } + } + assert_eq!(transactions, 1, "the failed transaction is in neither store"); + Ok(()) + } + + /// The records control.cc holds re-encode byte for byte. The hex was + /// captured at base 804ed75, before this cut touched anything: a runner + /// failure adds no field, no type and no schema to the control store. + #[test] + fn control_records_encode_exactly_as_they_did_at_base() -> Result<()> { + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "up-service".into(), + kind: CommandKind::Deploy, + selector: "service".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + let mut transaction = DeploymentTransaction::new(&command, "service".into(), 0, None, 100)?; + transaction.transaction_id = "tx-pinned".into(); + transaction.last_error = Some("runner step test-daemon on runner rust exited with exit status 3".into()); + transaction.pre_fencing_abort = Some(PreFencingAbort { + error: "runner step test-daemon on runner rust exited with exit status 3".into(), + candidate_cleanup: CleanupEvidence::Skipped, + topology_reconciliation: CleanupEvidence::Skipped, + source_cleanup: CleanupEvidence::Complete, + }); + let hex = |bytes: &[u8]| bytes.iter().map(|byte| format!("{byte:02x}")).collect::(); + let command_hex = hex(&command_envelope(&command, 100)?.payload); + let transaction_hex = hex(&transaction_envelope(&transaction, 100)?.payload); + assert_eq!(command_hex, PINNED_COMMAND_HEX, "command bytes changed"); + assert_eq!(transaction_hex, PINNED_TRANSACTION_HEX, "transaction bytes changed"); + Ok(()) + } + + const PINNED_COMMAND_HEX: &str = "96bb6964756e6e2e6465706c6f796d656e745f636f6d6d616e642e7632aa75702d73657276696365a66465706c6f79a773657276696365a47465737464"; + const PINNED_TRANSACTION_HEX: &str = "dc0025bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7634a974782d70696e6e6564aa75702d73657276696365a66465706c6f79a77365727669636500a77365616c696e676464c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c000d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033c094d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033a7736b6970706564a7736b6970706564a8636f6d706c657465c0c0c0c0"; + // END runner-failure-evidence tests } diff --git a/src/deployment.rs b/src/deployment.rs index a67b4bc..b7cbdc8 100644 --- a/src/deployment.rs +++ b/src/deployment.rs @@ -9,12 +9,27 @@ pub const TARGET_DECLARATION_SCHEMA: &str = "gamecult.idunn.target_declaration.v pub const OPERATOR_BINDING_SCHEMA: &str = "gamecult.idunn.operator_binding.v2"; pub const IDUNN_RUNTIME_BUNDLE_ENVIRONMENT: &str = "GAMECULT_IDUNN_RUNTIME_BUNDLE"; pub const IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT: &str = "GAMECULT_IDUNN_CANDIDATE_BIND"; +pub const IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT: &str = "GAMECULT_IDUNN_MEMBER_BIND"; pub const IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT: &str = "GAMECULT_IDUNN_PROCESS_WRITE_LEASE"; pub const RUNTIME_PRESENCE_IDENTITY_BINDING: &str = "GAMECULT_RUNTIME_PRESENCE_IDENTITY"; /// A host actuator hands the workload its activation credential as a file /// named by this variable; on the Idunn host it arrives as a parent-only /// descriptor instead and this name is never set. pub const IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT: &str = "GAMECULT_IDUNN_ACTIVATION_CREDENTIAL"; +/// systemd hands a unit its credential directory under this name; Idunn owns +/// the directory, so no binding may set it. +pub const SYSTEMD_CREDENTIALS_DIRECTORY_ENVIRONMENT: &str = "CREDENTIALS_DIRECTORY"; +/// Every launch-environment name Idunn sets or reserves. A binding may name +/// none of them; this is the one list both the binding check and the +/// launch-input check read. +pub(crate) const IDUNN_OWNED_LAUNCH_ENVIRONMENT: [&str; 6] = [ + IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, + IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, + IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT, + IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, + IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, + SYSTEMD_CREDENTIALS_DIRECTORY_ENVIRONMENT, +]; pub const RUNTIME_PRESENCE_IDENTITY_FD_NAME: &str = "gamecult-runtime-presence-identity"; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -98,6 +113,10 @@ pub struct ArtifactOutput { pub expected_sha256: Option, #[serde(default)] pub executable: bool, + /// Regular files inside a directory artifact that are sealed executable + /// (0555); every other file in the tree is sealed 0444. + #[serde(default)] + pub executable_members: BTreeSet, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -184,6 +203,11 @@ pub enum StateKind { /// Process-owned state imposed by an external runtime contract. Idunn /// preserves and fences the file but never interprets its payload. ExternalFile, + /// A process-owned directory imposed by an external runtime contract (for + /// example Synapse's media store). Idunn preserves and fences it and never + /// interprets its contents. Same writer, recovery and startup rules as + /// `ExternalFile`. + ExternalDirectory, } #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] @@ -504,8 +528,7 @@ impl WorkloadBinding { { require_environment_name(name)?; ensure!( - name != IDUNN_RUNTIME_BUNDLE_ENVIRONMENT - && name != IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, + !IDUNN_OWNED_LAUNCH_ENVIRONMENT.contains(&name.as_str()), "operator binding attempts to replace Idunn-owned workload environment" ); } @@ -638,9 +661,7 @@ pub struct ProcessWriteLeaseBinding { impl ProcessWriteLeaseBinding { pub fn lock_path(&self) -> PathBuf { - let mut lock_path = self.record_path.as_os_str().to_os_string(); - lock_path.push(".lock"); - PathBuf::from(lock_path) + crate::drivers::authority_lock_path(&self.record_path) } } @@ -662,40 +683,99 @@ pub struct RouteBinding { pub private_port_end: u16, pub config_path: PathBuf, pub reload_unit: String, + /// A second listener for the route's members. When set, members reach the + /// workload straight through a second nginx server block, and the + /// workload's launcher answers presence only. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub member: Option, +} + +/// The member listener of a route: where members connect, and the first of the +/// private ports the workload binds for them. The private range is as wide as +/// the route's own. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MemberListenerBinding { + pub stable_endpoint: String, + pub private_port_start: u16, + /// Most member connections nginx holds open at once for this route. Each + /// costs nginx two connections (client and upstream) from the pool the + /// route's presence block shares, so this limit is what keeps members from + /// exhausting it. + #[serde(default = "default_member_max_connections")] + pub max_connections: u16, +} + +const DEFAULT_MEMBER_MAX_CONNECTIONS: u16 = 128; + +fn default_member_max_connections() -> u16 { + DEFAULT_MEMBER_MAX_CONNECTIONS } impl RouteBinding { pub(crate) fn stable_socket(&self) -> Result<(IpAddr, u16)> { - let authority = match self.driver { - RouteDriver::NginxStreamTcp => self - .stable_endpoint - .strip_prefix("http://") - .or_else(|| self.stable_endpoint.strip_prefix("tcp://")), - RouteDriver::NginxStreamUdp => self.stable_endpoint.strip_prefix("rudp://"), - } - .context("stable endpoint transport does not match its route driver")?; - ensure!( - !authority - .chars() - .any(|character| matches!(character, '/' | '?' | '#' | '@' | '[' | ']')), - "stable endpoint is not a plain IP address and port" - ); - let (host, port) = authority - .rsplit_once(':') - .context("stable endpoint has no port")?; - let host = host - .parse::() - .context("stable endpoint host is not an IP address")?; - ensure!( - matches!(host, IpAddr::V4(_)), - "route binding v2 stable endpoints use explicit IPv4 addresses" - ); - let port = port - .parse::() - .context("stable endpoint port is not a u16")?; - ensure!(port > 0, "stable endpoint port is zero"); - Ok((host, port)) + parse_stable_socket(self.driver, &self.stable_endpoint, "stable endpoint") } + + /// The member listener's stable socket, when the route has one. + pub(crate) fn member_stable_socket(&self) -> Result> { + self.member + .as_ref() + .map(|member| { + parse_stable_socket(self.driver, &member.stable_endpoint, "member endpoint") + }) + .transpose() + } + + /// The private port the workload binds for members when its presence + /// candidate is `candidate_port`. Derived, never stored: the member range + /// is the private range shifted to the member start. + pub(crate) fn member_candidate_port(&self, candidate_port: u16) -> Result> { + let Some(member) = self.member.as_ref() else { + return Ok(None); + }; + ensure!( + (self.private_port_start..=self.private_port_end).contains(&candidate_port), + "candidate port is outside the route binding" + ); + let offset = candidate_port - self.private_port_start; + let port = member + .private_port_start + .checked_add(offset) + .context("member private port overflows")?; + Ok(Some(port)) + } +} + +fn parse_stable_socket(driver: RouteDriver, endpoint: &str, what: &str) -> Result<(IpAddr, u16)> { + let authority = match driver { + RouteDriver::NginxStreamTcp => endpoint + .strip_prefix("http://") + .or_else(|| endpoint.strip_prefix("tcp://")), + RouteDriver::NginxStreamUdp => endpoint.strip_prefix("rudp://"), + } + .with_context(|| format!("{what} transport does not match its route driver"))?; + ensure!( + !authority + .chars() + .any(|character| matches!(character, '/' | '?' | '#' | '@' | '[' | ']')), + "{what} is not a plain IP address and port" + ); + let (host, port) = authority + .rsplit_once(':') + .with_context(|| format!("{what} has no port"))?; + let host = host + .parse::() + .with_context(|| format!("{what} host is not an IP address"))?; + ensure!( + matches!(host, IpAddr::V4(_)), + "route binding v2 endpoints use explicit IPv4 addresses" + ); + let port = port + .parse::() + .with_context(|| format!("{what} port is not a u16"))?; + ensure!(port > 0, "{what} port is zero"); + Ok((host, port)) } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -836,6 +916,14 @@ impl TargetDeclaration { if let Some(expected_sha256) = &artifact.expected_sha256 { require_sha256(expected_sha256, "expected artifact sha256")?; } + ensure!( + !(artifact.executable && !artifact.executable_members.is_empty()), + "artifact {} declares both executable and executable_members", + artifact.id + ); + for member in &artifact.executable_members { + require_relative_path(member, "executable member", false)?; + } ensure!( destinations.insert(artifact.destination.as_path()), "artifact destination {} is declared twice", @@ -919,6 +1007,14 @@ impl TargetDeclaration { == self.service.route_required, "routed services must require exactly one Idunn-owned candidate bind" ); + ensure!( + !self + .service + .required_environment + .contains(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT) + || self.service.route_required, + "only routed services may require the Idunn-owned member bind" + ); ensure!( self.service .required_environment @@ -1376,7 +1472,39 @@ impl OperatorBinding { route.private_port_start < route.private_port_end, "candidate rollout requires at least two private ports" ); - route.stable_socket()?; + let stable = route.stable_socket()?; + if let Some(member) = route.member.as_ref() { + ensure!( + route.driver == RouteDriver::NginxStreamTcp, + "only a TCP route may carry a member listener" + ); + require_value(&member.stable_endpoint, "member endpoint")?; + let member_stable = route + .member_stable_socket()? + .context("route has no member listener")?; + ensure!( + member_stable != stable, + "member endpoint equals the route's stable endpoint" + ); + ensure!( + member.private_port_start > 0, + "member private port range starts at zero" + ); + ensure!( + member.max_connections > 0, + "member connection limit is zero" + ); + let width = route.private_port_end - route.private_port_start; + let member_end = member + .private_port_start + .checked_add(width) + .context("member private port range overflows")?; + ensure!( + member_end < route.private_port_start + || member.private_port_start > route.private_port_end, + "member private port range overlaps the route's private range" + ); + } require_absolute_path(&route.config_path, "route config")?; require_unit(&route.reload_unit, "route reload unit")?; } @@ -1554,12 +1682,7 @@ impl OperatorBinding { .keys() .chain(self.workload.secret_environment_names()) .collect(); - for owned in [ - IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, - IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, - IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, - IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, - ] { + for owned in IDUNN_OWNED_LAUNCH_ENVIRONMENT { ensure!( !workload_names.contains(&owned.to_string()), "operator binding cannot replace Idunn-owned launch environment {owned}" @@ -1581,6 +1704,9 @@ impl OperatorBinding { } if declaration.service.route_required { available_environment.insert(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT.into()); + if self.route.as_ref().is_some_and(|route| route.member.is_some()) { + available_environment.insert(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT.into()); + } } if declaration.write_lease_required() { available_environment.insert(IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT.into()); @@ -1928,10 +2054,10 @@ fn require_host(value: &str) -> Result<()> { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; - const RECIPE: &str = r#" + pub(crate) const RECIPE: &str = r#" schema = "gamecult.idunn.target_declaration.v1" target = "test-service" source_stamp_environment = "TEST_SERVICE_BUILD_COMMIT" @@ -2003,7 +2129,7 @@ compatibility = "v1" startup = "before-promotion" "#; - const BINDING: &str = r#" + pub(crate) const BINDING: &str = r#" schema = "gamecult.idunn.operator_binding.v2" target = "test-service" profiles = ["aetheria", "full-gamecult"] @@ -2097,6 +2223,76 @@ nodes = ["yggdrasil"] assert!(TargetDeclaration::parse(&input).is_err()); } + const MEMBER_CANARY: &str = "zzcanary"; + + fn recipe_with_members(members: &str) -> String { + let input = RECIPE.replace("executable = true", &format!("executable_members = [{members}]")); + assert_ne!(input, RECIPE, "the fixture lost its executable line"); + input + } + + /// A refused member list is refused by name of the field, never by echoing + /// the member. + fn assert_members_refused(members: &[&str]) { + let control = TargetDeclaration::parse(&recipe_with_members("\"bin/daemon\"")).unwrap(); + assert!(control.artifacts[0].executable_members.contains(Path::new("bin/daemon"))); + for member in members { + let error = TargetDeclaration::parse(&recipe_with_members(&format!("\"{member}\""))) + .expect_err(member); + assert!( + !format!("{error:#}").contains(MEMBER_CANARY), + "the refusal echoed a member path" + ); + } + } + + #[test] + fn recipe_executable_member_refuses_parent_traversal() { + assert_members_refused(&[ + "zzcanary/../b", + "../zzcanary", + "bin/zzcanary/..", + "bin/../../zzcanary", + ]); + } + + #[test] + fn recipe_executable_member_refuses_absolute_path() { + assert_members_refused(&["/zzcanary/sh", "/zzcanary"]); + } + + #[test] + fn recipe_executable_member_refuses_empty_and_current_dir() { + assert_members_refused(&["", ".", "./", "zzcanary/.."]); + } + + #[test] + fn recipe_refuses_executable_flag_with_members() { + let input = RECIPE.replace( + "executable = true", + "executable = true\nexecutable_members = [\"bin/daemon\"]", + ); + let error = TargetDeclaration::parse(&input).unwrap_err(); + assert!(format!("{error:#}").contains("executable_members")); + assert!(!format!("{error:#}").contains("bin/daemon")); + // executable = true alone, and members alone, both stay admitted. + assert!(TargetDeclaration::parse(RECIPE).is_ok()); + assert!(TargetDeclaration::parse(&recipe_with_members("\"bin/daemon\"")).is_ok()); + } + + #[test] + fn recipe_admits_members_on_both_source_kinds() { + let runner_output = TargetDeclaration::parse(&recipe_with_members("\"a/b\", \"c\"")).unwrap(); + assert_eq!(runner_output.artifacts[0].source_kind, ArtifactSource::RunnerOutput); + assert_eq!(runner_output.artifacts[0].executable_members.len(), 2); + let worktree = recipe_with_members("\"a/b\", \"c\"") + .replace("source_kind = \"runner-output\"\nrunner = \"rust\"\n", "source_kind = \"worktree-tree\"\n") + .replace("expected_sha256 = \"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\"\n", ""); + let worktree = TargetDeclaration::parse(&worktree).unwrap(); + assert_eq!(worktree.artifacts[0].source_kind, ArtifactSource::WorktreeTree); + assert_eq!(worktree.artifacts[0].executable_members.len(), 2); + } + #[test] fn operator_cannot_omit_an_affordance_derived_from_the_recipe() { let recipe = TargetDeclaration::parse(RECIPE).unwrap(); @@ -2230,6 +2426,203 @@ nodes = ["yggdrasil"] } } + /// The recipe and binding of a routed service whose members connect to a + /// listener of their own. + pub(crate) fn member_listener_pair() -> (String, String) { + let recipe = RECIPE.replace( + "required_environment = [", + "required_environment = [\"GAMECULT_IDUNN_MEMBER_BIND\", ", + ); + let binding = BINDING.replace( + "reload_unit = \"nginx.service\"\n", + "reload_unit = \"nginx.service\"\n\n[route.member]\nstable_endpoint = \"http://127.0.0.1:18840\"\nprivate_port_start = 18841\n", + ); + (recipe, binding) + } + + fn member_binding_with(replacement: (&str, &str)) -> Result { + let (recipe, binding) = member_listener_pair(); + let recipe = TargetDeclaration::parse(&recipe)?; + let binding = OperatorBinding::parse(&binding.replace(replacement.0, replacement.1))?; + binding.admit(&recipe)?; + Ok(binding) + } + + #[test] + fn a_member_listener_admits_with_a_recipe_that_requires_the_member_bind() { + let binding = member_binding_with(("never-matches", "")).unwrap(); + let route = binding.route.unwrap(); + assert_eq!(route.member_stable_socket().unwrap().unwrap().1, 18840); + assert_eq!(route.member_candidate_port(18831).unwrap(), Some(18841)); + assert_eq!(route.member_candidate_port(18839).unwrap(), Some(18849)); + assert!(route.member_candidate_port(18840).is_err()); + } + + #[test] + fn a_binding_without_a_member_listener_has_no_member_ports() { + let binding = OperatorBinding::parse(BINDING).unwrap(); + let route = binding.route.unwrap(); + assert_eq!(route.member_stable_socket().unwrap(), None); + assert_eq!(route.member_candidate_port(18831).unwrap(), None); + } + + #[test] + fn a_member_listener_needs_a_recipe_that_requires_the_member_bind() { + let (recipe, binding) = member_listener_pair(); + // Bound but not required: the recipe never asked for the variable. + let plain_recipe = TargetDeclaration::parse(RECIPE).unwrap(); + assert!( + OperatorBinding::parse(&binding) + .unwrap() + .admit(&plain_recipe) + .is_err() + ); + // Required but not bound: the variable would never be set. + let recipe = TargetDeclaration::parse(&recipe).unwrap(); + assert!( + OperatorBinding::parse(BINDING) + .unwrap() + .admit(&recipe) + .is_err() + ); + } + + #[test] + fn only_a_routed_recipe_may_require_the_member_bind() { + let recipe = RECIPE + .replace("route_required = true", "route_required = false") + .replace( + "required_environment = [\"GAMECULT_IDUNN_CANDIDATE_BIND\", ", + "required_environment = [\"GAMECULT_IDUNN_MEMBER_BIND\", ", + ); + assert!(recipe.contains("GAMECULT_IDUNN_MEMBER_BIND")); + assert!(TargetDeclaration::parse(&recipe).is_err()); + } + + #[test] + fn member_listener_bindings_are_validated() { + let start = |value: &str| { + member_binding_with(( + "private_port_start = 18841", + &format!("private_port_start = {value}"), + )) + }; + // Overlapping the route's own private range, in either direction. + assert!(start("18835").is_err()); + assert!(start("18825").is_err()); + // Adjacent but disjoint is fine, above the route's range or below it. + assert!(start("18840").is_ok()); + assert!(start("18822").is_ok()); + assert!(start("18823").is_err()); + // The connection limit defaults, and zero is refused. + let limit = |value: &str| { + member_binding_with(( + "private_port_start = 18841", + &format!("private_port_start = 18841 +max_connections = {value}"), + )) + }; + assert_eq!( + member_binding_with(("never-matches", "")).unwrap().route.unwrap().member.unwrap().max_connections, + 128 + ); + assert_eq!(limit("7").unwrap().route.unwrap().member.unwrap().max_connections, 7); + assert!(limit("0").is_err()); + assert!(limit("65535").is_ok()); + assert!(limit("65536").is_err()); + // Zero, and a width that overflows u16. + assert!(start("0").is_err()); + assert!(start("65530").is_err()); + // The member endpoint may not be the route's stable endpoint. + assert!(member_binding_with(("http://127.0.0.1:18840", "http://127.0.0.1:18830")).is_err()); + // The same transport and address rules as the stable endpoint. + assert!(member_binding_with(("http://127.0.0.1:18840", "rudp://127.0.0.1:18840")).is_err()); + assert!(member_binding_with(("http://127.0.0.1:18840", "http://example.test:18840")).is_err()); + } + + #[test] + fn a_udp_route_cannot_carry_a_member_listener() { + let (recipe, binding) = member_listener_pair(); + let binding = binding + .replace("nginx-stream-tcp", "nginx-stream-udp") + .replace("http://127.0.0.1:18830", "rudp://127.0.0.1:18830") + .replace("http://127.0.0.1:18840", "rudp://127.0.0.1:18840"); + let recipe = TargetDeclaration::parse(&recipe).unwrap(); + assert!( + OperatorBinding::parse(&binding) + .and_then(|binding| binding.admit(&recipe)) + .is_err() + ); + } + + #[test] + fn idunn_owned_names_are_reserved_in_one_list() { + let recipe = TargetDeclaration::parse(RECIPE).unwrap(); + let owned = [ + "GAMECULT_IDUNN_RUNTIME_BUNDLE", + "GAMECULT_IDUNN_CANDIDATE_BIND", + "GAMECULT_IDUNN_MEMBER_BIND", + "GAMECULT_IDUNN_PROCESS_WRITE_LEASE", + "GAMECULT_IDUNN_ACTIVATION_CREDENTIAL", + "CREDENTIALS_DIRECTORY", + ]; + let mut listed = IDUNN_OWNED_LAUNCH_ENVIRONMENT.to_vec(); + listed.sort_unstable(); + let mut expected = owned.to_vec(); + expected.sort_unstable(); + assert_eq!(listed, expected); + + let as_environment = |name: &str| { + BINDING.replace( + "[workload.argument_bindings]", + &format!("[workload.environment]\n{name} = \"value\"\n\n[workload.argument_bindings]"), + ) + }; + let as_secret = |name: &str| { + BINDING.replace( + "[workload.secret_files]", + &format!("[workload.secret_files]\n{name} = \"/etc/gamecult/test-service/other.cc\""), + ) + }; + // The fixtures are well formed: an unowned name parses in both + // shapes, so the refusals below are about the name. + assert!(OperatorBinding::parse(&as_environment("UNOWNED_NAME")).is_ok()); + assert!(OperatorBinding::parse(&as_secret("UNOWNED_NAME")).is_ok()); + for name in owned { + // Refused when the binding is parsed... + assert!( + OperatorBinding::parse(&as_environment(name)).is_err(), + "{name} accepted as environment at parse" + ); + assert!( + OperatorBinding::parse(&as_secret(name)).is_err(), + "{name} accepted as a secret file at parse" + ); + // ...and again at admission, for a binding that did not come + // through parse. Each site reads the one list. + for secret in [false, true] { + let mut binding = OperatorBinding::parse(BINDING).unwrap(); + let WorkloadBinding::SystemdTransient(workload) = &mut binding.workload else { + panic!("the fixture is a systemd workload"); + }; + if secret { + workload.secret_files.insert(name.into(), "/etc/gamecult/test-service/other.cc".into()); + } else { + workload.environment.insert(name.into(), "forged".into()); + } + assert!( + binding.workload.validate_launch_environment().is_err(), + "{name} (secret: {secret}) accepted by the binding check" + ); + let error = binding.admit(&recipe).unwrap_err().to_string(); + assert!( + error.contains("Idunn-owned launch environment"), + "{name} (secret: {secret}) accepted by the launch-input check: {error}" + ); + } + } + } + #[test] fn state_slot_path_cannot_escape_the_operator_owned_root() { let input = RECIPE.replace( @@ -2261,6 +2654,17 @@ nodes = ["yggdrasil"] assert!(declaration.write_lease_required()); } + #[test] + fn external_directory_state_is_lease_gated_like_external_file() { + let input = RECIPE.replace("kind = \"cultcache-file\"", "kind = \"external-directory\""); + let declaration = TargetDeclaration::parse(&input).unwrap(); + let slot = &declaration.state.as_ref().unwrap().slots[0]; + assert_eq!(slot.kind, StateKind::ExternalDirectory); + assert_eq!(slot.writer, StateWriter::ProcessBoundSingleWriter); + assert_eq!(slot.startup, StateStartup::CreateOrOpenAfterWriteLease); + assert!(declaration.write_lease_required()); + } + #[test] fn state_slots_cannot_claim_overlapping_paths() { let input = RECIPE.replace( diff --git a/src/deployment_plan.rs b/src/deployment_plan.rs index 603c9d2..eceb513 100644 --- a/src/deployment_plan.rs +++ b/src/deployment_plan.rs @@ -475,6 +475,13 @@ pub const DEFAULT_AWAITING_READY_DEADLINE_SECONDS: u32 = 300; pub const DEFAULT_ROUTING_DEADLINE_SECONDS: u32 = 120; pub const DEFAULT_COMMITTING_DEADLINE_SECONDS: u32 = 120; +/// Idunn's own deadlines, not binding-declared: a new `PhaseDeadlines` field +/// would change every v3 plan id (`recomputed_plan_id` re-encodes the plan). +/// Warming is timed for every transaction, owner or not. +pub const WARMING_DEADLINE_SECONDS: u32 = 600; +/// A pre-fencing abort is timed from its beginning. +pub const PRE_FENCING_ABORT_DEADLINE_SECONDS: u32 = 300; + /// The resolved per-phase deadline durations a plan carries. Entering a /// post-fencing phase stamps its deadline from them, and the deadline /// resolver ends the phase once it passes. @@ -727,6 +734,11 @@ pub struct ArtifactReceipt { pub sha256: String, pub size_bytes: u64, pub executable: bool, + /// Files sealed executable inside a directory artifact. Skipped when empty + /// so a receipt with no members keeps its original encoding, and so its + /// sealed release id. + #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] + pub executable_members: BTreeSet, } impl ArtifactReceipt { @@ -740,7 +752,9 @@ impl ArtifactReceipt { .find(|artifact| artifact.id == self.artifact_id) .with_context(|| format!("plan declares no artifact {}", self.artifact_id))?; ensure!( - self.destination == declared.destination && self.executable == declared.executable, + self.destination == declared.destination + && self.executable == declared.executable + && self.executable_members == declared.executable_members, "artifact receipt differs from declared output" ); if let Some(expected_sha256) = &declared.expected_sha256 { @@ -1418,6 +1432,7 @@ nodes = ["yggdrasil"] sha256: digest(b'c'), size_bytes: 42, executable: true, + executable_members: BTreeSet::new(), } } @@ -1784,6 +1799,113 @@ nodes = ["yggdrasil"] assert!(changed.validate_against(&plan).is_err()); } + fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() + } + + #[test] + fn receipt_without_members_encodes_as_before() { + // Both values were computed outside this code, from the five-field + // msgpack array the receipt encoded to at base d74ac8d. + const BASE_RECEIPT: &str = "95a66461656d6f6ea773657276696365d9477368613235362d636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363632ac3"; + const BASE_RELEASE_ID: &str = + "sha256-b9143d8f546ad71c493c70037813547aaec10fd87994a6f3bdc77ac3a8b3495e"; + let receipt = artifact_receipt(); + let encoded = rmp_serde::to_vec(&receipt).unwrap(); + assert_eq!(hex(&encoded), BASE_RECEIPT); + let bytes = (0..BASE_RECEIPT.len()) + .step_by(2) + .map(|at| u8::from_str_radix(&BASE_RECEIPT[at..at + 2], 16).unwrap()) + .collect::>(); + assert_eq!(rmp_serde::from_slice::(&bytes).unwrap(), receipt); + let release = SealedRelease { + schema: SEALED_RELEASE_SCHEMA.into(), + sealed_release_id: String::new(), + plan_id: digest(b'a'), + artifacts: vec![receipt], + external_inputs: vec![], + sealed_at_unix_millis: 0, + }; + assert_eq!(release.recomputed_release_id().unwrap(), BASE_RELEASE_ID); + } + + #[test] + fn sealed_release_address_covers_executable_members() { + let release_with = |members: &[&str]| SealedRelease { + schema: SEALED_RELEASE_SCHEMA.into(), + sealed_release_id: String::new(), + plan_id: digest(b'a'), + artifacts: vec![ArtifactReceipt { + executable: false, + executable_members: members.iter().map(PathBuf::from).collect(), + ..artifact_receipt() + }], + external_inputs: vec![], + sealed_at_unix_millis: 0, + }; + let none = release_with(&[]).recomputed_release_id().unwrap(); + let one = release_with(&["bin/daemon"]).recomputed_release_id().unwrap(); + let other = release_with(&["bin/other"]).recomputed_release_id().unwrap(); + let two = release_with(&["bin/daemon", "bin/other"]) + .recomputed_release_id() + .unwrap(); + let all = [&none, &one, &other, &two]; + for (index, left) in all.iter().enumerate() { + for right in &all[index + 1..] { + assert_ne!(left, right); + } + } + + // The declaration is the other half: a receipt must carry exactly the + // declared list, in a plan that declares one and in one that does not. + let plain = plan(); + let mut with_members = artifact_receipt(); + with_members.executable_members = BTreeSet::from([PathBuf::from("bin/daemon")]); + assert!( + SealedRelease::new(&plain, vec![with_members], vec![external_input_receipt()], 120) + .is_err() + ); + let recipe = RECIPE.replace("executable = true", "executable_members = [\"bin/daemon\", \"lib/tool\"]"); + let declaring = compile_deployment_plan( + recipe.as_bytes(), + BINDING.as_bytes(), + source(&recipe), + "service-incarnation-1", + Some(18001), + 110, + &[ready_odin_provider("odin", "odin-yggdrasil", 1)], + ) + .unwrap(); + let honest = ArtifactReceipt { + executable: false, + executable_members: BTreeSet::from([PathBuf::from("bin/daemon"), PathBuf::from("lib/tool")]), + ..artifact_receipt() + }; + let release = SealedRelease::new( + &declaring, + vec![honest.clone()], + vec![external_input_receipt()], + 120, + ) + .unwrap(); + release.validate_against(&declaring).unwrap(); + for drifted in [ + BTreeSet::from([PathBuf::from("bin/daemon")]), + BTreeSet::from([PathBuf::from("bin/daemon"), PathBuf::from("lib/other")]), + BTreeSet::new(), + ] { + let mut receipt = honest.clone(); + receipt.executable_members = drifted; + assert!( + SealedRelease::new(&declaring, vec![receipt], vec![external_input_receipt()], 120) + .is_err() + ); + } + let mut changed = release.clone(); + changed.artifacts[0].executable_members.insert(PathBuf::from("lib/extra")); + assert!(changed.validate_against(&declaring).is_err()); + } + fn binding_with_deadlines(table: &str) -> String { BINDING.replace("[placement]", &format!("{table}\n[placement]")) } diff --git a/src/drivers.rs b/src/drivers.rs index 072d49d..3ea1145 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::ffi::{OsStr, OsString}; use std::fs::{self, OpenOptions}; use std::io::{BufRead, BufReader, ErrorKind, Read, Write}; @@ -22,7 +22,8 @@ use crate::control_plane::SequenceAdmittedWarming; use crate::deployment::{ ArtifactOutput, ArtifactSource, DockerRunnerBinding, GitlinkBinding, IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, - IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, LaunchArgument, OperatorBinding, + IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT, + LaunchArgument, OperatorBinding, RUNTIME_PRESENCE_IDENTITY_BINDING, RUNTIME_PRESENCE_IDENTITY_FD_NAME, RouteBinding, RouteDriver, SourceSelectionPolicy, TargetDeclaration, WorkloadNetwork, }; @@ -156,6 +157,24 @@ pub struct FrozenSourceReceipt { pub snapshot_sha256: String, } +#[cfg(test)] +impl FrozenSource { + /// A frozen source over an existing tree, for tests that drive a runner. + pub(crate) fn fixture( + receipt: FrozenSourceReceipt, + facts: SourceSelectionFacts, + recipe_bytes: Vec, + root: PathBuf, + ) -> Self { + Self { + receipt, + facts, + recipe_bytes, + root, + } + } +} + impl FrozenSourceReceipt { pub fn validate_against(&self, plan: &CompiledDeploymentPlan) -> Result<()> { plan.validate()?; @@ -469,6 +488,9 @@ pub struct SystemdWorkloadObservation { pub no_new_privileges: bool, pub umask: String, pub inaccessible_paths: String, + /// Retired wire slot. systemd 257 displays `LoadCredential` as `[unprintable]` for + /// every unit, so it was never evidence: written empty, never compared. Kept so + /// compact (positional) MessagePack records of earlier Idunns still decode. pub load_credential: String, pub main_pid: u32, pub process_start_time: u64, @@ -583,10 +605,23 @@ pub trait WorkloadPort { /// /// Idunn launches candidates as transient units with `Restart=no`, so a /// unit that has failed, or that systemd no longer knows, will not come - /// back on its own. Past the fence every error is otherwise treated as + /// back on its own. Those units are collected once they fail + /// (`INCARNATION_UNIT_LIFECYCLE`), so a failed candidate usually reads as + /// forgotten rather than failed. Past the fence every error is otherwise treated as /// resumable and retried forever, which is right while a candidate can /// still recover and a trap once it cannot. fn is_permanently_stopped(&self, observation: &WorkloadObservation) -> Result; + + /// True only when the host lost the workload's unit to a reboot after the + /// generation was admitted at `admitted_at_unix_millis`. The driver owns + /// the evidence; continuity supervision owns the budget it buys (one + /// uncharged re-materialization per generation). Every implementation + /// answers explicitly. + fn lost_to_host_reboot( + &self, + observation: &WorkloadObservation, + admitted_at_unix_millis: u64, + ) -> Result; } pub trait TopologyPort { @@ -2137,6 +2172,115 @@ pub(crate) fn docker_run_args( Ok(args) } +/// Bytes of each output stream that a failed runner step keeps. The daemon +/// never holds more than this per stream while the step runs. +pub(crate) const RUNNER_OUTPUT_TAIL_BYTES: usize = 8192; + +/// What a runner container was running: a recipe step or an external input, +/// by the id the recipe gave it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum RunnerSubject { + Step(String), + ExternalInput(String), +} + +impl RunnerSubject { + pub(crate) fn id(&self) -> &str { + match self { + Self::Step(id) | Self::ExternalInput(id) => id, + } + } + + fn noun(&self) -> &'static str { + match self { + Self::Step(_) => "step", + Self::ExternalInput(_) => "external input", + } + } +} + +/// A runner container that did not exit successfully: what failed, how, and +/// the last `RUNNER_OUTPUT_TAIL_BYTES` of each stream with its total size. +/// The tails are evidence for `runner-evidence.cc`. `Display` and `Debug` +/// carry none of them, so no error chain, record or log line renders output. +#[derive(Clone, PartialEq, Eq)] +pub(crate) struct RunnerStepFailure { + pub(crate) subject: RunnerSubject, + pub(crate) runner: String, + pub(crate) exit_code: Option, + pub(crate) signal: Option, + pub(crate) stdout_tail: Vec, + pub(crate) stdout_total_bytes: u64, + pub(crate) stderr_tail: Vec, + pub(crate) stderr_total_bytes: u64, +} + +impl RunnerStepFailure { + /// The status as an operator reads it. + pub(crate) fn status(&self) -> String { + runner_status(self.exit_code, self.signal) + } +} + +/// How a container ended, as an operator reads it. +pub(crate) fn runner_status(exit_code: Option, signal: Option) -> String { + match (exit_code, signal) { + (Some(code), _) => format!("exit status {code}"), + (None, Some(signal)) => format!("signal {signal}"), + (None, None) => "an unknown status".to_owned(), + } +} + +impl std::fmt::Display for RunnerStepFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "runner {} {} on runner {} exited with {}; its output tail is kept in runner-evidence.cc", + self.subject.noun(), + self.subject.id(), + self.runner, + self.status() + ) + } +} + +impl std::fmt::Debug for RunnerStepFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RunnerStepFailure") + .field("subject", &self.subject) + .field("runner", &self.runner) + .field("exit_code", &self.exit_code) + .field("signal", &self.signal) + .field("stdout_total_bytes", &self.stdout_total_bytes) + .field("stderr_total_bytes", &self.stderr_total_bytes) + .finish_non_exhaustive() + } +} + +impl std::error::Error for RunnerStepFailure {} + +/// Read `reader` to its end keeping only the last `cap` bytes and counting +/// every byte, so a verbose step never buffers whole in the daemon. +fn drain_tail(mut reader: impl Read, cap: usize) -> std::io::Result<(Vec, u64)> { + let mut tail = Vec::new(); + let mut total = 0u64; + let mut chunk = [0u8; 8192]; + loop { + let read = match reader.read(&mut chunk) { + Ok(0) => return Ok((tail, total)), + Ok(read) => read, + Err(error) if error.kind() == ErrorKind::Interrupted => continue, + Err(error) => return Err(error), + }; + total += read as u64; + tail.extend_from_slice(&chunk[..read]); + if tail.len() > cap { + tail.drain(..tail.len() - cap); + } + } +} + /// Docker is only a runner substrate. Recipe argv stays an argv vector, the /// operator binding selects the exact image/network/mount affordances, and the /// driver returns complete materialization receipts rather than build truth by @@ -2155,7 +2299,7 @@ impl Default for DockerRunnerDriver { } impl DockerRunnerDriver { - fn docker(&self, args: I) -> Result + fn docker(&self, subject: RunnerSubject, runner_id: &str, args: I) -> Result<()> where I: IntoIterator, S: AsRef, @@ -2164,23 +2308,50 @@ impl DockerRunnerDriver { self.docker_program.is_absolute(), "Docker runner program is not absolute" ); - let output = Command::new(&self.docker_program) + let mut child = Command::new(&self.docker_program) .args(args) .stdin(Stdio::null()) - .output() + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() .with_context(|| format!("starting Docker runner {}", self.docker_program.display()))?; - if !output.status.success() { - bail!( - "Docker runner exited with {}: {}", - output.status, - String::from_utf8_lossy(&output.stderr).trim() - ); + let stdout = child.stdout.take().context("Docker runner has no stdout pipe")?; + let stderr = child.stderr.take().context("Docker runner has no stderr pipe")?; + let stdout = thread::spawn(move || drain_tail(stdout, RUNNER_OUTPUT_TAIL_BYTES)); + let stderr = thread::spawn(move || drain_tail(stderr, RUNNER_OUTPUT_TAIL_BYTES)); + let status = child.wait().context("waiting for the Docker runner")?; + let (stdout_tail, stdout_total_bytes) = stdout + .join() + .map_err(|_| anyhow!("the runner stdout reader panicked"))? + .context("reading the runner stdout")?; + let (stderr_tail, stderr_total_bytes) = stderr + .join() + .map_err(|_| anyhow!("the runner stderr reader panicked"))? + .context("reading the runner stderr")?; + if status.success() { + return Ok(()); } - Ok(output) + #[cfg(unix)] + let signal = std::os::unix::process::ExitStatusExt::signal(&status); + #[cfg(not(unix))] + let signal = None; + Err(RunnerStepFailure { + subject, + runner: runner_id.to_owned(), + exit_code: status.code(), + signal, + stdout_tail, + stdout_total_bytes, + stderr_tail, + stderr_total_bytes, + } + .into()) } fn run_in_workspace( &self, + step_id: &str, + runner_id: &str, runner: &DockerRunnerBinding, workspace: &Path, working_directory: &Path, @@ -2197,8 +2368,7 @@ impl DockerRunnerDriver { build_machine_id_file(workspace)?; let spec = ContainerSpec::for_step(runner, required_environment, source_stamp)?; let args = docker_run_args(&spec, workspace, working_directory, argv)?; - self.docker(args)?; - Ok(()) + self.docker(RunnerSubject::Step(step_id.to_owned()), runner_id, args) } fn materialize_external_input( @@ -2240,7 +2410,11 @@ impl DockerRunnerDriver { input.url.clone(), ]; let args = docker_run_args(&spec, workspace, Path::new("."), &curl_argv)?; - self.docker(args)?; + self.docker( + RunnerSubject::ExternalInput(input.id.clone()), + &input.runner, + args, + )?; let bytes = fs::read(&destination).with_context(|| { format!( "reading materialized external input {}", @@ -2290,27 +2464,7 @@ impl DockerRunnerDriver { } ArtifactSource::WorktreeTree => (source.root.clone(), source.root.join(&artifact.source)), }; - ensure!(source_path.exists(), "declared artifact output is absent"); - let destination = staging_root.join(&artifact.destination); - ensure!( - destination.starts_with(staging_root), - "artifact destination escaped its release staging root" - ); - copy_artifact(&containment_root, &source_path, &destination)?; - let (sha256, size_bytes) = digest_artifact(&destination)?; - if let Some(expected) = &artifact.expected_sha256 { - ensure!( - &sha256 == expected, - "artifact output differs from its recipe-pinned digest" - ); - } - Ok(ArtifactReceipt { - artifact_id: artifact.id.clone(), - destination: artifact.destination.clone(), - sha256: format!("sha256-{sha256}"), - size_bytes, - executable: artifact.executable, - }) + seal_artifact(artifact, &containment_root, &source_path, staging_root) } } @@ -2382,6 +2536,8 @@ impl RunnerPort for DockerRunnerDriver { ); } self.run_in_workspace( + &step.id, + &step.runner, runner, workspace, &step.working_directory, @@ -2420,6 +2576,13 @@ impl RunnerPort for DockerRunnerDriver { } } +/// How every incarnation unit lives and dies. It runs once (`Restart=no`) +/// and systemd forgets it once it is inactive or failed (`--collect`): the +/// outcome is Idunn's to record in its own store, and a failed transient unit +/// left loaded is only clutter that marks the host degraded. +const INCARNATION_UNIT_LIFECYCLE: [&str; 3] = + ["--collect", "--property=Type=exec", "--property=Restart=no"]; + /// systemd owns process execution. This driver lowers one validated launch /// contract into a transient unit and then proves the native process and /// executable that systemd actually started. It does not decide admission, @@ -2467,7 +2630,6 @@ fn changed_systemd_workload_identity_fields( no_new_privileges, umask, inaccessible_paths, - load_credential, main_pid, process_start_time, process_uids, @@ -2867,7 +3029,6 @@ impl SystemdTransientWorkloadDriver { OsString::from("--property=NoNewPrivileges"), OsString::from("--property=UMask"), OsString::from("--property=InaccessiblePaths"), - OsString::from("--property=LoadCredential"), OsString::from("--property=OpenFile"), OsString::from("--property=WorkingDirectory"), OsString::from("--property=ControlGroup"), @@ -2979,38 +3140,7 @@ impl SystemdTransientWorkloadDriver { let umask = required_systemd_property(&values, "UMask")?.to_owned(); let inaccessible_paths = required_systemd_property(&values, "InaccessiblePaths")?.to_owned(); - let load_credential = required_systemd_property(&values, "LoadCredential")?.to_owned(); - ensure!( - service_type == "exec" - && restart_policy == "no" - && kill_mode == "mixed" - // `User` is deliberately not asserted here. Idunn never emits a - // User property, and systemd reports the identity it *allocated* - // for a DynamicUser unit -- the unit name, or a generated `_du…` - // when that is too long -- so the property is never empty for a - // running workload and cannot tell an allocated identity from a - // pinned one. The invariant it was reaching for, that the workload - // does not run as a persistent named account, is carried by - // `dynamic_user` below and by Idunn never passing User at all. - && dynamic_user - && supplementary_groups.is_empty() - && capability_bounding_set.is_empty() - && ambient_capabilities.is_empty() - && private_mounts - && private_pids - && protect_proc == "invisible" - && proc_subset == "all" - && no_new_privileges - && umask == "0007" - && inaccessible_paths == self.credential_root.display().to_string() - && load_credential - == if service_credential_names.is_empty() { - "" - } else { - "[unprintable]" - }, - "systemd workload isolation properties differ from the Idunn contract" - ); + require_systemd_isolation_contract(&values, &self.credential_root)?; let working_directory = PathBuf::from(required_systemd_property(&values, "WorkingDirectory")?); ensure!( @@ -3152,6 +3282,11 @@ impl SystemdTransientWorkloadDriver { ); Some(path) }; + require_delivered_credentials( + &process_root.join("root"), + unit, + service_credential_names, + )?; let activation_descriptor = &parent_only_file_descriptors[0]; ensure!( activation_descriptor.fd_name == IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME @@ -3259,7 +3394,7 @@ impl SystemdTransientWorkloadDriver { no_new_privileges, umask, inaccessible_paths, - load_credential, + load_credential: String::new(), main_pid, process_start_time, process_uids: process_security.uids, @@ -3366,10 +3501,32 @@ impl SystemdTransientWorkloadDriver { .is_none(), "operator binding attempts to replace the Idunn candidate bind" ); + let member_port = binding + .route + .as_ref() + .map(|binding_route| binding_route.member_candidate_port(port)) + .transpose()? + .flatten(); + match member_port { + Some(member_port) => ensure!( + environment + .insert( + IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT.into(), + format!("{host}:{member_port}"), + ) + .is_none(), + "operator binding attempts to replace the Idunn member bind" + ), + None => ensure!( + !environment.contains_key(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT), + "workload without a member listener carries an Idunn member bind" + ), + } } None => ensure!( - !environment.contains_key(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT), - "unrouted workload carries an Idunn candidate bind" + !environment.contains_key(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT) + && !environment.contains_key(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT), + "unrouted workload carries an Idunn route bind" ), } match &binding.process_write_lease { @@ -3451,12 +3608,7 @@ impl SystemdTransientWorkloadDriver { // workload. The same applies to `Group` when no fixed state group is // bound. In that case the numeric primary GID must match the allocated // UID; a fixed state group still requires both its exact name and GID. - let expected_load_credential = if regular_credential_names.is_empty() { - "" - } else { - "[unprintable]" - }; - let launch_contract: [(bool, &str); 27] = [ + let launch_contract: [(bool, &str); 26] = [ ( observation.unit_description == expected_description, "unit description", @@ -3496,10 +3648,6 @@ impl SystemdTransientWorkloadDriver { observation.inaccessible_paths == self.credential_root.display().to_string(), "inaccessible paths", ), - ( - observation.load_credential == expected_load_credential, - "load credential", - ), ( observation.working_directory == installed, "working directory", @@ -3625,8 +3773,9 @@ impl SystemdTransientWorkloadDriver { OsString::from("--no-ask-password"), OsString::from("--expand-environment=no"), OsString::from(format!("--unit={unit}")), - OsString::from("--property=Type=exec"), - OsString::from("--property=Restart=no"), + ]; + args.extend(INCARNATION_UNIT_LIFECYCLE.map(OsString::from)); + args.extend([ OsString::from("--property=KillMode=mixed"), OsString::from("--property=DynamicUser=yes"), OsString::from("--property=SupplementaryGroups="), @@ -3662,7 +3811,7 @@ impl SystemdTransientWorkloadDriver { OsString::from(format!("--working-directory={}", installed.display())), OsString::from(format!("--property=ReadOnlyPaths={}", installed.display())), OsString::from(format!("--property=ReadOnlyPaths={}", bundle.display())), - ]; + ]); for open_file in parent_only_open_files { args.push(OsString::from(format!("--property=OpenFile={open_file}"))); } @@ -4000,6 +4149,29 @@ impl WorkloadPort for SystemdTransientWorkloadDriver { Ok(unit.properties.get("ActiveState").map(String::as_str) == Some("failed")) } + fn lost_to_host_reboot( + &self, + observation: &WorkloadObservation, + admitted_at_unix_millis: u64, + ) -> Result { + let observation = observation.systemd()?; + if self.show_unit(&observation.unit)?.is_some() { + return Ok(false); + } + // A transient unit does not survive a reboot, so an absent unit is a + // reboot's loss only when the kernel booted after admission. + let stat = std::fs::read_to_string(self.proc_root.join("stat")) + .context("could not read the kernel boot time")?; + let btime = stat + .lines() + .find_map(|line| line.strip_prefix("btime ")) + .context("the kernel boot time is missing from stat")? + .trim() + .parse::() + .context("the kernel boot time is not a number")?; + Ok(btime.saturating_mul(1000) > admitted_at_unix_millis) + } + fn stop(&self, observation: &WorkloadObservation) -> Result<()> { let observation = observation.systemd()?; let Some(unit_observation) = self.show_unit(&observation.unit)? else { @@ -4076,6 +4248,20 @@ impl CultCacheWriteLeaseDriver { fn current(&self) -> Result> { validate_root_authority_path(&self.record_path)?; + self.read_record() + } + + /// The lease as its holder reads it: the same custody, envelope, key and + /// target validation Idunn applies, without requiring the reader to be + /// root. The holder is a service uid by construction; root is who writes. + pub(crate) fn current_for_holder( + &self, + ) -> Result> { + validate_authority_path_custody(&self.record_path)?; + self.read_record() + } + + fn read_record(&self) -> Result> { if !self.record_path.exists() { return Ok(None); } @@ -4122,36 +4308,61 @@ impl CultCacheWriteLeaseDriver { expected.validate()?; activation.validate()?; lease.validate()?; - let expected_sha256 = expected.canonical_sha256()?; - let activation_sha256 = activation.canonical_sha256()?; ensure!( - self.target == expected.target - && lease.target == expected.target - && lease.expected_projection_sha256 == expected_sha256 - && lease.plan_id == expected.plan_id - && lease.incarnation_id == expected.incarnation_id - && lease.sealed_release_id == expected.sealed_release_id - && lease.activation_witness_sha256 == activation_sha256 - && lease.runtime_id == expected.runtime_id - && lease.runtime_instance_id == activation.runtime_instance_id - && lease.state_schema_generation - == expected - .state_schema_generation - .as_deref() - .context("write-lease Expected has no state generation")? - && lease.state_contract_sha256 - == expected - .state_contract_sha256 - .as_deref() - .context("write-lease Expected has no state contract")? - && lease.warming_presence_sha256 == warming.signed_presence_sha256() - && warming.runtime_instance_id() == activation.runtime_instance_id.as_str(), + self.target == expected.target, "process write lease does not bind the exact warming candidate" ); - Ok(()) + ensure!( + warming.runtime_instance_id() == activation.runtime_instance_id.as_str(), + "process write lease does not bind the exact warming candidate" + ); + lease_binds_candidate( + expected, + activation, + lease, + std::slice::from_ref(&warming.signed_presence_sha256().to_owned()), + ) } } +/// The one rule for what makes a process write lease this incarnation's: it +/// binds this Expected, this activation and one of the given provider-signed +/// Warming proofs. Idunn grants with it and the lease holder accepts with it. +pub(crate) fn lease_binds_candidate( + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + lease: &IdunnProcessWriteLeaseRecord, + warming_presence_sha256es: &[String], +) -> Result<()> { + let expected_sha256 = expected.canonical_sha256()?; + let activation_sha256 = activation.canonical_sha256()?; + ensure!( + lease.target == expected.target + && lease.expected_projection_sha256 == expected_sha256 + && lease.plan_id == expected.plan_id + && lease.incarnation_id == expected.incarnation_id + && lease.sealed_release_id == expected.sealed_release_id + && lease.activation_witness_sha256 == activation_sha256 + && lease.runtime_id == expected.runtime_id + && lease.runtime_instance_id == activation.runtime_instance_id + && lease.state_schema_generation + == expected + .state_schema_generation + .as_deref() + .context("write-lease Expected has no state generation")? + && lease.state_contract_sha256 + == expected + .state_contract_sha256 + .as_deref() + .context("write-lease Expected has no state contract")? + && warming_presence_sha256es + .iter() + .any(|sha256| sha256 == &lease.warming_presence_sha256), + "process write lease does not bind the exact warming candidate" + ); + Ok(()) +} + impl WriteLeasePort for CultCacheWriteLeaseDriver { fn revoke_exact(&self, incumbent: Option<&IdunnProcessWriteLeaseRecord>) -> Result<()> { if let Some(incumbent) = incumbent { @@ -4247,6 +4458,10 @@ impl WriteLeasePort for CultCacheWriteLeaseDriver { /// facts here. Service presence is absent by construction; only Odin may /// correlate these records with signed runtime observation into Present/Ready. /// +/// The store is published 0644 and read lock-free by Odin and Idunn; its lock +/// is Idunn's alone (`exchange_behind_private_lock`), so no other uid can hold +/// it and stall a publish. +/// /// Every record is keyed by the incarnation it describes, never by the target /// alone. A target being replaced has two incarnations at once -- the admitted /// incumbent and the sealed candidate -- and both are projected side by side. @@ -4390,22 +4605,28 @@ impl CultCacheTopologyDriver { /// Apply one whole-snapshot mutation with compare-and-swap. `mutate` /// returns `None` when the snapshot already has the shape it wants, and /// the replacement set otherwise; every writer below is one of these. + /// + /// Never waits: a publish is done exactly when this returns `Ok`. Only a + /// lost race (the store moved on) is retried at once; a held lock is an + /// error and a no-op, retried by the caller's own retry. fn mutate(&self, mutate: F) -> Result<()> where F: Fn(&[CultCacheEnvelope]) -> Result>>, { - if let Some(parent) = self.projection_store.parent() { - fs::create_dir_all(parent)?; - } - let store = SingleFileMessagePackBackingStore::new(&self.projection_store); for _ in 0..8 { let entries = self.snapshot()?; let Some(replacement) = mutate(&entries)? else { return Ok(()); }; - if store.compare_exchange_snapshot(&entries, &replacement)? { - publish_projection_mode(&self.projection_store)?; - return Ok(()); + match exchange_behind_private_lock(&self.projection_store, &entries, &replacement)? { + TryCompareExchangeSnapshotOutcome::Exchanged => { + publish_file_mode(&self.projection_store)?; + return Ok(()); + } + TryCompareExchangeSnapshotOutcome::Mismatch => {} + TryCompareExchangeSnapshotOutcome::LockContended => { + return Err(LockContended { store: "topology projection" }.into()); + } } } bail!("CultCache projection changed repeatedly during publication") @@ -5097,6 +5318,22 @@ pub enum RouteActuation { Survival, } +/// A publish that lost to a held projection-directory lock. Transient by +/// kind: the holder lets go, and the caller's own retry decides what a later +/// attempt does. It names the store kind, never its path. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct LockContended { + pub(crate) store: &'static str, +} + +impl std::fmt::Display for LockContended { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "the {} is locked by another holder", self.store) + } +} + +impl std::error::Error for LockContended {} + /// A forward route actuation the ceiling refused. `reopens_at_unix_millis` is /// when the oldest counted actuation leaves the window. #[derive(Debug, Clone, PartialEq, Eq)] @@ -5142,50 +5379,67 @@ impl NginxRouteDriver { } } - /// The one firewall rule this route owns: inbound to the stable endpoint's - /// address and port, its transport's protocol, tagged with the route id. - fn endpoint_rule(&self) -> Result> { - let (host, port) = self.binding.stable_socket()?; + /// The firewall rules this route owns, one per stable socket (the + /// presence endpoint, then the member endpoint when the binding has one): + /// inbound to the socket's address and port, its transport's protocol. + fn endpoint_rules(&self) -> Result>> { let protocol = match self.binding.driver { RouteDriver::NginxStreamTcp => "tcp", RouteDriver::NginxStreamUdp => "udp", }; - Ok(vec![ - OsString::from("allow"), - OsString::from("in"), - OsString::from("to"), - OsString::from(host.to_string()), - OsString::from("port"), - OsString::from(port.to_string()), - OsString::from("proto"), - OsString::from(protocol), - ]) + let mut sockets = vec![self.binding.stable_socket()?]; + sockets.extend(self.binding.member_stable_socket()?); + Ok(sockets + .into_iter() + .map(|(host, port)| { + vec![ + OsString::from("allow"), + OsString::from("in"), + OsString::from("to"), + OsString::from(host.to_string()), + OsString::from("port"), + OsString::from(port.to_string()), + OsString::from("proto"), + OsString::from(protocol), + ] + }) + .collect()) } fn admit_endpoint(&self) -> Result<()> { - let mut args = self.endpoint_rule()?; - args.push(OsString::from("comment")); - args.push(OsString::from(format!( - "Idunn route {}", - self.binding.route_id - ))); - self.command(&self.ufw_program, args) - .context("admitting the stable endpoint on the host firewall")?; + for mut args in self.endpoint_rules()? { + args.push(OsString::from("comment")); + args.push(OsString::from(format!( + "Idunn route {}", + self.binding.route_id + ))); + self.command(&self.ufw_program, args) + .context("admitting a route endpoint on the host firewall")?; + } Ok(()) } fn withdraw_endpoint(&self) -> Result<()> { - let mut args = vec![OsString::from("delete")]; - args.extend(self.endpoint_rule()?); - match self.command(&self.ufw_program, args) { - Ok(_) => Ok(()), - // ufw reports a rule that is already gone as a failure; for a - // withdrawal that is the state being asked for. - Err(error) if format!("{error:#}").contains("non-existent") => Ok(()), - Err(error) => { - Err(error).context("withdrawing the stable endpoint from the host firewall") + let mut first_error = None; + for rule in self.endpoint_rules()? { + let mut args = vec![OsString::from("delete")]; + args.extend(rule); + match self.command(&self.ufw_program, args) { + Ok(_) => {} + // ufw reports a rule that is already gone as a failure; for a + // withdrawal that is the state being asked for. + Err(error) if format!("{error:#}").contains("non-existent") => {} + Err(error) => { + first_error.get_or_insert( + error.context("withdrawing a route endpoint from the host firewall"), + ); + } } } + match first_error { + Some(error) => Err(error), + None => Ok(()), + } } fn command(&self, program: &Path, args: I) -> Result @@ -5246,8 +5500,28 @@ impl NginxRouteDriver { RouteDriver::NginxStreamTcp => { let (stable_host, stable_port) = endpoint_host_port(&route.stable_endpoint, endpoint_prefix)?; + let member_blocks = match ( + self.binding.member_stable_socket()?, + self.binding.member_candidate_port(candidate_port)?, + ) { + (Some((member_host, member_stable_port)), Some(member_port)) => { + let member_upstream = format!("{upstream}_member"); + let max_connections = self + .binding + .member + .as_ref() + .map_or(0, |member| member.max_connections); + // One constant key: the limit is on the route's members + // together, so a flood of them can never take the + // connections the presence block needs. + format!( + "limit_conn_zone \"{member_upstream}\" zone={member_upstream}:64k;\nupstream {member_upstream} {{\n server {candidate_host}:{member_port};\n}}\nserver {{\n listen {member_host}:{member_stable_port};\n limit_conn {member_upstream} {max_connections};\n proxy_pass {member_upstream};\n}}\n" + ) + } + _ => String::new(), + }; format!( - "# Idunn Expected {expected_projection_sha256}\nupstream {upstream} {{\n server {candidate_host}:{candidate_port};\n}}\nserver {{\n listen {stable_host}:{stable_port};\n proxy_pass {upstream};\n}}\n" + "# Idunn Expected {expected_projection_sha256}\nupstream {upstream} {{\n server {candidate_host}:{candidate_port};\n}}\nserver {{\n listen {stable_host}:{stable_port};\n proxy_pass {upstream};\n}}\n{member_blocks}" ) } RouteDriver::NginxStreamUdp => { @@ -5801,7 +6075,7 @@ impl Answer { } } -fn request_tcp_snapshot(target: SocketAddr, payload: &[u8]) -> Result, ChallengeFailure> { +pub(crate) fn request_tcp_snapshot(target: SocketAddr, payload: &[u8]) -> Result, ChallengeFailure> { let mut stream = connect_route_socket(target) .with_context(|| format!("connecting stable CultNet TCP route {target}")) .map_err(silent)?; @@ -6010,7 +6284,7 @@ fn connect_route_socket(target: SocketAddr) -> std::io::Result { } } -fn exact_route_snapshot_response( +pub(crate) fn exact_route_snapshot_response( response: CultNetMessage, message_id: &str, target: &str, @@ -6651,21 +6925,7 @@ fn hash_frozen_source_tree(root: &Path, current: &Path, hasher: &mut Sha256) -> ); hasher.update(b"\0"); hasher.update(metadata.len().to_le_bytes()); - // S9: streamed in fixed-size chunks rather than `fs::read`ing the - // whole file, so peak RSS is bounded by the buffer, not by the - // largest file in the tree. - let mut file = fs::File::open(&path) - .with_context(|| format!("opening {} to hash it", path.display()))?; - let mut buffer = [0u8; 1 << 16]; - loop { - let read = file - .read(&mut buffer) - .with_context(|| format!("reading {} to hash it", path.display()))?; - if read == 0 { - break; - } - hasher.update(&buffer[..read]); - } + hash_file_exact(&path, metadata.len(), hasher)?; } else if metadata.file_type().is_symlink() { let target = fs::read_link(&path)?; hasher.update(b"link\0"); @@ -6687,20 +6947,28 @@ fn frozen_source_sha256(_root: &Path) -> Result { #[cfg(unix)] fn harden_installed_release(root: &Path, artifacts: &[ArtifactReceipt]) -> Result<()> { - let executable_paths = artifacts - .iter() - .filter(|artifact| artifact.executable) - .map(|artifact| root.join(&artifact.destination)) - .collect::>(); - harden_root_tree(root, root, &executable_paths) + let mut executable_paths = BTreeSet::new(); + for artifact in artifacts { + let destination = root.join(&artifact.destination); + for member in &artifact.executable_members { + executable_paths.insert(destination.join(member)); + } + if artifact.executable { + executable_paths.insert(destination); + } + } + harden_root_tree(root, &mut executable_paths)?; + ensure!( + executable_paths.is_empty(), + "declared executable is missing or not a regular file" + ); + Ok(()) } +/// Each regular file the walk seals 0555 is removed from `executable_paths`; +/// whatever remains was declared executable and never sealed. #[cfg(unix)] -fn harden_root_tree( - root: &Path, - current: &Path, - executable_paths: &std::collections::BTreeSet, -) -> Result<()> { +fn harden_root_tree(current: &Path, executable_paths: &mut BTreeSet) -> Result<()> { use std::os::unix::fs::{MetadataExt, PermissionsExt}; let metadata = fs::symlink_metadata(current)?; @@ -6712,48 +6980,28 @@ fn harden_root_tree( let mut entries = fs::read_dir(current)?.collect::>>()?; entries.sort_by_key(|entry| entry.file_name()); for entry in entries { - harden_root_tree(root, &entry.path(), executable_paths)?; + harden_root_tree(&entry.path(), &mut *executable_paths)?; } fs::set_permissions(current, fs::Permissions::from_mode(0o555))?; } else if metadata.is_file() { fs::set_permissions( current, - fs::Permissions::from_mode(if executable_paths.contains(current) { + fs::Permissions::from_mode(if executable_paths.remove(current) { 0o555 } else { 0o444 }), )?; } else if metadata.file_type().is_symlink() { - validate_release_symlink(root, current)?; + // Left untouched: containment was decided by check_release_link in + // digest_tree, which install_release and validate_installed_release + // run over every receipt before this walk (map T16). } else { bail!("sealed release contains a special filesystem entry") } Ok(()) } -#[cfg(unix)] -fn validate_release_symlink(root: &Path, path: &Path) -> Result<()> { - let target = fs::read_link(path)?; - ensure!(target.is_relative(), "sealed release symlink is absolute"); - ensure!( - target.components().all(|component| matches!( - component, - std::path::Component::CurDir | std::path::Component::Normal(_) - )), - "sealed release symlink has a parent traversal" - ); - let destination = path - .parent() - .context("sealed release symlink has no parent")? - .join(target); - ensure!( - destination.starts_with(root), - "sealed release symlink escaped its release" - ); - Ok(()) -} - #[cfg(not(unix))] fn harden_installed_release(_root: &Path, _artifacts: &[ArtifactReceipt]) -> Result<()> { bail!("systemd workload installation requires Unix permissions") @@ -7156,6 +7404,44 @@ fn path_inside_process_root(process_root: &Path, absolute_path: &Path) -> Result Ok(process_root.join(absolute_path.strip_prefix(Path::new("/"))?)) } +/// The workload holds exactly the credentials Idunn delivered: the entries of +/// `/run/credentials/` in the MainPID's root (`process_root` is that root) +/// equal `expected`, and an absent directory is the empty set. Only entry names +/// are read; a credential is never opened here. The message names no credential. +fn require_delivered_credentials( + process_root: &Path, + unit: &str, + expected: &[String], +) -> Result<()> { + let directory = path_inside_process_root( + process_root, + &Path::new("/run/credentials").join(unit), + )?; + let mut delivered = Vec::new(); + match fs::read_dir(&directory) { + Ok(entries) => { + for entry in entries { + let name = entry + .context("reading a delivered credential entry")? + .file_name() + .into_string() + .map_err(|_| anyhow!("delivered credential name is not UTF-8"))?; + delivered.push(name); + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).context("reading the delivered credential directory"); + } + } + delivered.sort(); + ensure!( + delivered == expected, + "systemd delivered credentials that differ from the Idunn launch" + ); + Ok(()) +} + fn linux_process_security( status_path: &Path, expected_host_pid: u32, @@ -7371,6 +7657,56 @@ fn parse_systemd_boolean(values: &BTreeMap, name: &str) -> Resul } } +/// The isolation properties of a transient workload unit, judged from `systemctl +/// show`. `LoadCredential` is absent on purpose: systemd 257 displays it as +/// `[unprintable]` whatever the unit holds, so what the workload was given is +/// judged by `require_delivered_credentials` from the delivered directory. +fn require_systemd_isolation_contract( + values: &BTreeMap, + credential_root: &Path, +) -> Result<()> { + let service_type = required_systemd_property(values, "Type")?; + let restart_policy = required_systemd_property(values, "Restart")?; + let kill_mode = required_systemd_property(values, "KillMode")?; + let dynamic_user = parse_systemd_boolean(values, "DynamicUser")?; + let supplementary_groups = systemd_property(values, "SupplementaryGroups")?; + let capability_bounding_set = systemd_property(values, "CapabilityBoundingSet")?; + let ambient_capabilities = systemd_property(values, "AmbientCapabilities")?; + let private_mounts = parse_systemd_boolean(values, "PrivateMounts")?; + let private_pids = parse_systemd_boolean(values, "PrivatePIDs")?; + let protect_proc = required_systemd_property(values, "ProtectProc")?; + let proc_subset = required_systemd_property(values, "ProcSubset")?; + let no_new_privileges = parse_systemd_boolean(values, "NoNewPrivileges")?; + let umask = required_systemd_property(values, "UMask")?; + let inaccessible_paths = required_systemd_property(values, "InaccessiblePaths")?; + ensure!( + service_type == "exec" + && restart_policy == "no" + && kill_mode == "mixed" + // `User` is deliberately not asserted here. Idunn never emits a + // User property, and systemd reports the identity it *allocated* + // for a DynamicUser unit -- the unit name, or a generated `_du…` + // when that is too long -- so the property is never empty for a + // running workload and cannot tell an allocated identity from a + // pinned one. The invariant it was reaching for, that the workload + // does not run as a persistent named account, is carried by + // `dynamic_user` below and by Idunn never passing User at all. + && dynamic_user + && supplementary_groups.is_empty() + && capability_bounding_set.is_empty() + && ambient_capabilities.is_empty() + && private_mounts + && private_pids + && protect_proc == "invisible" + && proc_subset == "all" + && no_new_privileges + && umask == "0007" + && inaccessible_paths == credential_root.display().to_string(), + "systemd workload isolation properties differ from the Idunn contract" + ); + Ok(()) +} + fn read_process_environment(path: &Path) -> Result>> { let bytes = fs::read(path) .with_context(|| format!("reading process environment {}", path.display()))?; @@ -7695,31 +8031,112 @@ fn ensure_bundle_is_reachable_by_workload( Ok(()) } -/// The topology store is Idunn's *published* surface: every managed target -/// reads it to verify its own Expected incarnation against the Idunn anchor. -/// -/// Idunn runs with `UMask=027`, which is right for its private state and wrong -/// for this one file -- it lands `0640 root:root`, and a `DynamicUser` workload -/// gets EACCES. A chmod by hand does not hold, because each publish writes a -/// new file. Integrity here comes from the signatures over the records, not -/// from the mode, so the published copy is readable. -fn publish_projection_mode(path: &Path) -> Result<()> { +/// One published file: `0644`, because Idunn's `UMask=027` would land it +/// `0640 root:root` and deny its readers. A chmod by hand does not hold, since +/// each publish writes a new file. +pub(crate) fn publish_file_mode(path: &Path) -> Result<()> { #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; - // The lock sibling too: CultCache opens it alongside the store, so a - // 0640 lock denies the read just as surely as a 0640 store, and it is - // created fresh under Idunn's umask on every publish. - for path in [path.to_path_buf(), authority_lock_path(path)] { - if path.exists() { - fs::set_permissions(&path, fs::Permissions::from_mode(0o644)) - .with_context(|| format!("publishing {}", path.display()))?; - } + if path.exists() { + fs::set_permissions(path, fs::Permissions::from_mode(0o644)) + .with_context(|| format!("publishing {}", path.display()))?; } } Ok(()) } +/// The one write primitive for a store in the projection directory: one +/// nonblocking compare-exchange of the whole store against the snapshot it was +/// decided on, behind a lock only Idunn can hold. +/// +/// Readers of a published store take no lock, so nobody but Idunn needs to +/// open its lock, and a lock another uid can open is a lock another uid can +/// hold to stall every publish. CultCache creates a lock with mode 0666, so +/// the directory's default ACL would decide who may (Yggdrasil's lands it +/// 0644). This makes the lock private first, by this rule: an absent lock is +/// created `O_EXCL` 0600; a lock that is not (a regular file, ours, 0600) is +/// replaced by a fresh 0600 file renamed over it, because a chmod does not +/// revoke a descriptor another uid already holds, while the rename leaves that +/// holder only an orphaned inode. A private lock is opened as it is. No +/// existing lock is ever chmodded. Nothing here names a path in an error. +pub(crate) fn exchange_behind_private_lock( + store: &Path, + expected: &[CultCacheEnvelope], + replacements: &[CultCacheEnvelope], +) -> Result { + make_lock_private(store)?; + SingleFileMessagePackBackingStore::new(store) + .try_compare_exchange_snapshot(expected, replacements) +} + +#[cfg(unix)] +fn make_lock_private(store: &Path) -> Result<()> { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let lock = authority_lock_path(store); + if let Some(parent) = lock.parent() { + fs::create_dir_all(parent).context("creating the store directory")?; + } + // SAFETY: geteuid has no preconditions. + let euid = unsafe { libc::geteuid() }; + // The daemon is the only writer, so a lost create race is defensive only. + for _ in 0..3 { + match fs::symlink_metadata(&lock) { + Ok(metadata) + if metadata.is_file() + && metadata.uid() == euid + && metadata.permissions().mode() & 0o777 == 0o600 => + { + return Ok(()); + } + Ok(_) => return replace_with_private_lock(&lock), + Err(error) if error.kind() == ErrorKind::NotFound => { + match create_private_file(&lock) { + Ok(()) => return Ok(()), + Err(error) if error.kind() == ErrorKind::AlreadyExists => {} + Err(error) => return Err(error).context("creating a private store lock"), + } + } + Err(error) => return Err(error).context("inspecting a store lock"), + } + } + bail!("a store lock kept changing while it was made private") +} + +#[cfg(not(unix))] +fn make_lock_private(store: &Path) -> Result<()> { + let _ = store; + Ok(()) +} + +/// A new file nothing else has open, mode 0600 (the create mode is intersected +/// with the umask and the directory's default ACL, which only narrow it). +#[cfg(unix)] +fn create_private_file(path: &Path) -> std::io::Result<()> { + use std::os::unix::fs::OpenOptionsExt; + + OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(path) + .map(drop) +} + +/// Stage a fresh private file beside `lock` and rename it over the lock. +#[cfg(unix)] +fn replace_with_private_lock(lock: &Path) -> Result<()> { + let mut staged = lock.as_os_str().to_os_string(); + staged.push(format!(".{}", Uuid::new_v4().simple())); + let staged = PathBuf::from(staged); + create_private_file(&staged).context("staging a private store lock")?; + fs::rename(&staged, lock).map_err(|error| { + let _ = fs::remove_file(&staged); + anyhow::Error::new(error).context("replacing a store lock with a private one") + }) +} + fn build_machine_id(workspace: &Path) -> Result { let text = workspace .to_str() @@ -7862,7 +8279,7 @@ fn copy_symlink(source: &Path, destination: &Path) -> Result<()> { /// checked to contain the fully resolved `source` with the same /// whole-chain-aware resolver `validate_frozen_source_symlink` uses, before /// anything is read from it. -pub(crate) fn copy_artifact(root: &Path, source: &Path, destination: &Path) -> Result<()> { +fn copy_artifact(root: &Path, source: &Path, destination: &Path) -> Result<()> { let canonical_root = root.canonicalize().context("resolving artifact root")?; let canonical_source = source .canonicalize() @@ -7885,11 +8302,124 @@ pub(crate) fn copy_artifact(root: &Path, source: &Path, destination: &Path) -> R } } +/// The one seal-time owner of mode declarations for a copied artifact: a +/// directory output is never `executable` (it declares `executable_members`), +/// a file output never declares members, and every member is a regular file +/// reached through real directories, never through a symlink. Errors name the +/// rule, never the member path. +fn check_artifact_modes(artifact: &ArtifactOutput, destination: &Path) -> Result<()> { + let is_dir = fs::symlink_metadata(destination) + .context("inspecting a copied artifact")? + .is_dir(); + ensure!( + !(is_dir && artifact.executable), + "directory artifact {} cannot be executable; declare executable_members", + artifact.id + ); + if !is_dir { + ensure!( + artifact.executable_members.is_empty(), + "file artifact {} cannot declare executable_members", + artifact.id + ); + return Ok(()); + } + for member in &artifact.executable_members { + ensure!(!member.as_os_str().is_empty(), "executable member is empty"); + let mut current = destination.to_path_buf(); + let mut components = member.components().peekable(); + while let Some(component) = components.next() { + let std::path::Component::Normal(name) = component else { + bail!("executable member has a forbidden path component"); + }; + current.push(name); + let metadata = match fs::symlink_metadata(¤t) { + Ok(metadata) => metadata, + Err(error) if error.kind() == ErrorKind::NotFound => { + bail!("executable member is missing") + } + Err(error) => return Err(error).context("inspecting an executable member"), + }; + if components.peek().is_some() { + ensure!( + metadata.is_dir(), + "executable member passes through a symlink or non-directory" + ); + } else { + ensure!(metadata.is_file(), "executable member is not a regular file"); + } + } + } + Ok(()) +} + +/// The one seal step for a declared artifact, for systemd and host targets +/// alike: copy it into the staging root, check its mode declarations, digest it +/// and check the recipe pin. Modes are applied by harden on systemd targets; +/// host targets (Windows, map T15) carry `executable` and `executable_members` +/// in the receipt and do not apply them. +pub(crate) fn seal_artifact( + artifact: &ArtifactOutput, + containment_root: &Path, + source_path: &Path, + staging_root: &Path, +) -> Result { + ensure!( + source_path.exists(), + "declared artifact output {} is absent", + artifact.id + ); + let destination = staging_root.join(&artifact.destination); + ensure!( + destination.starts_with(staging_root), + "artifact {} destination escaped its staging root", + artifact.id + ); + copy_artifact(containment_root, source_path, &destination)?; + check_artifact_modes(artifact, &destination)?; + let (sha256, size_bytes) = digest_artifact(&destination)?; + if let Some(expected) = &artifact.expected_sha256 { + ensure!( + &sha256 == expected, + "artifact {} differs from its recipe-pinned digest", + artifact.id + ); + } + Ok(ArtifactReceipt { + artifact_id: artifact.id.clone(), + destination: artifact.destination.clone(), + sha256: format!("sha256-{sha256}"), + size_bytes, + executable: artifact.executable, + executable_members: artifact.executable_members.clone(), + }) +} + +/// Streams `path` into `hasher` in 64 KiB chunks, so peak memory is the buffer +/// and not the largest file in the tree, and refuses if the bytes read differ +/// from `expected_len`. The only owner of the length check. +pub(crate) fn hash_file_exact(path: &Path, expected_len: u64, hasher: &mut Sha256) -> Result<()> { + let mut file = fs::File::open(path).context("opening a file to hash it")?; + let mut buffer = [0u8; 1 << 16]; + let mut total = 0_u64; + loop { + let read = file.read(&mut buffer).context("reading a file to hash it")?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + total += read as u64; + } + ensure!(total == expected_len, "file changed while hashing"); + Ok(()) +} + pub(crate) fn digest_artifact(path: &Path) -> Result<(String, u64)> { let metadata = fs::symlink_metadata(path)?; if metadata.is_file() { - let bytes = fs::read(path)?; - return Ok((raw_sha256(&bytes), bytes.len().try_into()?)); + let mut hasher = Sha256::new(); + hash_file_exact(path, metadata.len(), &mut hasher)?; + return Ok((format!("{:x}", hasher.finalize()), metadata.len())); } ensure!(metadata.is_dir(), "artifact is not a file or directory"); let mut hasher = Sha256::new(); @@ -7913,15 +8443,16 @@ fn digest_tree(root: &Path, current: &Path, hasher: &mut Sha256, size: &mut u64) hasher.update(b"\0"); digest_tree(root, &path, hasher, size)?; } else if metadata.is_file() { - let bytes = fs::read(&path)?; + let length = metadata.len(); hasher.update(b"file\0"); hasher.update(relative.as_bytes()); hasher.update(b"\0"); - hasher.update((bytes.len() as u64).to_le_bytes()); - hasher.update(&bytes); - *size = size.saturating_add(bytes.len().try_into()?); + hasher.update(length.to_le_bytes()); + hash_file_exact(&path, length, hasher)?; + *size = size.saturating_add(length); } else if metadata.file_type().is_symlink() { let target = fs::read_link(&path)?; + check_release_link(path.strip_prefix(root)?, &target)?; hasher.update(b"link\0"); hasher.update(relative.as_bytes()); hasher.update(b"\0"); @@ -7934,6 +8465,35 @@ fn digest_tree(root: &Path, current: &Path, hasher: &mut Sha256, size: &mut u64) Ok(()) } +/// The one judge of link containment in a release tree, pure and static: +/// `link` is the link's path relative to its artifact's top and `target` what +/// it reads. A target is relative, its `..` steps all come first, and they +/// climb no higher than the link's own directory depth, so the link can only +/// land inside the artifact. Lexical normalisation of parent and target is not +/// enough: with `s -> ..` it admits `s/../x` and escapes (map T14). +fn check_release_link(link: &Path, target: &Path) -> Result<()> { + use std::path::Component; + + let depth = link.components().count().saturating_sub(1); + let mut climbs = 0_usize; + let mut named = false; + for component in target.components() { + match component { + Component::CurDir => {} + Component::RootDir | Component::Prefix(_) => { + bail!("sealed release symlink is absolute") + } + Component::ParentDir => { + ensure!(!named, "sealed release symlink climbs after a name"); + climbs += 1; + } + Component::Normal(_) => named = true, + } + } + ensure!(climbs <= depth, "sealed release symlink escapes its artifact"); + Ok(()) +} + pub(crate) fn raw_sha256(bytes: &[u8]) -> String { format!("{:x}", Sha256::digest(bytes)) } @@ -8186,7 +8746,7 @@ fn ensure_source_directory(path: &Path, identity: Option) -> Re } } -fn authority_lock_path(path: &Path) -> PathBuf { +pub(crate) fn authority_lock_path(path: &Path) -> PathBuf { let mut lock = path.as_os_str().to_os_string(); lock.push(".lock"); PathBuf::from(lock) @@ -8194,12 +8754,17 @@ fn authority_lock_path(path: &Path) -> PathBuf { #[cfg(unix)] fn validate_root_authority_path(path: &Path) -> Result<()> { - use std::os::unix::fs::{MetadataExt, PermissionsExt}; - ensure!( unsafe { libc::geteuid() } == 0, "process write-lease actuation requires root Idunn" ); + validate_authority_path_custody(path) +} + +#[cfg(unix)] +fn validate_authority_path_custody(path: &Path) -> Result<()> { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + let parent = path .parent() .context("process write-lease path has no parent")?; @@ -8242,6 +8807,11 @@ fn validate_root_authority_path(_path: &Path) -> Result<()> { bail!("process write-lease actuation requires a Unix authority path") } +#[cfg(not(unix))] +fn validate_authority_path_custody(_path: &Path) -> Result<()> { + bail!("process write-lease actuation requires a Unix authority path") +} + #[cfg(unix)] fn harden_root_authority_file(path: &Path) -> Result<()> { use std::os::unix::fs::{MetadataExt, PermissionsExt}; @@ -8508,6 +9078,7 @@ mod tests { private_port_end: port, config_path: temp.path().join("service.conf"), reload_unit: "nginx.service".into(), + member: None, }) .request_candidate_runtime_presence(&candidate, "candidate-probe") } @@ -9331,6 +9902,7 @@ Content-Le".to_vec()), |_| {}), private_port_end: 17998, config_path: temp.path().join("service.conf"), reload_unit: "nginx.service".into(), + member: None, }); let mut candidate = expected(); candidate.route = Some(cultnet_rs::IdunnExpectedRoute { @@ -9358,21 +9930,38 @@ Content-Le".to_vec()), |_| {}), let address = reservation.local_addr()?; drop(reservation); + // The listener never accepts: the kernel completes a handshake from + // its backlog, so the client connects as soon as it is bound. Nothing + // here can block on a connection that never arrives; the listener only + // has to outlive the connect, and gives up waiting after a bound. + let (connected, connect_done) = std::sync::mpsc::channel::<()>(); let server = thread::spawn(move || -> std::io::Result<()> { thread::sleep(Duration::from_millis(75)); let listener = std::net::TcpListener::bind(address)?; - let (_stream, _) = listener.accept()?; + let _ = connect_done.recv_timeout(Duration::from_secs(30)); + drop(listener); Ok(()) }); - let stream = connect_route_socket(address)?; - drop(stream); + let stream = connect_route_socket(address); + let _ = connected.send(()); server .join() .map_err(|_| anyhow!("route readiness test server panicked"))??; + drop(stream?); Ok(()) } + /// An incarnation unit runs once and is collected when it ends, so a + /// failed candidate does not stay loaded and mark the host degraded. + /// Collection is safe because a forgotten unit already reads as + /// permanently stopped (the test below). + #[test] + fn incarnation_units_run_once_and_are_collected() { + assert!(INCARNATION_UNIT_LIFECYCLE.contains(&"--collect")); + assert!(INCARNATION_UNIT_LIFECYCLE.contains(&"--property=Restart=no")); + } + /// Admission audit, claim 3: `is_permanently_stopped` now lets the /// Isolation gate skip `prove_isolation` against the incumbent. Enumerate /// what it answers `true` for, using a fake `systemctl show` that replays @@ -9462,6 +10051,75 @@ Content-Le".to_vec()), |_| {}), Ok(()) } + /// Continuity's one free re-materialization rests on this: an absent unit + /// is a reboot's loss only when the kernel booted after admission. + #[cfg(unix)] + #[test] + fn lost_to_host_reboot_needs_an_absent_unit_and_a_boot_after_admission() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir()?; + let systemctl = temp.path().join("systemctl"); + let state = temp.path().join("systemctl.state"); + std::fs::write(&systemctl, "#!/bin/sh +cat \"$0.state\" +exit 0 +")?; + std::fs::set_permissions(&systemctl, std::fs::Permissions::from_mode(0o755))?; + let proc_root = temp.path().join("proc"); + std::fs::create_dir(&proc_root)?; + let driver = SystemdTransientWorkloadDriver { + systemctl_program: systemctl, + proc_root: proc_root.clone(), + ..Default::default() + }; + let observation = audit_workload_observation("no"); + let absent = "LoadState=not-found +ActiveState=inactive +SubState=dead +"; + let failed = "LoadState=loaded +ActiveState=failed +SubState=failed +InvocationID=inv +Description=Idunn test +"; + let boot = |btime: &str| -> Result<()> { + Ok(std::fs::write( + proc_root.join("stat"), + format!("cpu 1 2 3 +btime {btime} +processes 9 +"), + )?) + }; + let admitted_at = 1_700_000_000_000_u64; + + std::fs::write(&state, absent)?; + boot("1700000001")?; + assert!(driver.lost_to_host_reboot(&observation, admitted_at)?); + boot("1700000000")?; + assert!(!driver.lost_to_host_reboot(&observation, admitted_at)?, "booted at admission"); + boot("1699999999")?; + assert!(!driver.lost_to_host_reboot(&observation, admitted_at)?, "booted before admission"); + + // A unit systemd still knows died on its own, not to a reboot. + std::fs::write(&state, failed)?; + boot("1700000001")?; + assert!(!driver.lost_to_host_reboot(&observation, admitted_at)?); + + // No boot time is not evidence either way. + std::fs::write(&state, absent)?; + std::fs::write(proc_root.join("stat"), "cpu 1 2 3 +")?; + assert!(driver.lost_to_host_reboot(&observation, admitted_at).is_err()); + boot("soon")?; + assert!(driver.lost_to_host_reboot(&observation, admitted_at).is_err()); + std::fs::remove_file(proc_root.join("stat"))?; + assert!(driver.lost_to_host_reboot(&observation, admitted_at).is_err()); + Ok(()) + } + fn host_audit_observation(pid: u32, created: u64) -> HostWorkloadObservation { HostWorkloadObservation { host: "raven".into(), @@ -9543,6 +10201,168 @@ Content-Le".to_vec()), |_| {}), ); } + #[cfg(unix)] + fn fixture_show_properties(text: &str) -> Result> { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir()?; + let shown = temp.path().join("shown"); + std::fs::write(&shown, text)?; + let systemctl = temp.path().join("systemctl"); + std::fs::write( + &systemctl, + format!("#!/bin/sh\ncat '{}'\nexit 0\n", shown.display()), + )?; + std::fs::set_permissions(&systemctl, std::fs::Permissions::from_mode(0o755))?; + let driver = SystemdTransientWorkloadDriver { + systemctl_program: systemctl, + ..Default::default() + }; + Ok(driver + .show_unit("idunn-test.service")? + .context("fixture unit was not found")? + .properties) + } + + /// systemd 257 prints `LoadCredential=[unprintable]` for every unit, so the + /// isolation contract must admit the real output of a unit with no service + /// credential as well as one with one. The inputs are real captures, not a + /// re-spelling. + #[cfg(unix)] + #[test] + fn systemd_257_isolation_contract_admits_a_presence_only_target_and_one_with_service_credentials( + ) -> Result<()> { + let root = Path::new("/run/idunn/activation-credentials"); + let vili = include_str!("../tests/fixtures/systemd-257/vili-presence-identity-only.show"); + let odin = include_str!( + "../tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show" + ); + for text in [vili, odin] { + assert!(text.contains("LoadCredential=[unprintable]")); + require_systemd_isolation_contract(&fixture_show_properties(text)?, root)?; + } + let default_proc = vili.replace("ProtectProc=invisible", "ProtectProc=default"); + assert_ne!(default_proc, vili); + let other_path = vili.replace( + "/run/idunn/activation-credentials", + "/run/idunn/elsewhere", + ); + assert_ne!(other_path, vili); + for text in [default_proc, other_path] { + let error = + require_systemd_isolation_contract(&fixture_show_properties(&text)?, root) + .expect_err("a drifted isolation property must be refused"); + assert_eq!( + error.to_string(), + "systemd workload isolation properties differ from the Idunn contract" + ); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn delivered_credentials_are_exactly_the_launch_names() -> Result<()> { + let unit = "idunn-test.service"; + let names = |list: &[&str]| list.iter().map(|name| name.to_string()).collect::>(); + // `Some(entries)` creates the unit's directory in a fresh process root. + let check = |delivered: Option<&[&str]>, expected: &[&str]| -> Result<()> { + let temp = tempfile::tempdir()?; + if let Some(entries) = delivered { + let directory = temp.path().join("run/credentials").join(unit); + std::fs::create_dir_all(&directory)?; + for entry in entries { + std::fs::write(directory.join(entry), "canary-credential-contents")?; + } + } + let result = require_delivered_credentials(temp.path(), unit, &names(expected)); + if let Err(error) = &result { + let text = format!("{error:#}"); + for canary in ["STOWAWAY", "ODIN_TOPOLOGY_IDENTITY", "canary-credential"] { + assert!(!text.contains(canary), "error echoed {canary}"); + } + assert!(!text.contains(&*temp.path().to_string_lossy())); + } + result + }; + check(None, &[])?; + check(Some(&[]), &[])?; + check(Some(&["ODIN_TOPOLOGY_IDENTITY"]), &["ODIN_TOPOLOGY_IDENTITY"])?; + check(Some(&["B", "A"]), &["A", "B"])?; + for (delivered, expected) in [ + (None, &["ODIN_TOPOLOGY_IDENTITY"][..]), + (Some(&[][..]), &["ODIN_TOPOLOGY_IDENTITY"][..]), + ( + Some(&["ODIN_TOPOLOGY_IDENTITY", "STOWAWAY"][..]), + &["ODIN_TOPOLOGY_IDENTITY"][..], + ), + (Some(&["STOWAWAY"][..]), &[][..]), + (Some(&["A"][..]), &["B"][..]), + (Some(&["A"][..]), &["A", "B"][..]), + ] { + let error = check(delivered, expected) + .expect_err("delivered credentials differing from the launch must be refused"); + assert_eq!( + error.to_string(), + "systemd delivered credentials that differ from the Idunn launch" + ); + } + // A directory that cannot be listed is a refusal, never the empty set. + let temp = tempfile::tempdir()?; + let parent = temp.path().join("run/credentials"); + std::fs::create_dir_all(&parent)?; + std::fs::write(parent.join(unit), "")?; + let error = require_delivered_credentials(temp.path(), unit, &[]) + .expect_err("an unreadable credential directory must be refused"); + assert!(!format!("{error:#}").contains(&*temp.path().to_string_lossy())); + // The rule reads the process root it is given, never the host's /run. + let temp = tempfile::tempdir()?; + let outside = tempfile::tempdir()?; + let directory = outside.path().join("run/credentials").join(unit); + std::fs::create_dir_all(&directory)?; + std::fs::write(directory.join("A"), "")?; + require_delivered_credentials(temp.path(), unit, &[])?; + assert!(require_delivered_credentials(temp.path(), unit, &names(&["A"])).is_err()); + Ok(()) + } + + /// Live Odin and StreamPixels hold records persisted by the Idunn that + /// compared `LoadCredential`: `[unprintable]` in the retired slot. The bytes + /// are that Idunn's own output (tests/fixtures/pre-cut-*); they must decode + /// under both encodings and read as no drift against a new observation, or + /// `stop` refuses to stop the incumbents. + #[test] + fn an_incumbent_recorded_with_the_display_form_is_not_identity_drift() -> Result<()> { + let decode_hex = |text: &str| -> Vec { + (0..text.trim().len()) + .step_by(2) + .map(|at| u8::from_str_radix(&text.trim()[at..at + 2], 16).unwrap()) + .collect() + }; + let mut recorded = systemd_audit_observation("no", 61_000); + recorded.load_credential = "[unprintable]".into(); + let observed = systemd_audit_observation("no", 61_000); + assert!(observed.load_credential.is_empty()); + for hex in [ + include_str!("../tests/fixtures/pre-cut-systemd-observation.compact.hex"), + include_str!("../tests/fixtures/pre-cut-systemd-observation.named.hex"), + ] { + let decoded: WorkloadObservation = rmp_serde::from_slice(&decode_hex(hex))?; + assert_eq!(decoded, WorkloadObservation::Systemd(recorded.clone())); + let WorkloadObservation::Systemd(prior) = decoded else { + unreachable!() + }; + assert!(changed_systemd_workload_identity_fields(&observed, &prior).is_empty()); + } + let mut drifted = observed.clone(); + drifted.main_pid += 1; + assert_eq!( + changed_systemd_workload_identity_fields(&drifted, &recorded), + vec!["main_pid"] + ); + Ok(()) + } + #[test] fn host_isolation_is_two_live_distinct_processes_on_one_host() { let live = |pid: u32, created: u64| { @@ -9658,27 +10478,335 @@ Content-Le".to_vec()), |_| {}), )); } - #[cfg(unix)] - #[test] - fn the_published_projection_is_readable_by_a_workload() { - use std::os::unix::fs::PermissionsExt; + /// A native process that is the Expected's: what `publish_observed_activation` + /// checks the activation against. + #[cfg(target_os = "linux")] + fn observation_of( + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> WorkloadObservation { + let mut host = host_audit_observation(1, 1); + host.runtime_instance_id = activation.runtime_instance_id.clone(); + host.executable_sha256 = expected.artifact_sha256.clone(); + WorkloadObservation::Host(host) + } + #[cfg(target_os = "linux")] + fn mode_of(path: &Path) -> u32 { + use std::os::unix::fs::PermissionsExt; + fs::metadata(path).unwrap().permissions().mode() & 0o777 + } + + #[cfg(target_os = "linux")] + fn inode_of(path: &Path) -> u64 { + use std::os::unix::fs::MetadataExt; + fs::metadata(path).unwrap().ino() + } + + /// Idunn's `UMask=027` alone lands a file 0640: the store still has to be + /// published, and its lock has to stay private. + #[cfg(target_os = "linux")] + #[test] + fn a_published_store_is_readable_by_a_workload_and_its_lock_is_not() -> Result<()> { + use crate::incident::tests::under_service_umask; + + let temp = tempfile::tempdir()?; + let (expected, _, _, anchor) = authenticated_warming(temp.path())?; + let driver = topology_driver(temp.path(), "topology"); + under_service_umask(|| driver.publish_expected(&expected, &anchor))?; + assert_eq!(mode_of(&driver.projection_store), 0o644); + assert_eq!(mode_of(&authority_lock_path(&driver.projection_store)), 0o600); + Ok(()) + } + + /// The body's condition: the projection directory carries the default ACL + /// `u::rw g::r o::r`, under which the umask is ignored and a lock CultCache + /// creates lands 0644. Every writer publishes the store 0644 and leaves + /// the lock 0600, and another uid cannot open the lock. + #[cfg(target_os = "linux")] + #[test] + fn under_a_default_acl_the_projection_is_published_and_its_lock_is_private() -> Result<()> { + use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; + use crate::incident::tests::{give_default_acl, readable_by_another_uid, under_service_umask}; + + let temp = tempfile::tempdir()?; + let (expected, activation, _, anchor) = authenticated_warming(temp.path())?; + let observation = observation_of(&expected, &activation); + let dir = temp.path().join("projection"); + fs::create_dir(&dir)?; + fs::set_permissions(&dir, fs::Permissions::from_mode(0o755))?; + give_default_acl(&dir); + let driver = topology_driver(&dir, "topology"); + let store = driver.projection_store.clone(); + let lock = authority_lock_path(&store); + + // Control: the lock CultCache would have made here. Without the ACL in + // force this reads 0640 and the assertions below prove nothing. + let control = dir.join("control.lock"); + under_service_umask(|| { + OpenOptions::new().create(true).write(true).mode(0o666).open(&control) + })?; + assert_eq!(mode_of(&control), 0o644, "the default ACL is not in force"); + + let published = |step: &str| { + assert_eq!(mode_of(&store), 0o644, "the store after {step}"); + assert_eq!(mode_of(&lock), 0o600, "the lock after {step}"); + }; + under_service_umask(|| driver.publish_expected(&expected, &anchor))?; + published("publish_expected"); + let private_lock = inode_of(&lock); + under_service_umask(|| { + driver.publish_observed_activation(&expected, &activation, &observation) + })?; + published("publish_observed_activation"); + under_service_umask(|| { + driver.demote_to_expected_only(&expected, &anchor, &activation, None) + })?; + published("demote_to_expected_only"); + under_service_umask(|| driver.withdraw_incarnation(&expected, &anchor, None, None))?; + published("withdraw_incarnation"); + assert_eq!(inode_of(&lock), private_lock, "a private lock was replaced"); + + // The consequence: another uid cannot open the lock, so it cannot hold + // it. Provable only as root with setpriv, and only if that uid can + // read the control file, which shows the switch itself works. + if readable_by_another_uid(&control) == Some(true) { + assert_eq!(readable_by_another_uid(&lock), Some(false)); + } else { + eprintln!("UNPROVEN: no uid switch here; the lock modes above stand alone"); + } + Ok(()) + } + + /// Another process holds the lock: the publish is an error and a no-op, and + /// it lands once the lock is free. Nothing waits (`run_while_held` counts + /// the opens of the lock), and the error names the store kind, not a path. + #[cfg(target_os = "linux")] + #[test] + fn a_topology_publish_never_waits_on_a_held_lock_and_lands_when_freed() -> Result<()> { + use crate::incident::tests::{eventually, hold_lock_of, run_while_held}; + + let temp = tempfile::tempdir()?; + let (expected, activation, _, anchor) = authenticated_warming(temp.path())?; + let observation = observation_of(&expected, &activation); + let driver = topology_driver(temp.path(), "topology"); + + let error = run_while_held(hold_lock_of(&driver.projection_store), 1, || { + driver.publish_expected(&expected, &anchor) + }) + .unwrap_err(); + assert_eq!(error.to_string(), "the topology projection is locked by another holder"); + assert!(!driver.projection_store.exists(), "a contended publish wrote the store"); + + eventually(|| driver.publish_expected(&expected, &anchor)); + let landed = fs::read(&driver.projection_store)?; + + // An existing store is unchanged by a contended write of any kind. + let error = run_while_held(hold_lock_of(&driver.projection_store), 1, || { + driver.publish_observed_activation(&expected, &activation, &observation) + }) + .unwrap_err(); + assert_eq!(error.to_string(), "the topology projection is locked by another holder"); + assert_eq!(fs::read(&driver.projection_store)?, landed); + eventually(|| driver.publish_observed_activation(&expected, &activation, &observation)); + assert_ne!(fs::read(&driver.projection_store)?, landed); + Ok(()) + } + + /// The load-bearing rule (a chmod does not revoke a holder): a wider lock + /// that another process already holds is replaced by a fresh private one, + /// so the holder keeps only an orphaned inode and cannot block a publish. + #[cfg(target_os = "linux")] + #[test] + fn a_holder_of_a_wider_lock_cannot_block_a_publish() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + use crate::incident::tests::hold_lock_of; + + let temp = tempfile::tempdir()?; + let (expected, _, _, anchor) = authenticated_warming(temp.path())?; + let driver = topology_driver(temp.path(), "topology"); + let lock = authority_lock_path(&driver.projection_store); + fs::write(&lock, b"")?; + fs::set_permissions(&lock, fs::Permissions::from_mode(0o644))?; + let wider = inode_of(&lock); + let holder = hold_lock_of(&driver.projection_store); + + driver.publish_expected(&expected, &anchor)?; + assert_eq!(mode_of(&lock), 0o600); + assert_ne!(inode_of(&lock), wider, "the wider lock was chmodded, not replaced"); + assert!(holder.is_held(), "the publish ended the holder instead of outliving it"); + assert_eq!(driver.snapshot()?.len(), 2); + Ok(()) + } + + /// Every lock that is not a regular file, ours, mode 0600 is replaced, and + /// a private one is left alone. + #[cfg(target_os = "linux")] + #[test] + fn every_lock_that_is_not_private_is_replaced_and_a_private_one_is_not() -> Result<()> { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let temp = tempfile::tempdir()?; + let (expected, activation, _, anchor) = authenticated_warming(temp.path())?; + let observation = observation_of(&expected, &activation); + let driver = topology_driver(temp.path(), "topology"); + let lock = authority_lock_path(&driver.projection_store); + driver.publish_expected(&expected, &anchor)?; + // Each call changes the projection, so each one exchanges. + let toggle = || -> Result<()> { + if driver.projected_activation_is_present(&expected)? { + driver + .demote_to_expected_only(&expected, &anchor, &activation, None) + .map(drop) + } else { + driver + .publish_observed_activation(&expected, &activation, &observation) + .map(drop) + } + }; + // SAFETY: geteuid has no preconditions. + let euid = unsafe { libc::geteuid() }; + + for (shape, mode, owner) in [ + ("0640", 0o640, None), + ("0666", 0o666, None), + ("0700", 0o700, None), + ("0400", 0o400, None), + ("0604", 0o604, None), + ("0600 owned by another uid", 0o600, Some(65534)), + ] { + if owner.is_some() && euid != 0 { + eprintln!("UNPROVEN: not root, so a lock owned by another uid cannot be made"); + continue; + } + fs::set_permissions(&lock, fs::Permissions::from_mode(mode))?; + if let Some(uid) = owner { + std::os::unix::fs::chown(&lock, Some(uid), Some(uid))?; + } + let before = inode_of(&lock); + toggle()?; + assert_ne!(inode_of(&lock), before, "a {shape} lock was kept"); + assert_eq!(mode_of(&lock), 0o600, "after a {shape} lock"); + assert_eq!(fs::metadata(&lock)?.uid(), euid, "after a {shape} lock"); + } + + // A lock that is a symlink is replaced, never followed. + let target = temp.path().join("elsewhere"); + fs::write(&target, b"")?; + fs::set_permissions(&target, fs::Permissions::from_mode(0o600))?; + fs::remove_file(&lock)?; + std::os::unix::fs::symlink(&target, &lock)?; + toggle()?; + assert!(!fs::symlink_metadata(&lock)?.file_type().is_symlink()); + assert_eq!(mode_of(&lock), 0o600); + assert!(target.exists()); + + // A private lock is opened as it is: two more exchanges, one inode. + let kept = inode_of(&lock); + toggle()?; + toggle()?; + assert_eq!(inode_of(&lock), kept); + Ok(()) + } + + /// A lock that cannot be inspected is an inspection error, and one that + /// cannot be created is a creation error: neither is mistaken for the + /// other, and neither is retried as if another creator had won. + #[cfg(target_os = "linux")] + #[test] + fn a_lock_that_cannot_be_inspected_or_created_says_which() { let temp = tempfile::tempdir().unwrap(); - let path = temp.path().join("topology.cc"); - std::fs::write(&path, b"x").unwrap(); - // What Idunn's UMask=027 leaves behind. - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o640)).unwrap(); - let lock = temp.path().join("topology.cc.lock"); - std::fs::write(&lock, b"").unwrap(); - std::fs::set_permissions(&lock, std::fs::Permissions::from_mode(0o640)).unwrap(); - publish_projection_mode(&path).unwrap(); - let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777; - assert_eq!( - mode, 0o644, - "every target must be able to read the projection" - ); - let lock_mode = std::fs::metadata(&lock).unwrap().permissions().mode() & 0o777; - assert_eq!(lock_mode, 0o644, "the lock is opened alongside the store"); + // The lock's name is one byte past NAME_MAX: inspecting it fails with + // an error that is not "not found". + let long = temp.path().join("a".repeat(251)); + let error = exchange_behind_private_lock(&long, &[], &[]).unwrap_err(); + assert!(format!("{error:#}").contains("inspecting a store lock"), "{error:#}"); + + // Absent, but nothing can be created beside it: procfs refuses. + let error = exchange_behind_private_lock(Path::new("/proc/self/idunn-store.cc"), &[], &[]) + .unwrap_err(); + assert!(format!("{error:#}").contains("creating a private store lock"), "{error:#}"); + } + + /// Many first publishers race to create the same lock. Whoever loses finds + /// a private lock and carries on: no call fails for having lost. + #[cfg(target_os = "linux")] + #[test] + fn publishers_racing_to_create_the_lock_all_find_it_private() { + use std::sync::Barrier; + + const PUBLISHERS: usize = 8; + for round in 0..100 { + let temp = tempfile::tempdir().unwrap(); + let store = temp.path().join("topology.cc"); + let start = Barrier::new(PUBLISHERS); + let outcomes = thread::scope(|scope| { + (0..PUBLISHERS) + .map(|_| { + scope.spawn(|| { + start.wait(); + exchange_behind_private_lock(&store, &[], &[]).map(drop) + }) + }) + .collect::>() + .into_iter() + .map(|publisher| publisher.join().unwrap()) + .collect::>() + }); + for outcome in outcomes { + outcome.unwrap_or_else(|error| panic!("round {round}: {error:#}")); + } + assert_eq!(mode_of(&authority_lock_path(&store)), 0o600); + } + } + + /// A lost race is retried at once, against the store as it now is; a race + /// lost every time is an error rather than a loop. + #[cfg(target_os = "linux")] + #[test] + fn a_lost_race_is_retried_against_the_new_store_and_never_forever() -> Result<()> { + use std::cell::Cell; + + let temp = tempfile::tempdir()?; + let driver = topology_driver(temp.path(), "topology"); + let rival = |key: &str| CultCacheEnvelope { + key: key.into(), + r#type: "test.rival".into(), + payload: vec![0x90], + stored_at: "2026-01-01T00:00:00+00:00".into(), + schema_id: Some("test.rival.v1".into()), + }; + + // Lost once: a rival writes between the snapshot and the exchange. + let calls = Cell::new(0); + driver.mutate(|entries| { + calls.set(calls.get() + 1); + if calls.get() == 1 { + upsert_record(&driver.projection_store, rival("first")).unwrap(); + } + let mut next = entries.to_vec(); + next.push(rival(&format!("mine-{}", calls.get()))); + Ok(Some(next)) + })?; + assert_eq!(calls.get(), 2); + let keys = driver.snapshot()?.into_iter().map(|e| e.key).collect::>(); + assert_eq!(keys, ["first", "mine-2"]); + + // Lost every time: bounded, and the store is the rival's. + let calls = Cell::new(0); + let error = driver + .mutate(|entries| { + calls.set(calls.get() + 1); + upsert_record(&driver.projection_store, rival(&format!("rival-{}", calls.get()))).unwrap(); + let mut next = entries.to_vec(); + next.push(rival("never")); + Ok(Some(next)) + }) + .unwrap_err(); + assert!(error.to_string().contains("changed repeatedly"), "{error}"); + assert_eq!(calls.get(), 8); + assert!(driver.snapshot()?.iter().all(|entry| entry.key != "never")); + Ok(()) } #[cfg(unix)] @@ -11679,6 +12807,7 @@ Content-Le".to_vec()), |_| {}), private_port_end: 17979, config_path: temp.path().join("odin.conf"), reload_unit: "nginx.service".into(), + member: None, }, nginx_program: program("nginx", "#!/bin/sh\nexit 0\n")?, systemd_run_program: program("systemd-run", "#!/bin/sh\nexit 64\n")?, @@ -11688,12 +12817,15 @@ Content-Le".to_vec()), |_| {}), }; assert_eq!( driver - .endpoint_rule()? + .endpoint_rules()? .iter() - .map(|value| value.to_string_lossy().into_owned()) - .collect::>() - .join(" "), - "allow in to 10.77.0.1 port 17971 proto udp" + .map(|rule| rule + .iter() + .map(|value| value.to_string_lossy().into_owned()) + .collect::>() + .join(" ")) + .collect::>(), + vec!["allow in to 10.77.0.1 port 17971 proto udp"] ); driver.restore(None, &Unmetered)?; @@ -11753,6 +12885,7 @@ Content-Le".to_vec()), |_| {}), private_port_end: 4109, config_path: temp.join("service.conf"), reload_unit: "nginx.service".into(), + member: None, }, &RouteActuators { nginx: program("nginx")?, @@ -11787,6 +12920,271 @@ Content-Le".to_vec()), |_| {}), }) } + fn with_member_listener(driver: &mut NginxRouteDriver) { + driver.binding.member = Some(crate::deployment::MemberListenerBinding { + stable_endpoint: "http://127.0.0.1:4120".into(), + private_port_start: 4121, + max_connections: 200, + }); + } + + /// The fragment as the base commit rendered it, computed from its format + /// string and not from the code under test. + #[cfg(unix)] + fn base_fragment(sha: &str, candidate_port: u16) -> String { + format!( + "# Idunn Expected {sha}\nupstream idunn_service_route {{\n server 127.0.0.1:{candidate_port};\n}}\nserver {{\n listen 127.0.0.1:4103;\n proxy_pass idunn_service_route;\n}}\n" + ) + } + + /// A route without a member listener renders the bytes it always did; a + /// member listener adds a second upstream and server block to the one + /// fragment, under the one Expected comment. + #[cfg(unix)] + #[test] + fn a_member_listener_renders_a_second_block_only_when_bound() -> Result<()> { + let temp = tempfile::tempdir()?; + let (mut driver, mut candidate, _calls) = stubbed_route_driver(temp.path())?; + let sha = candidate.canonical_sha256()?; + assert_eq!( + String::from_utf8(driver.render(&candidate)?)?, + base_fragment(&sha, 4104) + ); + + with_member_listener(&mut driver); + let member_blocks = |member_port: u16| { + format!( + "limit_conn_zone \"idunn_service_route_member\" zone=idunn_service_route_member:64k;\nupstream idunn_service_route_member {{\n server 127.0.0.1:{member_port};\n}}\nserver {{\n listen 127.0.0.1:4120;\n limit_conn idunn_service_route_member 200;\n proxy_pass idunn_service_route_member;\n}}\n" + ) + }; + let rendered = String::from_utf8(driver.render(&candidate)?)?; + assert_eq!(rendered, base_fragment(&sha, 4104) + &member_blocks(4121)); + assert_eq!(rendered.matches("# Idunn Expected").count(), 1); + assert_eq!(rendered.matches("server {").count(), 2); + + candidate.route.as_mut().unwrap().candidate_endpoint = "http://127.0.0.1:4106".into(); + let sha = candidate.canonical_sha256()?; + assert_eq!( + String::from_utf8(driver.render(&candidate)?)?, + base_fragment(&sha, 4106) + &member_blocks(4123) + ); + Ok(()) + } + + /// Both stable sockets are admitted on the firewall, and both withdrawn. + #[cfg(unix)] + #[test] + fn both_route_endpoints_are_admitted_and_withdrawn() -> Result<()> { + let temp = tempfile::tempdir()?; + let (mut driver, _candidate, calls) = stubbed_route_driver(temp.path())?; + with_member_listener(&mut driver); + driver.admit_endpoint()?; + driver.withdraw_endpoint()?; + let log = fs::read_to_string(&calls)?; + let ufw: Vec<&str> = log.lines().filter(|line| line.starts_with("ufw ")).collect(); + assert_eq!( + ufw, + vec![ + "ufw allow in to 127.0.0.1 port 4103 proto tcp comment Idunn route service-route", + "ufw allow in to 127.0.0.1 port 4120 proto tcp comment Idunn route service-route", + "ufw delete allow in to 127.0.0.1 port 4103 proto tcp", + "ufw delete allow in to 127.0.0.1 port 4120 proto tcp", + ] + ); + Ok(()) + } + + /// Real nginx over the rendered two-block fragment, a real presence + /// acceptor behind the presence block and a flood of idle members on the + /// member block. Needs an `nginx` on PATH and the fixed loopback ports + /// 4103, 4104, 4120 and 4121, so it runs on demand in a container: + /// `cargo test --lib real_nginx -- --ignored`. + #[cfg(unix)] + fn challenges_answered_during_a_member_flood(max_connections: u16) -> Result { + use std::net::{TcpListener, TcpStream}; + use std::process::{Child, Command}; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + /// Stopped through nginx itself: killing the master would orphan its + /// workers, which keep the test's output pipe open. + struct Nginx(Child, PathBuf, PathBuf); + impl Drop for Nginx { + fn drop(&mut self) { + let _ = Command::new("nginx") + .arg("-p") + .arg(&self.1) + .arg("-c") + .arg(&self.2) + .args(["-s", "stop"]) + .status(); + let _ = self.0.wait(); + } + } + + let temp = tempfile::tempdir()?; + let (mut driver, candidate, _calls) = stubbed_route_driver(temp.path())?; + with_member_listener(&mut driver); + driver.binding.member.as_mut().unwrap().max_connections = max_connections; + let fragment = temp.path().join("route.conf"); + fs::write(&fragment, driver.render(&candidate)?)?; + // Debian builds the stream module dynamic. + let module = Path::new("/usr/lib/nginx/modules/ngx_stream_module.so"); + let load = if module.exists() { + format!("load_module {}; +", module.display()) + } else { + String::new() + }; + let conf = temp.path().join("nginx.conf"); + fs::write( + &conf, + format!( + "{load}worker_processes 1;\npid {dir}/nginx.pid;\nerror_log {dir}/error.log;\nevents {{ worker_connections 512; }}\nstream {{ include {fragment}; }}\n", + dir = temp.path().display(), + fragment = fragment.display() + ), + )?; + let halt = Arc::new(AtomicBool::new(false)); + let presence = crate::launcher::PresenceAcceptor::new( + TcpListener::bind("127.0.0.1:4104")?, + Arc::new(|request| match request { + CultNetMessage::SnapshotRequest { message_id, .. } => { + Ok(CultNetMessage::SnapshotResponseRaw { message_id, documents: vec![] }) + } + _ => bail!("not a snapshot request"), + }), + ); + let presence_halt = halt.clone(); + let presence = thread::spawn(move || presence.serve(presence_halt)); + // The member backend accepts and holds every connection. + let members = TcpListener::bind("127.0.0.1:4121")?; + members.set_nonblocking(true)?; + let member_halt = halt.clone(); + let members = thread::spawn(move || { + let mut held = Vec::new(); + while !member_halt.load(Ordering::SeqCst) { + match members.accept() { + Ok((stream, _)) => held.push(stream), + Err(_) => thread::sleep(Duration::from_millis(5)), + } + } + }); + let _nginx = Nginx( + Command::new("nginx") + .args(["-p"]) + .arg(temp.path()) + .arg("-c") + .arg(&conf) + .args(["-g", "daemon off;"]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn()?, + temp.path().to_path_buf(), + conf.clone(), + ); + let stable: SocketAddr = "127.0.0.1:4103".parse()?; + let ready = Instant::now(); + while TcpStream::connect(stable).is_err() { + ensure!(ready.elapsed() < Duration::from_secs(10), "nginx never listened"); + thread::sleep(Duration::from_millis(50)); + } + // The probe connection above reached the presence acceptor as an idle + // peer; let its deadline lapse so only the flood is measured. + thread::sleep(Duration::from_millis(1200)); + + let mut flood = Vec::new(); + for _ in 0..300 { + if let Ok(mut stream) = TcpStream::connect("127.0.0.1:4120") { + let _ = stream.write_all(b"x"); + flood.push(stream); + } + } + thread::sleep(Duration::from_millis(500)); + + let payload = encode_cultnet_message_to_vec( + &CultNetMessage::SnapshotRequest { + message_id: "flood".into(), + schema_ids: None, + record_keys: None, + }, + CultNetWireContract::CultNetSchemaV0, + )?; + let mut answered = 0; + for _ in 0..20 { + let started = Instant::now(); + if request_tcp_snapshot(stable, &payload).is_ok() + && started.elapsed() < Duration::from_secs(3) + { + answered += 1; + } + } + eprintln!("answered {answered} of 20 challenges with a member limit of {max_connections}"); + drop(flood); + halt.store(true, Ordering::SeqCst); + presence.join().unwrap(); + members.join().unwrap(); + Ok(answered) + } + + #[cfg(unix)] + #[test] + #[ignore = "needs nginx on PATH and fixed loopback ports; run in a container"] + fn real_nginx_keeps_presence_answerable_under_a_member_flood() -> Result<()> { + // The rendered default limit keeps the route's members out of the + // presence block's share of a 512-connection pool. + assert_eq!(challenges_answered_during_a_member_flood(128)?, 20); + Ok(()) + } + + #[cfg(unix)] + #[test] + #[ignore = "needs nginx on PATH and fixed loopback ports; run in a container"] + fn real_nginx_flood_starves_presence_without_a_member_limit() -> Result<()> { + // The control: the same flood against a limit too high to bind (nginx allows up to 65535) starves + // presence, so the test above measures the limit and not the load. + assert!(challenges_answered_during_a_member_flood(60_000)? < 20); + Ok(()) + } + + /// Idunn sets the member bind for a bound member listener, derived from the + /// candidate, and for no other workload. + #[test] + fn the_member_bind_is_set_only_for_a_bound_member_listener() -> Result<()> { + use crate::deployment::tests::{BINDING, RECIPE, member_listener_pair}; + + let driver = SystemdTransientWorkloadDriver::default(); + let mut routed = expected(); + routed.route = Some(cultnet_rs::IdunnExpectedRoute { + route_id: "test-service-public".into(), + transport: "http".into(), + stable_endpoint: "http://127.0.0.1:18830".into(), + candidate_endpoint: "http://127.0.0.1:18833".into(), + }); + let bundle = Path::new("/etc/gamecult/test-service/runtime/bundle"); + let unit = "idunn-test-service-abc"; + + let (recipe, binding) = member_listener_pair(); + let binding = OperatorBinding::parse(&binding)?; + binding.admit(&TargetDeclaration::parse(&recipe)?)?; + let environment = driver.launch_environment(&binding, bundle, &routed, unit)?; + assert_eq!( + environment.get(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT).map(String::as_str), + Some("127.0.0.1:18843") + ); + assert_eq!( + environment.get(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT).map(String::as_str), + Some("127.0.0.1:18833") + ); + + let plain = OperatorBinding::parse(BINDING)?; + plain.admit(&TargetDeclaration::parse(RECIPE)?)?; + let environment = driver.launch_environment(&plain, bundle, &routed, unit)?; + assert!(!environment.contains_key(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT)); + assert!(environment.contains_key(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT)); + Ok(()) + } + /// A refused forward change stops before any host mutation: no program /// runs, no fragment is written, and no private-mount unit starts. #[cfg(unix)] @@ -12106,6 +13504,7 @@ Content-Le".to_vec()), |_| {}), private_port_end: 4109, config_path: config.clone(), reload_unit: "nginx.service".into(), + member: None, }, nginx_program: program("nginx")?, systemd_run_program: program("systemd-run")?, @@ -12166,6 +13565,7 @@ Content-Le".to_vec()), |_| {}), private_port_end: 27879, config_path: config_path.clone(), reload_unit: "nginx.service".into(), + member: None, }); assert_eq!(driver.current_configuration()?, None); // What systemd leaves behind when it materializes the preflight's bind @@ -12192,6 +13592,7 @@ Content-Le".to_vec()), |_| {}), private_port_end: 27879, config_path: "/etc/nginx/idunn-stream-routes/odin-rudp.conf".into(), reload_unit: "nginx.service".into(), + member: None, }); let mut candidate = expected(); candidate.route = Some(cultnet_rs::IdunnExpectedRoute { @@ -12253,6 +13654,7 @@ Content-Le".to_vec()), |_| {}), private_port_end: 4109, config_path: config.clone(), reload_unit: "nginx.service".into(), + member: None, }; let driver = NginxRouteDriver { binding, @@ -12899,6 +14301,40 @@ Content-Le".to_vec()), |_| {}), Ok(()) } + #[cfg(unix)] + #[test] + fn write_lease_grant_binds_the_exact_warming_candidate_and_the_holder_read_keeps_custody() + -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir()?; + let path = temp.path().join("lease.cc"); + let driver = CultCacheWriteLeaseDriver::new("service", &path); + let (expected, activation, warming, _) = authenticated_warming(temp.path())?; + let good = lease(&expected, &activation, &warming); + let mut foreign = good.clone(); + foreign.warming_presence_sha256 = digest('8'); + assert!( + driver + .grant(&expected, &activation, &warming, &foreign) + .is_err() + ); + let other = CultCacheWriteLeaseDriver::new("other", temp.path().join("other.cc")); + assert!(other.grant(&expected, &activation, &warming, &good).is_err()); + assert!(driver.current_for_holder()?.is_none()); + driver.grant(&expected, &activation, &warming, &good)?; + assert_eq!( + driver.current_for_holder()?.map(|(_, held)| held), + Some(good) + ); + fs::set_permissions(&path, fs::Permissions::from_mode(0o666))?; + assert!( + driver.current_for_holder().is_err(), + "a service-writable lease store is not authority" + ); + Ok(()) + } + #[cfg(unix)] #[test] fn installed_release_is_root_owned_and_nonwritable() -> Result<()> { @@ -12918,6 +14354,7 @@ Content-Le".to_vec()), |_| {}), sha256: sha256_id(b"sealed"), size_bytes: 6, executable: true, + executable_members: BTreeSet::new(), }], )?; let root_metadata = fs::symlink_metadata(&root)?; @@ -12929,6 +14366,554 @@ Content-Le".to_vec()), |_| {}), Ok(()) } + #[cfg(unix)] + const MEMBER_CANARY: &str = "zzcanary"; + + #[cfg(unix)] + fn tree_output(executable: bool, members: &[&str]) -> ArtifactOutput { + ArtifactOutput { + id: "upstream".into(), + source_kind: ArtifactSource::WorktreeTree, + runner: None, + source: PathBuf::from("upstream"), + destination: PathBuf::from("upstream"), + expected_sha256: None, + executable, + executable_members: members.iter().map(PathBuf::from).collect(), + } + } + + #[cfg(unix)] + fn write_file(path: &Path, bytes: &[u8]) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + fs::create_dir_all(path.parent().context("file has no parent")?)?; + fs::write(path, bytes)?; + // Builds and git leave arbitrary exec bits; the seal must not inherit them. + fs::set_permissions(path, fs::Permissions::from_mode(0o777))?; + Ok(()) + } + + #[cfg(unix)] + fn sealed_mode(path: &Path) -> u32 { + use std::os::unix::fs::PermissionsExt; + + fs::symlink_metadata(path).unwrap().permissions().mode() & 0o777 + } + + /// `/src/upstream` is the worktree tree; `/staging` the release. + #[cfg(unix)] + fn sample_tree(root: &Path) -> Result<(PathBuf, PathBuf)> { + let source = root.join("src"); + let staging = root.join("staging"); + fs::create_dir_all(&staging)?; + let tree = source.join("upstream"); + write_file(&tree.join("bin/daemon"), b"daemon")?; + write_file(&tree.join("bin/helper"), b"helper")?; + write_file(&tree.join("lib/tool"), b"tool")?; + write_file(&tree.join("lib/data.txt"), b"data")?; + write_file(&tree.join("NOTICE"), b"notice")?; + Ok((source, staging)) + } + + #[cfg(unix)] + fn collect_tree( + source: &Path, + staging: &Path, + output: ArtifactOutput, + ) -> Result { + let mut declaration = + TargetDeclaration::parse(crate::deployment::tests::RECIPE)?; + let id = output.id.clone(); + declaration.artifacts = vec![output]; + let frozen = FrozenSource { + receipt: FrozenSourceReceipt { + transaction_id: "tx".into(), + plan_id: "plan".into(), + snapshot_sha256: "snapshot".into(), + }, + facts: crate::deployment_plan::tests::source(crate::deployment::tests::RECIPE), + recipe_bytes: Vec::new(), + root: source.to_path_buf(), + }; + DockerRunnerDriver::default().collect_artifact( + &declaration, + &frozen, + &BTreeMap::new(), + staging, + &id, + ) + } + + #[cfg(unix)] + fn refusal_text(result: Result) -> String { + let Err(error) = result else { + panic!("the release was not refused"); + }; + let text = format!("{error:#}"); + assert!( + !text.contains(MEMBER_CANARY), + "the refusal echoed a member path: {text}" + ); + text + } + + #[cfg(unix)] + #[test] + fn tree_member_sealed_0555_and_siblings_0444() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let receipt = collect_tree( + &source, + &staging, + tree_output(false, &["bin/daemon", "lib/tool"]), + )?; + assert_eq!( + receipt.executable_members, + BTreeSet::from([PathBuf::from("bin/daemon"), PathBuf::from("lib/tool")]) + ); + harden_installed_release(&staging, &[receipt])?; + let tree = staging.join("upstream"); + assert_eq!(sealed_mode(&tree.join("bin/daemon")), 0o555); + assert_eq!(sealed_mode(&tree.join("lib/tool")), 0o555); + assert_eq!(sealed_mode(&tree.join("bin/helper")), 0o444); + assert_eq!(sealed_mode(&tree.join("lib/data.txt")), 0o444); + assert_eq!(sealed_mode(&tree.join("NOTICE")), 0o444); + for directory in [&staging, &tree, &tree.join("bin"), &tree.join("lib")] { + assert_eq!(sealed_mode(directory), 0o555); + } + Ok(()) + } + + #[test] + fn release_link_rule_is_containment() { + for (link, target) in [ + ("a/b/l", "../t/x"), + ("a/l", ".."), + ("a/l", "./../b"), + ("a/l", "b/./c"), + ("l", "python3.12"), + ("a/b/c/l", "../../x"), + ] { + check_release_link(Path::new(link), Path::new(target)) + .unwrap_or_else(|error| panic!("({link}, {target}) refused: {error:#}")); + } + for (link, target, rule) in [ + ("l", "/x", "absolute"), + ("a/l", "s/../x", "climbs after a name"), + ("a/l", "D/D/../../../etc/passwd", "climbs after a name"), + ("a/l", "../../x", "escapes its artifact"), + ("l", "..", "escapes its artifact"), + ("a/b/l", "../../../x", "escapes its artifact"), + ("a/l", "./s/../x", "climbs after a name"), + ] { + let text = format!( + "{:#}", + check_release_link(Path::new(link), Path::new(target)) + .expect_err(&format!("({link}, {target}) was admitted")) + ); + assert!(text.contains(rule), "({link}, {target}): {text}"); + } + } + + #[cfg(unix)] + #[test] + fn missing_member_refuses_the_release() -> Result<()> { + // The bad member sorts after the good one in one case and before it in + // the other, so a check that stops at the first (or last) member fails. + for members in [ + &["bin/daemon", "lib/zzcanary-missing"][..], + &["aaa-zzcanary-missing", "bin/daemon"][..], + &["zzcanary-dir/tool"][..], + ] { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let text = refusal_text(collect_tree(&source, &staging, tree_output(false, members))); + assert!(text.contains("executable member is missing"), "{text}"); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn symlink_member_refuses_the_release() -> Result<()> { + for link_target in ["daemon", "../zzcanary-nowhere"] { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + std::os::unix::fs::symlink( + link_target, + source.join("upstream/bin/zzcanary-link"), + )?; + let text = refusal_text(collect_tree( + &source, + &staging, + tree_output(false, &["bin/daemon", "bin/zzcanary-link"]), + )); + assert!(text.contains("not a regular file"), "{text}"); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn member_through_symlinked_directory_refuses_the_release() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let tree = source.join("upstream"); + write_file(&tree.join("real-bin/zzcanary-tool"), b"tool")?; + std::os::unix::fs::symlink("real-bin", tree.join("linked-bin"))?; + let text = refusal_text(collect_tree( + &source, + &staging, + tree_output(false, &["linked-bin/zzcanary-tool"]), + )); + assert!(text.contains("symlink or non-directory"), "{text}"); + // A member below a regular file is refused the same way. + let second_staging = temp.path().join("second-staging"); + fs::create_dir_all(&second_staging)?; + let text = refusal_text(collect_tree( + &source, + &second_staging, + tree_output(false, &["bin/daemon/zzcanary-below-file"]), + )); + assert!(text.contains("symlink or non-directory"), "{text}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn directory_member_refuses_the_release() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let text = refusal_text(collect_tree( + &source, + &staging, + tree_output(false, &["bin/daemon", "lib"]), + )); + assert!(text.contains("not a regular file"), "{text}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn executable_flag_on_tree_output_refuses() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let text = refusal_text(collect_tree(&source, &staging, tree_output(true, &[]))); + assert!(text.contains("cannot be executable"), "{text}"); + // The same flag on a file output stays admitted. + let file = temp.path().join("file-artifact"); + fs::write(&file, b"x")?; + let mut output = tree_output(true, &[]); + output.id = "file".into(); + check_artifact_modes(&output, &file)?; + Ok(()) + } + + #[cfg(unix)] + #[test] + fn members_on_file_output_refuse() -> Result<()> { + let temp = tempfile::tempdir()?; + let file = temp.path().join("file-artifact"); + fs::write(&file, b"x")?; + check_artifact_modes(&tree_output(false, &[]), &file)?; + let text = refusal_text(check_artifact_modes( + &tree_output(false, &["zzcanary"]), + &file, + )); + assert!(text.contains("cannot declare executable_members"), "{text}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn check_artifact_modes_refuses_a_member_that_is_not_a_clean_relative_path() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, _) = sample_tree(temp.path())?; + let tree = source.join("upstream"); + write_file(&temp.path().join("zzcanary-outside"), b"x")?; + for member in ["../../zzcanary-outside", "bin/../bin/daemon", "/zzcanary", ""] { + refusal_text(check_artifact_modes(&tree_output(false, &[member]), &tree)); + } + check_artifact_modes(&tree_output(false, &["bin/daemon"]), &tree)?; + Ok(()) + } + + #[cfg(unix)] + #[test] + fn an_unreadable_member_is_not_reported_as_missing() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, _) = sample_tree(temp.path())?; + // A component past NAME_MAX fails with an io error that is not NotFound. + let long = "z".repeat(300); + let text = refusal_text(check_artifact_modes( + &tree_output(false, &[long.as_str()]), + &source.join("upstream"), + )); + assert!(!text.contains("missing"), "{text}"); + assert!(text.contains("inspecting an executable member"), "{text}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn harden_fails_when_a_declared_member_is_absent() -> Result<()> { + let receipt = |members: &[&str]| ArtifactReceipt { + artifact_id: "upstream".into(), + destination: PathBuf::from("upstream"), + sha256: sha256_id(b"x"), + size_bytes: 1, + executable: false, + executable_members: members.iter().map(PathBuf::from).collect(), + }; + let installed = |root: &Path| -> Result<()> { + write_file(&root.join("upstream/bin/daemon"), b"daemon")?; + write_file(&root.join("upstream/lib/tool"), b"tool")?; + fs::create_dir_all(root.join("upstream/zzcanary-dir"))?; + std::os::unix::fs::symlink("daemon", root.join("upstream/bin/zzcanary-link"))?; + Ok(()) + }; + let temp = tempfile::tempdir()?; + installed(temp.path())?; + harden_installed_release(temp.path(), &[receipt(&["bin/daemon", "lib/tool"])])?; + for members in [ + &["bin/daemon", "lib/zzcanary-gone"][..], + &["zzcanary-gone", "bin/daemon"][..], + &["zzcanary-dir"][..], + &["bin/zzcanary-link"][..], + &["bin/daemon/zzcanary-below-file"][..], + ] { + let temp = tempfile::tempdir()?; + installed(temp.path())?; + let text = refusal_text(harden_installed_release(temp.path(), &[receipt(members)])); + assert!(text.contains("declared executable is missing"), "{text}"); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn relative_in_tree_symlink_survives_the_seal() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let tree = source.join("upstream"); + write_file(&tree.join("bin/python3.12"), b"python")?; + std::os::unix::fs::symlink("python3.12", tree.join("bin/python3"))?; + write_file(&tree.join("lib/terminfo/b/real"), b"terminfo")?; + fs::create_dir_all(tree.join("lib/terminfo/a"))?; + std::os::unix::fs::symlink("../b/real", tree.join("lib/terminfo/a/alias"))?; + let receipt = collect_tree(&source, &staging, tree_output(false, &["bin/python3.12"]))?; + let (before, _) = digest_artifact(&staging.join("upstream"))?; + harden_installed_release(&staging, &[receipt])?; + let link = staging.join("upstream/bin/python3"); + assert!(fs::symlink_metadata(&link)?.file_type().is_symlink()); + assert_eq!(fs::read_link(&link)?, PathBuf::from("python3.12")); + assert_eq!(sealed_mode(&staging.join("upstream/bin/python3.12")), 0o555); + let alias = staging.join("upstream/lib/terminfo/a/alias"); + assert!(fs::symlink_metadata(&alias)?.file_type().is_symlink()); + assert_eq!(fs::read_link(&alias)?, PathBuf::from("../b/real")); + assert_eq!(digest_artifact(&staging.join("upstream"))?.0, before); + + // A link that escapes, is absolute or climbs after a name is refused at + // the seal, before any receipt exists. + for (target, rule) in [ + ("../../zzcanary-up", "escapes its artifact"), + ("/zzcanary/absolute", "absolute"), + ("zzcanary-s/../x", "climbs after a name"), + ] { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + std::os::unix::fs::symlink(target, source.join("upstream/lib/zzcanary-link"))?; + let text = refusal_text(collect_tree(&source, &staging, tree_output(false, &["bin/daemon"]))); + assert!(text.contains(rule), "{text}"); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn links_to_existing_targets_are_refused_at_the_seal() -> Result<()> { + // Every refused shape, aimed at a target that EXISTS (file and + // directory): a judge guarded by `exists()`, `is_dir()` or a file-type + // test passes the dangling fixtures and fails these. + for kind in ["file", "dir"] { + for (shape, rule) in [ + ("absolute", "absolute"), + ("escape", "escapes its artifact"), + ("after-name", "climbs after a name"), + ] { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let target = match (shape, kind) { + ("absolute", "file") => source.join("upstream/bin/daemon"), + ("absolute", _) => source.join("upstream/lib"), + ("escape", "file") => { + write_file(&source.join("outside-file"), b"outside")?; + PathBuf::from("../../outside-file") + } + ("escape", _) => { + fs::create_dir_all(source.join("outside-dir"))?; + PathBuf::from("../../outside-dir") + } + (_, "file") => PathBuf::from("../bin/../NOTICE"), + _ => PathBuf::from("../bin/../lib"), + }; + let link = source.join("upstream/lib/zzcanary-link"); + std::os::unix::fs::symlink(&target, &link)?; + assert!(fs::metadata(&link).is_ok(), "{shape} {kind} link must resolve"); + let text = refusal_text(collect_tree(&source, &staging, tree_output(false, &["bin/daemon"]))); + assert!(text.contains(rule), "{shape} {kind}: {text}"); + } + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn streamed_digest_is_byte_compatible() -> Result<()> { + // Not a multiple of the 64 KiB read buffer, so a short last chunk + // matters; the aligned and empty files are the other boundaries. + let big = (0..3 * 65536 + 17).map(|i| (i % 251) as u8).collect::>(); + let aligned = vec![7_u8; 65536]; + let temp = tempfile::tempdir()?; + let tree = temp.path().join("tree"); + write_file(&tree.join("a.txt"), b"alpha")?; + write_file(&tree.join("empty"), b"")?; + write_file(&tree.join("sub/aligned.bin"), &aligned)?; + write_file(&tree.join("sub/big.bin"), &big)?; + std::os::unix::fs::symlink("a.txt", tree.join("zlink"))?; + + // The encoding digest_tree had before it streamed, written out by hand. + fn add_file(hasher: &mut Sha256, name: &str, bytes: &[u8]) { + hasher.update(b"file\0"); + hasher.update(name.as_bytes()); + hasher.update(b"\0"); + hasher.update((bytes.len() as u64).to_le_bytes()); + hasher.update(bytes); + } + let mut expected = Sha256::new(); + add_file(&mut expected, "a.txt", b"alpha"); + add_file(&mut expected, "empty", b""); + let mut expected_size = 5_u64; + expected.update(b"dir\0sub\0"); + add_file(&mut expected, "sub/aligned.bin", &aligned); + add_file(&mut expected, "sub/big.bin", &big); + expected_size += (aligned.len() + big.len()) as u64; + expected.update(b"link\0zlink\0a.txt"); + expected_size += "a.txt".len() as u64; + assert_eq!( + digest_artifact(&tree)?, + (format!("{:x}", expected.finalize()), expected_size) + ); + + let single = tree.join("sub/big.bin"); + assert_eq!( + digest_artifact(&single)?, + (raw_sha256(&big), big.len() as u64) + ); + let mut hasher = Sha256::new(); + hash_file_exact(&single, big.len() as u64, &mut hasher)?; + assert_eq!(format!("{:x}", hasher.finalize()), raw_sha256(&big)); + for wrong in [big.len() as u64 + 1, big.len() as u64 - 1] { + let text = refusal_text(hash_file_exact(&single, wrong, &mut Sha256::new())); + assert!(text.contains("changed while hashing"), "{text}"); + } + Ok(()) + } + + /// Acceptance on the real trees: `IDUNN_BIG_TREE` holds `python/` and + /// `blender/`. Run on Yggdrasil as its own job (see the cut spec). + #[cfg(unix)] + #[test] + #[ignore = "needs IDUNN_BIG_TREE: a CPython+torch tree and a Blender tree"] + fn big_tree_seals_from_env() -> Result<()> { + use std::time::Instant; + + /// (links, links with a `..` step, regular files, files at 0555), never + /// following a link; every regular file must be 0444 or a member's 0555. + fn census( + root: &Path, + path: &Path, + members: &BTreeSet, + modes: bool, + totals: &mut [usize; 4], + ) -> Result<()> { + for entry in fs::read_dir(path)? { + let entry = entry?; + let metadata = fs::symlink_metadata(entry.path())?; + if metadata.is_dir() { + census(root, &entry.path(), members, modes, totals)?; + } else if metadata.file_type().is_symlink() { + totals[0] += 1; + let target = fs::read_link(entry.path())?; + if target + .components() + .any(|part| matches!(part, std::path::Component::ParentDir)) + { + totals[1] += 1; + } + } else { + totals[2] += 1; + let member = members.contains(entry.path().strip_prefix(root)?); + let mode = sealed_mode(&entry.path()); + if modes { + assert_eq!(mode, if member { 0o555 } else { 0o444 }); + totals[3] += usize::from(mode == 0o555); + } + } + } + Ok(()) + } + fn peak_rss() -> String { + fs::read_to_string("/proc/self/status") + .ok() + .and_then(|status| { + status + .lines() + .find(|line| line.starts_with("VmHWM")) + .map(str::to_owned) + }) + .unwrap_or_default() + } + + let tree = PathBuf::from( + std::env::var_os("IDUNN_BIG_TREE").context("IDUNN_BIG_TREE is not set")?, + ); + let members = ["python/bin/python3.12", "blender/blender"]; + let temp = tempfile::tempdir()?; + let staging = temp.path().join("staging"); + fs::create_dir_all(&staging)?; + let destination = staging.join("tree"); + let output = ArtifactOutput { + destination: PathBuf::from("tree"), + ..tree_output(false, &members) + }; + + let started = Instant::now(); + let receipt = seal_artifact(&output, &tree, &tree, &staging)?; + println!("seal {:?} {}", started.elapsed(), peak_rss()); + let started = Instant::now(); + let again = digest_artifact(&destination)?; + assert_eq!(format!("sha256-{}", again.0), receipt.sha256); + assert_eq!(again.1, receipt.size_bytes); + println!("digest {:?} {}", started.elapsed(), peak_rss()); + let mut before = [0; 4]; + census(&destination, &destination, &output.executable_members, false, &mut before)?; + let started = Instant::now(); + harden_installed_release(&staging, &[receipt.clone()])?; + println!("harden {:?} {}", started.elapsed(), peak_rss()); + let mut totals = [0; 4]; + census(&destination, &destination, &output.executable_members, true, &mut totals)?; + assert_eq!(totals[0], before[0], "links changed across harden"); + assert_eq!(totals[3], members.len()); + println!( + "dotdot links {} links before {} after {} files {} exec {}", + totals[1], before[0], totals[0], totals[2], totals[3] + ); + println!("bytes {}", receipt.size_bytes); + Ok(()) + } + #[cfg(unix)] #[test] fn runner_cache_is_created_once_for_one_exact_identity() -> Result<()> { @@ -14425,4 +16410,292 @@ Content-Le".to_vec()), |_| {}), ); Ok(()) } + + // BEGIN runner-failure-evidence tests + // The runner decides what failed and what it kept: these drive the real + // DockerRunnerDriver over a fake docker program, never the capture alone. + + const RUNNER_CANARY: &str = "CANARY-5d2e91b7"; + + #[cfg(unix)] + fn fake_docker(dir: &Path, body: &str) -> Result { + use std::os::unix::fs::PermissionsExt; + let program = dir.join("docker"); + fs::write(&program, format!("#!/bin/sh\n{body}\n"))?; + fs::set_permissions(&program, fs::Permissions::from_mode(0o755))?; + Ok(DockerRunnerDriver { + docker_program: program, + }) + } + + /// One recipe step through the real `run_in_workspace` path. + #[cfg(unix)] + fn run_step(driver: &DockerRunnerDriver, dir: &Path, step_id: &str) -> Result<()> { + let workspace = dir.join("workspace"); + fs::create_dir_all(&workspace)?; + driver.run_in_workspace( + step_id, + "rust", + &fixture_docker_runner_binding(), + &workspace, + Path::new("."), + &["cargo".to_owned(), "test".to_owned()], + &std::collections::BTreeSet::new(), + ("IDUNN_STAMP", "rev"), + ) + } + + #[cfg(unix)] + fn failure_of(result: Result<()>) -> Result { + let error = result.expect_err("the step did not fail"); + error + .downcast_ref::() + .cloned() + .with_context(|| format!("the failure is not typed: {error:#}")) + } + + #[cfg(unix)] + #[test] + fn a_failed_step_is_named_with_both_tails() -> Result<()> { + let temp = tempfile::tempdir()?; + // The first step passes, the second fails: the count is the fake's. + let driver = fake_docker( + temp.path(), + &format!( + "n=$(cat \"$0.count\" 2>/dev/null || echo 0); n=$((n+1)); echo $n > \"$0.count\"\n\ + [ \"$n\" -lt 2 ] && exit 0\n\ + echo 'stdout line one'; echo 'stdout report {RUNNER_CANARY}'\n\ + echo 'stderr report' >&2\nexit 3" + ), + )?; + run_step(&driver, temp.path(), "first")?; + let failure = failure_of(run_step(&driver, temp.path(), "second"))?; + assert_eq!(failure.subject, RunnerSubject::Step("second".into())); + assert_eq!(failure.runner, "rust"); + assert_eq!(failure.exit_code, Some(3)); + assert_eq!(failure.signal, None); + assert!( + failure + .stdout_tail + .ends_with(format!("stdout report {RUNNER_CANARY}\n").as_bytes()), + "stdout was dropped: {:?}", + String::from_utf8_lossy(&failure.stdout_tail) + ); + assert_eq!(failure.stderr_tail, b"stderr report\n"); + assert_eq!(failure.stdout_total_bytes, failure.stdout_tail.len() as u64); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_step_killed_by_a_signal_is_named_by_its_signal() -> Result<()> { + let temp = tempfile::tempdir()?; + let driver = fake_docker(temp.path(), "kill -9 $$")?; + let failure = failure_of(run_step(&driver, temp.path(), "build"))?; + assert_eq!((failure.exit_code, failure.signal), (None, Some(9))); + assert!(failure.to_string().contains("exited with signal 9"), "{failure}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_failed_external_input_is_named_as_one() -> Result<()> { + let temp = tempfile::tempdir()?; + let driver = fake_docker(temp.path(), "echo curl failed >&2\nexit 22")?; + let declaration = TargetDeclaration::parse(crate::deployment_plan::tests::RECIPE)?; + let binding = OperatorBinding::parse(crate::deployment_plan::tests::BINDING)?; + let input_id = declaration.external_inputs[0].id.clone(); + let workspaces = BTreeMap::from([("rust".to_owned(), temp.path().join("workspace"))]); + fs::create_dir_all(&workspaces["rust"])?; + let error = driver + .materialize_external_input(&declaration, &binding, &workspaces, &input_id) + .expect_err("the download did not fail"); + let failure = error + .downcast_ref::() + .with_context(|| format!("the failure is not typed: {error:#}"))?; + assert_eq!(failure.subject, RunnerSubject::ExternalInput(input_id.clone())); + assert_eq!(failure.exit_code, Some(22)); + assert!( + failure.to_string().contains(&format!("runner external input {input_id} on runner rust")), + "{failure}" + ); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn runner_output_never_reaches_the_error_text() -> Result<()> { + let temp = tempfile::tempdir()?; + let driver = fake_docker( + temp.path(), + &format!("echo 'out {RUNNER_CANARY}'\necho 'err {RUNNER_CANARY}' >&2\nexit 1"), + )?; + let failure = failure_of(run_step(&driver, temp.path(), "build"))?; + assert!(String::from_utf8_lossy(&failure.stdout_tail).contains(RUNNER_CANARY)); + assert!(String::from_utf8_lossy(&failure.stderr_tail).contains(RUNNER_CANARY)); + let error = anyhow::Error::new(failure.clone()).context("sealing"); + for text in [ + format!("{error:#}"), + format!("{error:?}"), + format!("{failure}"), + format!("{failure:?}"), + ] { + assert!(!text.contains(RUNNER_CANARY), "runner output reached text: {text}"); + } + Ok(()) + } + + /// A reader that answers from a script, so a read error the OS rarely + /// produces (an interrupted call) is produced on demand. + struct ScriptedReader(Vec>>); + + impl Read for ScriptedReader { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + if self.0.is_empty() { + return Ok(0); + } + let bytes = self.0.remove(0)?; + buffer[..bytes.len()].copy_from_slice(&bytes); + Ok(bytes.len()) + } + } + + struct BrokenReader; + + impl Read for BrokenReader { + fn read(&mut self, _: &mut [u8]) -> std::io::Result { + Err(std::io::Error::other("persistent read failure")) + } + } + + #[test] + fn an_interrupted_read_is_retried_and_loses_no_byte() { + let reader = ScriptedReader(vec![ + Ok(b"ab".to_vec()), + Err(std::io::Error::from(ErrorKind::Interrupted)), + Ok(b"cd".to_vec()), + Err(std::io::Error::from(ErrorKind::Interrupted)), + ]); + let drained = drain_tail(reader, 3).expect("an interrupted read ended the drain"); + assert_eq!(drained, (b"bcd".to_vec(), 4)); + } + + #[test] + fn a_persistent_read_error_ends_the_drain_instead_of_spinning() { + let (sender, receiver) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = sender.send(drain_tail(BrokenReader, 8).is_err()); + }); + assert_eq!( + receiver.recv_timeout(Duration::from_secs(5)), + Ok(true), + "a persistent read error spun or was swallowed" + ); + } + + #[test] + fn a_non_interrupted_read_error_after_data_is_returned_not_retried() { + let reader = ScriptedReader(vec![ + Ok(b"ab".to_vec()), + Err(std::io::Error::from(ErrorKind::BrokenPipe)), + Ok(b"cd".to_vec()), + ]); + let error = drain_tail(reader, 8).expect_err("a broken pipe was retried past"); + assert_eq!(error.kind(), ErrorKind::BrokenPipe); + } + + #[cfg(unix)] + #[test] + fn each_tail_is_the_last_bound_bytes_with_the_total() -> Result<()> { + let temp = tempfile::tempdir()?; + let data = temp.path().join("data"); + for size in [ + 0, + 1, + RUNNER_OUTPUT_TAIL_BYTES - 1, + RUNNER_OUTPUT_TAIL_BYTES, + RUNNER_OUTPUT_TAIL_BYTES + 1, + 1024 * 1024, + ] { + // Distinct neighbours, so a shifted or head-keeping tail differs. + let bytes: Vec = (0..size).map(|index| (index % 251) as u8).collect(); + fs::write(&data, &bytes)?; + let driver = fake_docker( + temp.path(), + &format!("cat '{}'\ncat '{}' >&2\nexit 1", data.display(), data.display()), + )?; + let failure = failure_of(run_step(&driver, temp.path(), "build"))?; + let expected = &bytes[size.saturating_sub(RUNNER_OUTPUT_TAIL_BYTES)..]; + assert_eq!(failure.stdout_tail, expected, "stdout tail at {size} bytes"); + assert_eq!(failure.stderr_tail, expected, "stderr tail at {size} bytes"); + assert_eq!(failure.stdout_total_bytes, size as u64); + assert_eq!(failure.stderr_total_bytes, size as u64); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_successful_step_and_a_missing_program_keep_their_outcomes() -> Result<()> { + let temp = tempfile::tempdir()?; + run_step(&fake_docker(temp.path(), "echo fine")?, temp.path(), "build")?; + let missing = DockerRunnerDriver { + docker_program: temp.path().join("no-such-docker"), + }; + let error = run_step(&missing, temp.path(), "build").expect_err("a missing program ran"); + assert!(error.downcast_ref::().is_none()); + assert!(format!("{error:#}").contains("starting Docker runner"), "{error:#}"); + Ok(()) + } + + /// The 2026-10-25 vili failure: `node --test` reported on stdout alone, + /// stderr stayed empty (the runner's /tmp is a noexec tmpfs) and the step + /// exited 1. The old capture dropped stdout and printed an empty message. + #[cfg(unix)] + #[test] + fn a_step_that_reports_only_on_stdout_keeps_that_report_and_its_name() -> Result<()> { + let temp = tempfile::tempdir()?; + let driver = fake_docker( + temp.path(), + &format!("echo 'not ok 1 - daemon {RUNNER_CANARY}'\necho '# fail 1'\nexit 1"), + )?; + let error = run_step(&driver, temp.path(), "test-daemon").expect_err("the step passed"); + // The error names the step and carries no output. + let text = format!("{error:#}"); + assert!(text.contains("runner step test-daemon on runner rust exited with exit status 1"), "{text}"); + assert!(!text.contains(RUNNER_CANARY) && !text.contains("not ok"), "{text}"); + // The evidence keeps the stdout tail; the empty stderr is a recorded 0. + let failure = error + .downcast_ref::() + .context("the failure is not typed")?; + let store = temp.path().join("runner-evidence.cc"); + let record = crate::runner_evidence::RunnerFailureRecord::from_failure( + "tx-vili", + "up-vili", + "vili", + failure, + 1, + ); + crate::runner_evidence::record(&store, &record)?; + let kept = crate::runner_evidence::for_transaction(&store, "tx-vili")?.context("no evidence kept")?; + assert_eq!(kept.subject_id, "test-daemon"); + assert_eq!(kept.exit_code, Some(1)); + assert_eq!( + String::from_utf8_lossy(&kept.stdout_tail), + format!("not ok 1 - daemon {RUNNER_CANARY}\n# fail 1\n") + ); + assert_eq!(kept.stdout_total_bytes, kept.stdout_tail.len() as u64); + assert!(kept.stderr_tail.is_empty()); + assert_eq!(kept.stderr_total_bytes, 0); + let mut shown = Vec::new(); + crate::runner_evidence::show_status(&mut shown, &store, "tx-vili")?; + let rendered = String::from_utf8(shown)?; + assert!(rendered.contains("stderr (last 0 of 0 bytes)"), "{rendered}"); + assert!(rendered.contains(&format!("| not ok 1 - daemon {RUNNER_CANARY}")), "{rendered}"); + // Debug names the failure and prints no tail. + let debug = format!("{failure:?}"); + assert!(debug.starts_with("RunnerStepFailure") && !debug.contains("not ok"), "{debug}"); + Ok(()) + } + // END runner-failure-evidence tests } diff --git a/src/host.rs b/src/host.rs index 5221f12..4c6cbec 100644 --- a/src/host.rs +++ b/src/host.rs @@ -30,13 +30,13 @@ use serde::Deserialize; use crate::control_plane::read_trust_anchor; use crate::deployment::{ - ArtifactSource, HostWorkloadBinding, IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, + HostWorkloadBinding, IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, LaunchArgument, OperatorBinding, TargetDeclaration, }; -use crate::deployment_plan::{ArtifactReceipt, CompiledDeploymentPlan, SealedRelease}; +use crate::deployment_plan::{CompiledDeploymentPlan, SealedRelease}; use crate::drivers::{ - HostWorkloadObservation, InstalledReleaseObservation, copy_artifact, copy_tree, - digest_artifact, release_artifact, remove_tree_inside, sha256_id, write_runtime_bundle_records, + HostWorkloadObservation, InstalledReleaseObservation, copy_tree, digest_artifact, + release_artifact, remove_tree_inside, seal_artifact, sha256_id, write_runtime_bundle_records, }; use crate::host_actuator::{ HOST_ACTUATOR_CHANNEL, HOST_ACTUATOR_CONNECTION_ID, HOST_ACTUATOR_PING_MS, @@ -1048,37 +1048,12 @@ fn materialize( fs::create_dir_all(&staging)?; let mut artifacts = Vec::new(); for artifact in &declaration.artifacts { - let source_path = match artifact.source_kind { - ArtifactSource::RunnerOutput | ArtifactSource::WorktreeTree => { - source.join(&artifact.source) - } - }; - ensure!( - source_path.exists(), - "declared artifact output {} is absent", - artifact.id - ); - let destination = staging.join(&artifact.destination); - ensure!( - destination.starts_with(&staging), - "artifact destination escaped its staging root" - ); - copy_artifact(&source, &source_path, &destination)?; - let (sha256, size_bytes) = digest_artifact(&destination)?; - if let Some(expected) = &artifact.expected_sha256 { - ensure!( - &sha256 == expected, - "artifact {} differs from its recipe-pinned digest", - artifact.id - ); - } - artifacts.push(ArtifactReceipt { - artifact_id: artifact.id.clone(), - destination: artifact.destination.clone(), - sha256: format!("sha256-{sha256}"), - size_bytes, - executable: artifact.executable, - }); + artifacts.push(seal_artifact( + artifact, + &source, + &source.join(&artifact.source), + &staging, + )?); } let release = SealedRelease::new(plan, artifacts, Vec::new(), sealed_at_unix_millis)?; Ok((release, staging)) diff --git a/src/host_actuator.rs b/src/host_actuator.rs index 90592be..8fd776c 100644 --- a/src/host_actuator.rs +++ b/src/host_actuator.rs @@ -963,6 +963,12 @@ impl WorkloadPort for HostActuatorWorkloadDriver<'_> { }; Ok(!running) } + + /// A managed host's reboot is not yet observable here: follow-up + /// host-actuator-reboot. Until then continuity charges its restarts. + fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { + Ok(false) + } } #[cfg(test)] @@ -1044,6 +1050,43 @@ mod tests { Ok(()) } + /// A managed host's reboot is not observable yet (follow-up + /// host-actuator-reboot): the driver never claims a reboot, so continuity + /// never grants a free re-materialization it cannot justify. + #[test] + fn a_managed_host_workload_is_never_claimed_lost_to_a_reboot() -> Result<()> { + let (_temp, idunn, _host) = identities()?; + let hub = HostActuatorHub::bind("127.0.0.1:0".parse()?)?; + let driver = HostActuatorWorkloadDriver { + access: HostActuatorAccess { + hub: Arc::new(Mutex::new(hub)), + anchors: BTreeMap::new(), + signer: &idunn, + }, + }; + let observation = WorkloadObservation::Host(HostWorkloadObservation { + host: "raven".into(), + actuator_identity_id: "actuator".into(), + process_id: 1, + process_creation_time: 1, + session_id: 1, + user_sid: "S-1-5-21-1-2-3-1001".into(), + executable: "C:/GameCult/idunn/releases/muninn/x/muninn.exe".into(), + executable_sha256: format!("sha256-{}", "1".repeat(64)), + command_line_sha256: format!("sha256-{}", "2".repeat(64)), + environment_names: Vec::new(), + environment_contract_sha256: format!("sha256-{}", "3".repeat(64)), + runtime_bundle: "C:/GameCult/idunn/runtime/muninn/x".into(), + runtime_instance_id: format!("sha256-{}", "a".repeat(64)), + activation_signer_identity_id: "activation".into(), + activation_signer_public_key: vec![1; 32], + exit_code: Some(1), + }); + assert!(!driver.lost_to_host_reboot(&observation, 1)?); + assert!(!driver.lost_to_host_reboot(&observation, u64::MAX)?); + Ok(()) + } + /// A real hub and a real client socket on loopback: Hello attaches under /// the bound anchor, a wrong host is refused, a request is answered. #[test] @@ -1108,7 +1151,7 @@ mod tests { let answerer = thread::spawn(move || -> Result<()> { for _ in 0..500 { client.poll_resends()?; - while let Some(frame) = client.receive_once()? { + if let Some(frame) = client.receive_once()? { let envelope = HostActuatorEnvelope::decode(&frame.payload)?; let request = envelope.open_request("raven", &idunn_anchor)?; assert_eq!(envelope.session_nonce, nonce); diff --git a/src/incident.rs b/src/incident.rs new file mode 100644 index 0000000..a0941ff --- /dev/null +++ b/src/incident.rs @@ -0,0 +1,1046 @@ +//! Operator incidents: the durable record that a condition needing a human +//! opened, and that it closed. +//! +//! One file, `incidents.cc`, holds one type. Idunn's daemon is its only writer, +//! at the site that decides the condition; it is also the file external +//! readers open, so no second copy exists to drift. It is never `control.cc`, +//! which refuses any type it does not know: a type added there would stop +//! continuity for every target on a rollback. +//! +//! A record carries no free text. A workload's error never enters it, so the +//! published file leaks nothing a reader should not have. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, bail, ensure}; +use cultcache_rs::{ + CultCacheEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, + TryCompareExchangeSnapshotOutcome, +}; +use serde::{Deserialize, Serialize}; + +use crate::control_plane::{decode_record, require_id, typed_envelope}; +use crate::drivers::{LockContended, exchange_behind_private_lock, publish_file_mode}; + +const INCIDENT_SCHEMA: &str = "idunn.operator_incident.v1"; + +/// How long a closed incident stays in the published file before it retires to +/// the incident history. +pub(crate) const INCIDENT_RETENTION_MILLIS: u64 = 7 * 24 * 60 * 60 * 1000; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum IncidentCondition { + ContinuityExhausted, + /// A transaction past its deadline in a state Idunn will not end by + /// itself. Subject: the transaction id. + OperatorRequired, +} + +impl IncidentCondition { + /// The spelling in the key and on `idunn status`. Pinned equal to the + /// serialized spelling by `the_incident_fixture_is_what_idunn_reads`. + pub(crate) const fn name(self) -> &'static str { + match self { + Self::ContinuityExhausted => "continuity-exhausted", + Self::OperatorRequired => "operator-required", + } + } + + /// Whether the subject is an admitted target, so that the target leaving + /// admission ends the condition. A condition about anything else (a + /// transaction, a store) answers false and is closed by its own owner. + pub(crate) const fn ends_with_admission(self) -> bool { + match self { + Self::ContinuityExhausted => true, + Self::OperatorRequired => false, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum CloseReason { + Recovered, + NoLongerAdmitted, +} + +impl CloseReason { + pub(crate) const fn name(self) -> &'static str { + match self { + Self::Recovered => "recovered", + Self::NoLongerAdmitted => "no-longer-admitted", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "idunn.operator_incident", schema = "idunn.operator_incident.v1")] +pub(crate) struct IncidentRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + /// `::`. Injective because at most one + /// incident per `(condition, subject)` is open at a time. + #[cultcache(key = 1)] + pub(crate) incident_key: String, + #[cultcache(key = 2)] + pub(crate) condition: IncidentCondition, + /// The target the condition is about. + #[cultcache(key = 3)] + pub(crate) subject: String, + #[cultcache(key = 4)] + pub(crate) opened_at_unix_millis: u64, + #[cultcache(key = 5)] + pub(crate) closed_at_unix_millis: Option, + #[cultcache(key = 6)] + pub(crate) close_reason: Option, +} + +impl IncidentRecord { + fn opened(condition: IncidentCondition, subject: &str, now: u64) -> Self { + Self { + schema_version: INCIDENT_SCHEMA.to_owned(), + incident_key: format!("{}:{subject}:{now}", condition.name()), + condition, + subject: subject.to_owned(), + opened_at_unix_millis: now, + closed_at_unix_millis: None, + close_reason: None, + } + } + + pub(crate) fn is_open(&self) -> bool { + self.closed_at_unix_millis.is_none() + } + + /// Whether the record still belongs in the published file: open, or closed + /// no more than `INCIDENT_RETENTION_MILLIS` ago. The one predicate behind + /// both retirement and `idunn status`. + pub(crate) fn inside_retention(&self, now: u64) -> bool { + self.closed_at_unix_millis + .is_none_or(|closed_at| now.saturating_sub(closed_at) <= INCIDENT_RETENTION_MILLIS) + } + + pub(crate) fn validate(&self) -> Result<()> { + ensure!( + self.schema_version == INCIDENT_SCHEMA, + "incident schema is unsupported" + ); + require_id(&self.subject, "incident subject")?; + ensure!( + self.incident_key + == format!( + "{}:{}:{}", + self.condition.name(), + self.subject, + self.opened_at_unix_millis + ), + "incident key is not its condition, subject and opening time" + ); + ensure!( + self.closed_at_unix_millis.is_some() == self.close_reason.is_some(), + "incident has a closing time or a reason but not both" + ); + if let Some(closed_at) = self.closed_at_unix_millis { + ensure!( + closed_at >= self.opened_at_unix_millis, + "incident closes before it opens" + ); + } + Ok(()) + } + + fn envelope(&self, now: u64) -> Result { + self.validate()?; + typed_envelope( + &self.incident_key, + Self::TYPE, + INCIDENT_SCHEMA, + self, + now, + ) + } +} + +/// One read of the incident file: its envelopes exactly as stored, and the +/// records decoded from them, in the same order. Every write expects exactly +/// these envelopes, so a write only lands on the file it was decided against. +struct Snapshot { + envelopes: Vec, + records: Vec, +} + +impl Snapshot { + fn open_record(&self, condition: IncidentCondition, subject: &str) -> Option { + self.records.iter().position(|record| { + record.is_open() && record.condition == condition && record.subject == subject + }) + } +} + +/// The incident file and its history sibling. Incidents are written from the +/// scheduler tick, so no write waits: each is one nonblocking compare-exchange +/// of the whole file against the snapshot it was decided on. A lost race or a +/// held lock is an error and a no-op, retried on a later pass. The lock beside +/// the file is not published and is Idunn's alone +/// (`exchange_behind_private_lock`): the readers of `incidents.cc` take no +/// lock, so only Idunn can ever hold it. +pub(crate) struct IncidentStore { + path: PathBuf, +} + +impl IncidentStore { + pub(crate) fn new(path: &Path) -> Self { + Self { + path: path.to_path_buf(), + } + } + + fn history_path(&self) -> PathBuf { + self.path.with_file_name("incident-history.cc") + } + + /// Every record, oldest key first. An absent file is empty. The file + /// refuses any document that is not an operator incident. + pub(crate) fn read(&self) -> Result> { + let mut records = self.snapshot()?.records; + records.sort_by(|left, right| left.incident_key.cmp(&right.incident_key)); + Ok(records) + } + + fn snapshot(&self) -> Result { + let envelopes = SingleFileMessagePackBackingStore::new(&self.path) + .pull_all_read_only_snapshot() + .context("reading the incident store")?; + let mut records = Vec::new(); + for envelope in &envelopes { + ensure!( + envelope.r#type == IncidentRecord::TYPE + && envelope.schema_id.as_deref() == Some(INCIDENT_SCHEMA), + "the incident store holds a document that is not an operator incident" + ); + let record: IncidentRecord = decode_record(envelope)?; + record.validate()?; + ensure!( + envelope.key == record.incident_key, + "incident store key differs from the incident's identity" + ); + records.push(record); + } + Ok(Snapshot { envelopes, records }) + } + + /// Replace `path`'s whole content with `replacements` if it is still + /// exactly `expected`. Never waits for the lock. + fn exchange( + path: &Path, + expected: &[CultCacheEnvelope], + replacements: &[CultCacheEnvelope], + ) -> Result<()> { + match exchange_behind_private_lock(path, expected, replacements)? { + TryCompareExchangeSnapshotOutcome::Exchanged => Ok(()), + TryCompareExchangeSnapshotOutcome::Mismatch => { + bail!("the incident store changed while it was being written") + } + TryCompareExchangeSnapshotOutcome::LockContended => { + Err(LockContended { store: "incident store" }.into()) + } + } + } + + /// Open an incident unless one is already open for `(condition, subject)`. + /// `Ok(true)` means this call wrote it. + pub(crate) fn open( + &self, + condition: IncidentCondition, + subject: &str, + now: u64, + ) -> Result { + self.open_on(&self.snapshot()?, condition, subject, now) + } + + fn open_on( + &self, + snapshot: &Snapshot, + condition: IncidentCondition, + subject: &str, + now: u64, + ) -> Result { + if snapshot.open_record(condition, subject).is_some() { + return Ok(false); + } + let mut replacements = snapshot.envelopes.clone(); + replacements.push(IncidentRecord::opened(condition, subject, now).envelope(now)?); + Self::exchange(&self.path, &snapshot.envelopes, &replacements)?; + publish_file_mode(&self.path)?; + Ok(true) + } + + /// The subjects of every open incident of `condition`. + pub(crate) fn open_subjects(&self, condition: IncidentCondition) -> Result> { + Ok(self + .snapshot()? + .records + .into_iter() + .filter(|record| record.is_open() && record.condition == condition) + .map(|record| record.subject) + .collect()) + } + + /// Close the open incident for `(condition, subject)`, if there is one. + /// `Ok(true)` means this call wrote the closure. + pub(crate) fn close( + &self, + condition: IncidentCondition, + subject: &str, + reason: CloseReason, + now: u64, + ) -> Result { + let snapshot = self.snapshot()?; + let Some(index) = snapshot.open_record(condition, subject) else { + return Ok(false); + }; + let mut closed = snapshot.records[index].clone(); + // A clock stepped back must not close an incident before it opened. + closed.closed_at_unix_millis = Some(now.max(closed.opened_at_unix_millis)); + closed.close_reason = Some(reason); + let mut replacements = snapshot.envelopes.clone(); + replacements[index] = closed.envelope(now)?; + Self::exchange(&self.path, &snapshot.envelopes, &replacements)?; + publish_file_mode(&self.path)?; + Ok(true) + } + + /// Move every incident that has left retention to the history file, then + /// drop it here. History first: a crash between the two leaves the record + /// in both, and the repeat finds it already archived. An open incident + /// never moves. + pub(crate) fn retire_closed(&self, now: u64) -> Result { + let snapshot = self.snapshot()?; + let (kept, retiring): (Vec<_>, Vec<_>) = snapshot + .envelopes + .iter() + .zip(&snapshot.records) + .partition(|(_, record)| record.inside_retention(now)); + if retiring.is_empty() { + return Ok(false); + } + let history_path = self.history_path(); + let archived = SingleFileMessagePackBackingStore::new(&history_path) + .pull_all_read_only_snapshot() + .context("reading the incident history")?; + let mut history = archived.clone(); + history.extend( + retiring + .iter() + .filter(|(envelope, _)| !archived.iter().any(|held| held.key == envelope.key)) + .map(|(envelope, _)| (*envelope).clone()), + ); + if history.len() != archived.len() { + Self::exchange(&history_path, &archived, &history)?; + } + let kept = kept + .into_iter() + .map(|(envelope, _)| envelope.clone()) + .collect::>(); + Self::exchange(&self.path, &snapshot.envelopes, &kept)?; + publish_file_mode(&self.path)?; + Ok(true) + } +} + +/// The lines `idunn status` prints for one target's incidents: each open one, +/// and each closed one still inside retention. +pub(crate) fn render_incidents(records: &[IncidentRecord], target: &str, now: u64) -> Vec { + records + .iter() + .filter(|record| record.subject == target && record.inside_retention(now)) + .map(|record| { + let mut line = format!( + " incident {} opened-at {}", + record.condition.name(), + record.opened_at_unix_millis + ); + if let (Some(closed_at), Some(reason)) = + (record.closed_at_unix_millis, record.close_reason) + { + line.push_str(&format!(" closed-at {closed_at} {}", reason.name())); + } + line + }) + .collect() +} + +#[cfg(test)] +pub(crate) mod tests { + #[cfg(target_os = "linux")] + use std::time::{Duration, Instant}; + + use tempfile::TempDir; + + use super::*; + + const OPENED: u64 = 1_700_000_000_000; + const CLOSED: u64 = 1_700_000_100_000; + const FIXTURE: &str = "tests/fixtures/idunn.operator_incident.v1.cc"; + + fn fixture_path() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join(FIXTURE) + } + + /// The two records the fixture holds, spelled independently of the store: + /// one open, one closed. + fn fixture_records() -> [IncidentRecord; 2] { + let closed_key = format!("continuity-exhausted:closed-target:{OPENED}"); + let open_key = format!("continuity-exhausted:open-target:{CLOSED}"); + [ + IncidentRecord { + schema_version: "idunn.operator_incident.v1".into(), + incident_key: closed_key, + condition: IncidentCondition::ContinuityExhausted, + subject: "closed-target".into(), + opened_at_unix_millis: OPENED, + closed_at_unix_millis: Some(CLOSED), + close_reason: Some(CloseReason::Recovered), + }, + IncidentRecord { + schema_version: "idunn.operator_incident.v1".into(), + incident_key: open_key, + condition: IncidentCondition::ContinuityExhausted, + subject: "open-target".into(), + opened_at_unix_millis: CLOSED, + closed_at_unix_millis: None, + close_reason: None, + }, + ] + } + + #[test] + fn incident_records_validate_their_key_and_closure() { + let [closed, open] = fixture_records(); + closed.validate().unwrap(); + open.validate().unwrap(); + + let mut wrong_key = open.clone(); + wrong_key.incident_key.push('x'); + assert!(wrong_key.validate().is_err()); + + let mut reason_missing = open.clone(); + reason_missing.closed_at_unix_millis = Some(CLOSED + 1); + assert!(reason_missing.validate().is_err()); + + let mut closing_missing = open.clone(); + closing_missing.close_reason = Some(CloseReason::Recovered); + assert!(closing_missing.validate().is_err()); + + let mut before_open = closed.clone(); + before_open.closed_at_unix_millis = Some(OPENED - 1); + assert!(before_open.validate().is_err()); + + let mut empty_subject = open.clone(); + empty_subject.subject = String::new(); + empty_subject.incident_key = format!("continuity-exhausted::{CLOSED}"); + assert!(empty_subject.validate().is_err()); + + let mut wrong_schema = open; + wrong_schema.schema_version = "idunn.operator_incident.v0".into(); + assert!(wrong_schema.validate().is_err()); + } + + #[test] + fn the_incident_fixture_is_what_idunn_reads() { + if std::env::var_os("IDUNN_WRITE_FIXTURES").is_some() { + let temp = TempDir::new().unwrap(); + let store = IncidentStore::new(&temp.path().join("incidents.cc")); + assert!( + store + .open(IncidentCondition::ContinuityExhausted, "closed-target", OPENED) + .unwrap() + ); + assert!( + store + .close( + IncidentCondition::ContinuityExhausted, + "closed-target", + CloseReason::Recovered, + CLOSED + ) + .unwrap() + ); + assert!( + store + .open(IncidentCondition::ContinuityExhausted, "open-target", CLOSED) + .unwrap() + ); + std::fs::copy(temp.path().join("incidents.cc"), fixture_path()).unwrap(); + } + // A copy, so reading the committed file can never leave a lock beside it. + let temp = TempDir::new().unwrap(); + let copy = temp.path().join("incidents.cc"); + std::fs::copy(fixture_path(), ©).unwrap(); + assert_eq!( + IncidentStore::new(©).read().unwrap(), + fixture_records().to_vec() + ); + // The wire spelling readers match on is the spelling the key and + // status use. + for condition in [IncidentCondition::ContinuityExhausted, IncidentCondition::OperatorRequired] { + let payload = rmp_serde::to_vec(&condition).unwrap(); + assert_eq!(rmp_serde::from_slice::(&payload).unwrap(), condition.name()); + } + assert_eq!(IncidentCondition::OperatorRequired.name(), "operator-required"); + assert!(!IncidentCondition::OperatorRequired.ends_with_admission()); + let payload = rmp_serde::to_vec(&CloseReason::NoLongerAdmitted).unwrap(); + assert_eq!( + rmp_serde::from_slice::(&payload).unwrap(), + CloseReason::NoLongerAdmitted.name() + ); + } + + #[test] + fn the_store_opens_once_closes_once_and_reopens_as_a_new_record() { + let temp = TempDir::new().unwrap(); + let store = IncidentStore::new(&temp.path().join("incidents.cc")); + let condition = IncidentCondition::ContinuityExhausted; + assert!(store.read().unwrap().is_empty()); + assert!(!store.close(condition, "service", CloseReason::Recovered, OPENED).unwrap()); + + assert!(store.open(condition, "service", OPENED).unwrap()); + assert!(!store.open(condition, "service", OPENED + 5).unwrap()); + assert!(store.open(condition, "other", OPENED).unwrap()); + + assert!(store.close(condition, "service", CloseReason::Recovered, CLOSED).unwrap()); + assert!(!store.close(condition, "service", CloseReason::Recovered, CLOSED).unwrap()); + + assert!(store.open(condition, "service", CLOSED + 1).unwrap()); + let records = store.read().unwrap(); + assert_eq!(records.len(), 3); + let keys = records.iter().map(|r| r.incident_key.as_str()).collect::>(); + assert!(keys.contains(&format!("continuity-exhausted:service:{OPENED}").as_str())); + assert!(keys.contains(&format!("continuity-exhausted:service:{}", CLOSED + 1).as_str())); + + // A clock stepped back closes at the opening time, never before it. + assert!(store.close(condition, "other", CloseReason::NoLongerAdmitted, OPENED - 9).unwrap()); + let other = store.read().unwrap().into_iter().find(|r| r.subject == "other").unwrap(); + assert_eq!(other.closed_at_unix_millis, Some(OPENED)); + assert_eq!(other.close_reason, Some(CloseReason::NoLongerAdmitted)); + } + + #[test] + fn closed_incidents_retire_to_history_after_retention() { + // The ruled retention: seven days. + assert_eq!(INCIDENT_RETENTION_MILLIS, 604_800_000); + let temp = TempDir::new().unwrap(); + let path = temp.path().join("incidents.cc"); + let store = IncidentStore::new(&path); + let condition = IncidentCondition::ContinuityExhausted; + for subject in ["old", "young", "open"] { + assert!(store.open(condition, subject, OPENED).unwrap()); + } + assert!(store.close(condition, "old", CloseReason::Recovered, OPENED + 1).unwrap()); + let young_closed = OPENED + INCIDENT_RETENTION_MILLIS; + assert!(store.close(condition, "young", CloseReason::Recovered, young_closed).unwrap()); + + // Exactly at the boundary nothing has aged out; one millisecond later + // only the older closure has. + let boundary = OPENED + 1 + INCIDENT_RETENTION_MILLIS; + assert!(!store.retire_closed(boundary).unwrap()); + assert!(store.retire_closed(boundary + 1).unwrap()); + assert!(!store.retire_closed(boundary + 1).unwrap()); + + let live = store.read().unwrap(); + let mut subjects = live.iter().map(|r| r.subject.as_str()).collect::>(); + subjects.sort_unstable(); + assert_eq!(subjects, ["open", "young"]); + + let history = SingleFileMessagePackBackingStore::new(&temp.path().join("incident-history.cc")) + .pull_all_read_only_snapshot() + .unwrap(); + assert_eq!(history.len(), 1); + assert_eq!(history[0].key, format!("continuity-exhausted:old:{OPENED}")); + + // Later the younger one retires too; the open one never does. + assert!(store.retire_closed(young_closed + INCIDENT_RETENTION_MILLIS + 1).unwrap()); + let live = store.read().unwrap(); + assert_eq!(live.len(), 1); + assert_eq!(live[0].subject, "open"); + } + + #[test] + fn the_store_refuses_any_document_that_is_not_an_incident() { + let temp = TempDir::new().unwrap(); + let path = temp.path().join("incidents.cc"); + let foreign = CultCacheEnvelope { + key: "x".into(), + r#type: "idunn.target_supervision".into(), + payload: vec![0x90], + stored_at: "2026-01-01T00:00:00+00:00".into(), + schema_id: Some("idunn.target_supervision.v1".into()), + }; + SingleFileMessagePackBackingStore::new(&path) + .insert_entry_if_absent(foreign) + .unwrap(); + let store = IncidentStore::new(&path); + assert!(store.read().is_err()); + assert!( + store + .open(IncidentCondition::ContinuityExhausted, "service", OPENED) + .is_err() + ); + } + + #[test] + fn status_renders_open_and_recently_closed_incidents_for_their_target() { + let [closed, open] = fixture_records(); + let records = [closed, open]; + let lines = render_incidents(&records, "closed-target", CLOSED + 1); + assert_eq!( + lines, + [format!( + " incident continuity-exhausted opened-at {OPENED} closed-at {CLOSED} recovered" + )] + ); + assert_eq!( + render_incidents(&records, "open-target", CLOSED + 1), + [format!(" incident continuity-exhausted opened-at {CLOSED}")] + ); + assert!(render_incidents(&records, "unrelated", CLOSED + 1).is_empty()); + // A closed incident past retention is history, not status. + assert!( + render_incidents(&records, "closed-target", CLOSED + INCIDENT_RETENTION_MILLIS + 1) + .is_empty() + ); + assert_eq!( + render_incidents(&records, "closed-target", CLOSED + INCIDENT_RETENTION_MILLIS).len(), + 1 + ); + } + + /// Idunn runs with `UMask=027`. The umask is process-wide, so it is held + /// only around each call and only one test at a time may hold it. + #[cfg(target_os = "linux")] + pub(crate) fn under_service_umask(write: impl FnOnce() -> T) -> T { + static UMASK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + let _held = UMASK.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + // SAFETY: umask only swaps the process file-creation mask. + let previous = unsafe { libc::umask(0o027) }; + let result = write(); + unsafe { libc::umask(previous) }; + result + } + + /// The body's condition: a directory carrying the default ACL + /// `u::rw g::r o::r` that Yggdrasil's idunn-projection directory has. Under + /// it the umask is ignored and a file created with mode 0666 lands 0644. + #[cfg(target_os = "linux")] + pub(crate) fn give_default_acl(dir: &Path) { + use std::os::unix::ffi::OsStrExt; + + // posix_acl_xattr: version 2, then (tag, perm, id) entries. The tags + // are ACL_USER_OBJ, ACL_GROUP_OBJ and ACL_OTHER; the id is undefined. + let mut value = 2u32.to_le_bytes().to_vec(); + for (tag, perm) in [(0x01u16, 6u16), (0x04, 4), (0x20, 4)] { + value.extend(tag.to_le_bytes()); + value.extend(perm.to_le_bytes()); + value.extend(u32::MAX.to_le_bytes()); + } + let dir = std::ffi::CString::new(dir.as_os_str().as_bytes()).unwrap(); + // SAFETY: both strings are NUL-terminated and the value is a live buffer. + let rc = unsafe { + libc::setxattr( + dir.as_ptr(), + c"system.posix_acl_default".as_ptr(), + value.as_ptr().cast(), + value.len(), + 0, + ) + }; + assert_eq!( + rc, + 0, + "the test filesystem must accept a default POSIX ACL: {}", + std::io::Error::last_os_error() + ); + } + + /// Whether uid 65534 can read `path`; `None` when the uid cannot be + /// switched (not root, or no setpriv), which leaves the question unproven. + #[cfg(target_os = "linux")] + pub(crate) fn readable_by_another_uid(path: &Path) -> Option { + std::process::Command::new("setpriv") + .args(["--reuid=65534", "--regid=65534", "--clear-groups", "cat"]) + .arg(path) + .output() + .ok() + .map(|output| output.status.success()) + } + + #[cfg(target_os = "linux")] + #[test] + fn under_a_default_acl_the_store_is_published_and_its_locks_are_private() { + use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; + + let temp = TempDir::new().unwrap(); + let dir = temp.path(); + std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o755)).unwrap(); + give_default_acl(dir); + let path = dir.join("incidents.cc"); + let lock = dir.join("incidents.cc.lock"); + let history_lock = dir.join("incident-history.cc.lock"); + let store = IncidentStore::new(&path); + let condition = IncidentCondition::ContinuityExhausted; + let mode = |path: &Path| std::fs::metadata(path).unwrap().permissions().mode() & 0o777; + + // Control: the lock CultCache would have made here, open(2) mode 0666 + // under Idunn's umask. Without the ACL in force this reads 0640 and the + // test below proves nothing about the body. + let control = dir.join("control.lock"); + under_service_umask(|| { + std::fs::OpenOptions::new().create(true).write(true).mode(0o666).open(&control) + }) + .unwrap(); + assert_eq!(mode(&control), 0o644, "the default ACL is not in force"); + + assert!(under_service_umask(|| store.open(condition, "service", OPENED)).unwrap()); + assert_eq!(mode(&path), 0o644, "after open"); + assert_eq!(mode(&lock), 0o600, "the lock after open"); + + assert!( + under_service_umask(|| store.close(condition, "service", CloseReason::Recovered, CLOSED)) + .unwrap() + ); + assert_eq!(mode(&path), 0o644, "after close"); + assert_eq!(mode(&lock), 0o600, "the lock after close"); + + assert!( + under_service_umask(|| store.retire_closed(CLOSED + INCIDENT_RETENTION_MILLIS + 1)) + .unwrap() + ); + assert_eq!(mode(&path), 0o644, "after retire"); + assert_eq!(mode(&lock), 0o600, "the lock after retire"); + assert_eq!(mode(&history_lock), 0o600, "the history lock after retire"); + + // The consequence: another uid cannot open the lock, so it cannot hold + // it. Provable only as root with setpriv, and only if the same uid can + // read the control file, which shows the switch itself works. + if readable_by_another_uid(&control) == Some(true) { + assert_eq!(readable_by_another_uid(&lock), Some(false)); + assert_eq!(readable_by_another_uid(&history_lock), Some(false)); + } else { + eprintln!("UNPROVEN: no uid switch here; the lock modes above stand alone"); + } + } + + #[cfg(unix)] + #[test] + fn a_lock_left_wider_by_an_earlier_run_is_replaced_before_it_is_used() { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let temp = TempDir::new().unwrap(); + let path = temp.path().join("incidents.cc"); + let lock = temp.path().join("incidents.cc.lock"); + std::fs::write(&lock, b"").unwrap(); + std::fs::set_permissions(&lock, std::fs::Permissions::from_mode(0o666)).unwrap(); + let before = std::fs::metadata(&lock).unwrap().ino(); + + assert!( + IncidentStore::new(&path) + .open(IncidentCondition::ContinuityExhausted, "service", OPENED) + .unwrap() + ); + let after = std::fs::metadata(&lock).unwrap(); + assert_eq!(after.permissions().mode() & 0o777, 0o600); + assert_ne!(after.ino(), before, "the wider lock was chmodded, not replaced"); + } + + /// The lock beside a store, spelled independently of the code under test. + #[cfg(target_os = "linux")] + fn lock_beside(store: &Path) -> PathBuf { + let mut name = store.file_name().unwrap().to_owned(); + name.push(".lock"); + store.with_file_name(name) + } + + /// An exclusive flock on a store's lock, held by a SEPARATE PROCESS + /// (`flock -x sleep`), so the holder shares no descriptor, thread + /// or address space with the code under test. A watch on the lock counts + /// every open of it from the moment it is held. + #[cfg(target_os = "linux")] + pub(crate) struct HeldLock { + holder: std::sync::Arc>, + watch: std::os::fd::OwnedFd, + } + + /// Hold the lock of `store`. An absent lock is created as production keeps + /// it (0600, ours); a lock a test put there first is held as it is, never + /// chmodded. + #[cfg(target_os = "linux")] + pub(crate) fn hold_lock_of(store: &Path) -> HeldLock { + use std::os::fd::{FromRawFd, OwnedFd}; + use std::os::unix::ffi::OsStrExt; + use std::os::unix::fs::OpenOptionsExt; + use std::os::unix::io::AsRawFd; + + let lock = lock_beside(store); + std::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(false) + .mode(0o600) + .open(&lock) + .unwrap(); + let mut holder = std::process::Command::new("flock") + .args(["--no-fork", "--exclusive"]) + .arg(&lock) + .args(["sleep", "3600"]) + .spawn() + .expect("flock (util-linux) must be installed to hold a lock from another process"); + + // Held once a nonblocking exclusive flock from here is refused. + let deadline = Instant::now() + STUCK_AFTER; + loop { + assert!(holder.try_wait().unwrap().is_none(), "the lock holder exited"); + let probe = std::fs::File::open(&lock).unwrap(); + // SAFETY: flock on a descriptor this function owns. + let refused = unsafe { libc::flock(probe.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } != 0; + if refused { + assert_eq!( + std::io::Error::last_os_error().kind(), + std::io::ErrorKind::WouldBlock + ); + break; + } + drop(probe); + assert!(Instant::now() < deadline, "the lock was never held"); + std::thread::sleep(Duration::from_millis(10)); + } + + // Open and close alternate, so two opens are never identical adjacent + // events for the kernel to coalesce into one. + // SAFETY: plain syscalls on descriptors and strings this function owns. + let watch = unsafe { + let fd = libc::inotify_init1(libc::IN_CLOEXEC | libc::IN_NONBLOCK); + assert!(fd >= 0, "{}", std::io::Error::last_os_error()); + let watch = OwnedFd::from_raw_fd(fd); + let path = std::ffi::CString::new(lock.as_os_str().as_bytes()).unwrap(); + let wd = libc::inotify_add_watch( + fd, + path.as_ptr(), + libc::IN_OPEN | libc::IN_CLOSE_WRITE | libc::IN_CLOSE_NOWRITE, + ); + assert!(wd >= 0, "{}", std::io::Error::last_os_error()); + watch + }; + HeldLock { + holder: std::sync::Arc::new(std::sync::Mutex::new(holder)), + watch, + } + } + + #[cfg(target_os = "linux")] + impl HeldLock { + /// Whether the holder process is still running and so still holds. + pub(crate) fn is_held(&self) -> bool { + self.holder.lock().unwrap().try_wait().unwrap().is_none() + } + + /// How many times the lock has been opened since it was held. + fn opens(&self) -> usize { + use std::os::unix::io::AsRawFd; + + let mut opens = 0; + let mut buffer = [0u8; 4096]; + loop { + // SAFETY: reads into a live buffer of the stated length. + let read = unsafe { + libc::read(self.watch.as_raw_fd(), buffer.as_mut_ptr().cast(), buffer.len()) + }; + if read <= 0 { + return opens; + } + let read = usize::try_from(read).unwrap(); + let mut at = 0; + // struct inotify_event: wd i32, mask u32, cookie u32, len u32, name[len]. + while at + 16 <= read { + let word = |offset: usize| { + u32::from_ne_bytes(buffer[at + offset..at + offset + 4].try_into().unwrap()) + }; + if word(4) & libc::IN_OPEN != 0 { + opens += 1; + } + at += 16 + usize::try_from(word(12)).unwrap(); + } + } + } + } + + #[cfg(target_os = "linux")] + impl Drop for HeldLock { + fn drop(&mut self) { + let mut holder = self.holder.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + let _ = holder.kill(); + let _ = holder.wait(); + } + } + + /// How long a test lets a held lock, or a released lock's last straggler, + /// stand in its way before it calls the store stuck. Nothing correct ever + /// spends it: the code under test finishes in milliseconds, and the limit + /// is only what stops a blocked write from hanging the suite. No + /// assertion compares elapsed time to a bound the machine's load can move. + #[cfg(target_os = "linux")] + pub(crate) const STUCK_AFTER: Duration = Duration::from_secs(30); + + /// Run `work` while `held` is held, and fail if it waited on it. The rule + /// is observed where it is decided, at the lock, not on a clock: every + /// exchange opens the lock exactly once, so `exchanges` is the number of + /// exchanges `work` makes against this lock, and a wait of any length (a + /// sleep and retry, a poll, a second attempt) opens it more often. A wait + /// that never ends is freed by the holder's release after `STUCK_AFTER`, + /// and is then seen to have waited, so the test fails instead of hanging. + /// The holder is still holding when `work` returns, and is released on the + /// way out. + #[cfg(target_os = "linux")] + pub(crate) fn run_while_held( + held: HeldLock, + exchanges: usize, + work: impl FnOnce() -> T, + ) -> T { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, mpsc}; + + let (finished, work_done) = mpsc::channel::<()>(); + let gave_up = Arc::new(AtomicBool::new(false)); + let watchdog = { + let gave_up = Arc::clone(&gave_up); + let holder = Arc::clone(&held.holder); + std::thread::spawn(move || { + if work_done.recv_timeout(STUCK_AFTER) == Err(mpsc::RecvTimeoutError::Timeout) { + gave_up.store(true, Ordering::SeqCst); + let _ = holder.lock().unwrap().kill(); + } + }) + }; + let result = work(); + let still_held = held.is_held(); + let _ = finished.send(()); + watchdog.join().unwrap(); + assert!(!gave_up.load(Ordering::SeqCst), "the work waited on a held lock"); + assert!(still_held, "the holder let go before the work returned"); + assert_eq!( + held.opens(), + exchanges, + "the work opened the held lock other than once per exchange: it waited or retried" + ); + result + } + + /// Retry `attempt` until it lands. A lock just released can take a moment + /// to be seen free. + #[cfg(target_os = "linux")] + pub(crate) fn eventually(mut attempt: impl FnMut() -> Result) -> T { + let deadline = Instant::now() + STUCK_AFTER; + loop { + match attempt() { + Ok(value) => return value, + Err(error) if Instant::now() >= deadline => panic!("never landed: {error:#}"), + Err(_) => std::thread::sleep(Duration::from_millis(20)), + } + } + } + + #[cfg(target_os = "linux")] + #[test] + fn no_incident_write_waits_on_a_held_lock_and_each_lands_when_it_is_freed() { + let temp = TempDir::new().unwrap(); + let path = temp.path().join("incidents.cc"); + let store = IncidentStore::new(&path); + let condition = IncidentCondition::ContinuityExhausted; + assert!(store.open(condition, "closing", OPENED).unwrap()); + assert!(store.close(condition, "closing", CloseReason::Recovered, OPENED + 1).unwrap()); + assert!(store.open(condition, "open", OPENED).unwrap()); + let before = std::fs::read(&path).unwrap(); + let retired = OPENED + 1 + INCIDENT_RETENTION_MILLIS + 1; + + // One exchange each: open, close, and the retirement's store delete. + let outcomes = run_while_held(hold_lock_of(&path), 3, || { + ( + store.open(condition, "new", CLOSED), + store.close(condition, "open", CloseReason::Recovered, CLOSED), + store.retire_closed(retired), + ) + }); + for outcome in [outcomes.0.map(drop), outcomes.1.map(drop), outcomes.2.map(drop)] { + let error = outcome.unwrap_err().to_string(); + assert_eq!(error, "the incident store is locked by another holder"); + } + assert_eq!(std::fs::read(&path).unwrap(), before, "a contended write changed the file"); + + // Nothing was lost by the refusal: the same calls land once it is free. + // (Retirement archives first, so its refused delete left the record in + // history too; the repeat archives nothing twice.) + assert!(eventually(|| store.open(condition, "new", CLOSED))); + assert!(eventually(|| store.close(condition, "open", CloseReason::Recovered, CLOSED))); + assert!(eventually(|| store.retire_closed(retired))); + let history = SingleFileMessagePackBackingStore::new(&temp.path().join("incident-history.cc")); + assert_eq!(history.pull_all_read_only_snapshot().unwrap().len(), 1); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_held_history_lock_keeps_the_incident_where_it_is() { + let temp = TempDir::new().unwrap(); + let path = temp.path().join("incidents.cc"); + let history = temp.path().join("incident-history.cc"); + let store = IncidentStore::new(&path); + let condition = IncidentCondition::ContinuityExhausted; + assert!(store.open(condition, "old", OPENED).unwrap()); + assert!(store.close(condition, "old", CloseReason::Recovered, OPENED + 1).unwrap()); + let before = std::fs::read(&path).unwrap(); + let retired = OPENED + 1 + INCIDENT_RETENTION_MILLIS + 1; + + let outcome = run_while_held(hold_lock_of(&history), 1, || store.retire_closed(retired)); + assert!(outcome.is_err()); + assert_eq!(std::fs::read(&path).unwrap(), before); + assert!(eventually(|| store.retire_closed(retired))); + } + + #[test] + fn a_write_decided_on_a_stale_snapshot_does_not_land() { + let temp = TempDir::new().unwrap(); + let path = temp.path().join("incidents.cc"); + let store = IncidentStore::new(&path); + let condition = IncidentCondition::ContinuityExhausted; + + // Two writers read the file with no incident open for the subject. + let stale = store.snapshot().unwrap(); + assert!(store.open(condition, "service", OPENED).unwrap()); + + // The slower one must lose: one open incident per subject. + let error = store + .open_on(&stale, condition, "service", OPENED + 1) + .unwrap_err(); + assert!(error.to_string().contains("changed"), "{error}"); + let records = store.read().unwrap(); + assert_eq!(records.len(), 1); + assert_eq!(records[0].opened_at_unix_millis, OPENED); + } + + #[test] + fn retiring_after_a_crash_between_history_and_delete_archives_once() { + let temp = TempDir::new().unwrap(); + let path = temp.path().join("incidents.cc"); + let store = IncidentStore::new(&path); + let condition = IncidentCondition::ContinuityExhausted; + assert!(store.open(condition, "old", OPENED).unwrap()); + assert!(store.close(condition, "old", CloseReason::Recovered, OPENED + 1).unwrap()); + // History already holds the record: the delete never ran. + let envelope = store.snapshot().unwrap().envelopes[0].clone(); + let history = SingleFileMessagePackBackingStore::new(&temp.path().join("incident-history.cc")); + assert!(history.insert_entry_if_absent(envelope).unwrap()); + + assert!(store.retire_closed(OPENED + 1 + INCIDENT_RETENTION_MILLIS + 1).unwrap()); + assert!(store.read().unwrap().is_empty()); + assert_eq!(history.pull_all_read_only_snapshot().unwrap().len(), 1); + } +} diff --git a/src/launcher.rs b/src/launcher.rs new file mode 100644 index 0000000..435f8b1 --- /dev/null +++ b/src/launcher.rs @@ -0,0 +1,3238 @@ +//! idunn-launch: the unit's process for an upstream daemon that cannot speak +//! Idunn's runtime contract itself (Synapse, MAS). +//! +//! Idunn observes the unit; the launcher, inside it and supervising the child, +//! signs the presence statement with the provider identity and the activation +//! key. The launcher binds the candidate endpoint Idunn challenges, because the +//! child cannot answer a CultNet frame, and answers presence there and nowhere +//! else: members reach the child on the member bind Idunn routes them to. It +//! starts a stateful child only after it holds the process write lease, renders +//! the child's config from a closed set of placeholders, and ends with the +//! child. +//! +//! Errors, logs and refusals name a field and a code. They never echo an +//! argument, a template line, a path or a credential. + +use std::collections::VecDeque; +use std::fs::{self, File, OpenOptions}; +use std::io::{self, Read, Write}; +use std::net::{SocketAddr, TcpListener, TcpStream, UdpSocket}; +use std::os::fd::{AsRawFd, RawFd}; +use std::os::unix::fs::OpenOptionsExt; +use std::os::unix::process::CommandExt; +use std::path::{Component, Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result, anyhow, bail, ensure}; +use cultcache_rs::{DatabaseEntry, SingleFileMessagePackBackingStore}; +use cultnet_rs::{ + CultNetMessage, CultNetRawDocumentRecord, CultNetRawPayloadEncoding, + CultNetRudpReliableSendStatus, CultNetRudpSocketTransportConnection, + CultNetRudpSocketTransportOptions, CultNetWireContract, + GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, GameCultProviderHealthIdentity, + GameCultRuntimeCapability, GameCultRuntimePresenceHealthPurpose, + GameCultRuntimePresenceHealthRecord, IDUNN_EXPECTED_INCARNATION_SCHEMA, + IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME, IDUNN_RUNTIME_ACTIVATION_SCHEMA, + IdunnExpectedIncarnationRecord, IdunnProcessWriteLeaseRecord, IdunnRuntimeActivationRecord, + IdunnRuntimeActivationSigner, ServiceIdentitySigner, decode_cultnet_message_from_slice, + encode_cultnet_message_to_vec, encode_frame, open_service_identity_credential_reader, +}; + +use crate::deployment::{RUNTIME_PRESENCE_IDENTITY_FD_NAME, SYSTEMD_CREDENTIALS_DIRECTORY_ENVIRONMENT}; +use crate::drivers::{CultCacheWriteLeaseDriver, authority_lock_path, lease_binds_candidate}; + +/// The route challenge is the only thing that begins with this byte: a +/// CultNet frame's 4-byte big-endian length is bounded at 1 MiB, and an HTTP +/// request starts with a method letter. +const CHALLENGE_FIRST_BYTE: u8 = 0x00; +const CHALLENGE_MAX_BYTES: usize = 1024 * 1024; +/// One whole presence exchange on the candidate endpoint, from accept to +/// answer, however the peer paces it. It is shorter than the 3 s Idunn waits +/// for a challenge answer, so one stalled peer cannot cost the next challenge +/// its reply. +const PRESENCE_DEADLINE: Duration = Duration::from_secs(1); +/// How long the presence acceptor waits after an accept error before trying +/// again. No accept error ends the acceptor. +const ACCEPT_BACKOFF: Duration = Duration::from_millis(50); +const HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(1); +/// How often the lifecycle probes the child once it is Active. +const HEALTH_REFRESH: Duration = Duration::from_millis(500); +/// A probe result older than this is not evidence of health: the lifecycle +/// that refreshes it is itself stuck. +const HEALTH_STALE: Duration = Duration::from_secs(5); +const POLL: Duration = Duration::from_millis(100); +/// The largest duration a recipe may declare for the lease wait, the startup +/// wait or the heartbeat: one day. Past it a deadline is not a deadline. +const MAX_DECLARED_SECONDS: u64 = 86_400; +const MAX_RECENT_WARMING_PROOFS: usize = 64; +const TERMINATE_GRACE: Duration = Duration::from_secs(20); +const ODIN_CULTMESH_CATALOG_CONNECTION_ID: u32 = 0x0d1d_0002; +const ODIN_CAPABILITY: &str = "odin.verse-rendezvous"; + +// ---------------------------------------------------------------- contract + +/// Everything the recipe declares about the child, parsed from the launcher's +/// argv (`--flag value` pairs; `--child-arg` repeats). The release root is the +/// sealed release directory that holds the launcher itself. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LaunchContract { + pub release_root: PathBuf, + pub child: PathBuf, + pub child_args: Vec, + /// The flag that carries the rendered config path to the child, when the + /// child takes one. + pub config_flag: Option, + pub template: PathBuf, + pub health_path: String, + pub health_contract: String, + pub state_root: Option, + pub lease_wait: Duration, + pub startup: Duration, + pub heartbeat: Duration, +} + +impl LaunchContract { + pub fn parse(release_root: PathBuf, args: impl IntoIterator) -> Result { + ensure!(release_root.is_absolute(), "the release root is not absolute"); + let mut child = None; + let mut child_args = Vec::new(); + let mut config_flag = None; + let mut template = None; + let mut health_path = None; + let mut health_contract = None; + let mut state_root = None; + let (mut lease_wait, mut startup, mut heartbeat) = (300_u64, 120_u64, 5_u64); + let mut args = args.into_iter(); + while let Some(flag) = args.next() { + let value = args.next().context("launcher argument has no value")?; + let seconds = |field: &str| -> Result { + value + .parse::() + .ok() + .filter(|seconds| (1..=MAX_DECLARED_SECONDS).contains(seconds)) + .with_context(|| { + format!("launcher argument {field} is outside 1 to {MAX_DECLARED_SECONDS} seconds") + }) + }; + match flag.as_str() { + "--child" => child = Some(release_relative(&value, "--child")?), + "--child-arg" => child_args.push(value), + "--config-flag" => config_flag = Some(value), + "--template" => template = Some(release_relative(&value, "--template")?), + "--health-path" => { + ensure!( + value.starts_with('/'), + "launcher argument --health-path is not absolute" + ); + health_path = Some(value) + } + "--health-contract" => health_contract = Some(value), + "--state-root" => state_root = Some(PathBuf::from(value)), + "--lease-wait-secs" => lease_wait = seconds("--lease-wait-secs")?, + "--startup-secs" => startup = seconds("--startup-secs")?, + "--heartbeat-secs" => heartbeat = seconds("--heartbeat-secs")?, + _ => bail!("unknown launcher argument"), + } + } + Ok(Self { + release_root, + child: child.context("launcher argument --child is required")?, + child_args, + config_flag, + template: template.context("launcher argument --template is required")?, + health_path: health_path.context("launcher argument --health-path is required")?, + health_contract: health_contract + .context("launcher argument --health-contract is required")?, + state_root, + lease_wait: Duration::from_secs(lease_wait), + startup: Duration::from_secs(startup), + heartbeat: Duration::from_secs(heartbeat), + }) + } +} + +fn release_relative(value: &str, field: &str) -> Result { + let path = PathBuf::from(value); + ensure!( + !value.is_empty() + && path + .components() + .all(|component| matches!(component, Component::Normal(_))), + "launcher argument {field} is not release-relative" + ); + Ok(path) +} + +// ---------------------------------------------------------------- authority + +/// The bundle, activation credential and provider identity Idunn hands the +/// unit, and the two signatures every presence statement needs. +pub struct RuntimeAuthority { + expected: IdunnExpectedIncarnationRecord, + expected_sha256: String, + activation: IdunnRuntimeActivationRecord, + activation_sha256: String, + activation_signer: IdunnRuntimeActivationSigner, + provider_signer: ServiceIdentitySigner, + sequence: u64, +} + +impl RuntimeAuthority { + pub fn open( + bundle: &Path, + activation_credential: impl Read, + provider_credential: impl Read, + ) -> Result { + ensure!(bundle.is_absolute(), "Idunn runtime bundle is not absolute"); + let (key, payload) = read_single_record( + &bundle.join("expected.cc"), + IdunnExpectedIncarnationRecord::TYPE, + IDUNN_EXPECTED_INCARNATION_SCHEMA, + )?; + let expected = IdunnExpectedIncarnationRecord::decode_canonical(&payload)?; + expected.validate()?; + ensure!(key == expected.target, "Expected key is substituted"); + let (key, payload) = read_single_record( + &bundle.join("activation.cc"), + IdunnRuntimeActivationRecord::TYPE, + IDUNN_RUNTIME_ACTIVATION_SCHEMA, + )?; + let activation = IdunnRuntimeActivationRecord::decode_canonical(&payload)?; + activation.validate()?; + ensure!(key == expected.target, "activation key is substituted"); + let expected_sha256 = expected.canonical_sha256()?; + ensure!( + activation.expected_projection_sha256 == expected_sha256 + && activation.runtime_id == expected.runtime_id, + "activation does not bind this Expected projection" + ); + let activation_signer = IdunnRuntimeActivationSigner::from_credential_reader( + activation_credential, + )?; + ensure!( + activation_signer.identity_id() == activation.activation_signer_identity_id + && activation_signer.public_key() == activation.activation_signer_public_key, + "activation credential does not belong to this activation" + ); + let provider_signer = + open_service_identity_credential_reader::( + provider_credential, + )?; + ensure!( + provider_signer.entry().identity_id == expected.expected_signer_identity_id, + "provider identity is not the signer selected by Expected" + ); + Ok(Self { + activation_sha256: activation.canonical_sha256()?, + expected, + expected_sha256, + activation, + activation_signer, + provider_signer, + sequence: 0, + }) + } + + pub fn expected(&self) -> &IdunnExpectedIncarnationRecord { + &self.expected + } + + pub fn activation(&self) -> &IdunnRuntimeActivationRecord { + &self.activation + } + + /// The candidate endpoint Idunn will challenge, which the launcher binds. + pub fn require_candidate_bind(&self, bind: &str) -> Result<()> { + let route = self.expected.route.as_ref().context("Expected has no route")?; + ensure!( + route.transport == "tcp" && route.candidate_endpoint == format!("tcp://{bind}"), + "candidate bind differs from the Expected route" + ); + Ok(()) + } + + /// The next presence record, signed by the provider identity and the + /// activation. The sequence is the clock: strictly increasing here and + /// above anything an earlier incarnation of this signer published. + pub fn sign( + &mut self, + state: &str, + detail: &str, + write_lease_sha256: Option, + now_millis: u64, + ) -> Result { + self.sequence = now_millis.max( + self.sequence + .checked_add(1) + .ok_or_else(|| anyhow!("runtime presence sequence exhausted"))?, + ); + let expected = &self.expected; + let mut record = GameCultRuntimePresenceHealthRecord { + schema_version: GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + target: expected.target.clone(), + expected_projection_sha256: self.expected_sha256.clone(), + plan_id: expected.plan_id.clone(), + incarnation_id: expected.incarnation_id.clone(), + sealed_release_id: expected.sealed_release_id.clone(), + activation_witness_sha256: self.activation_sha256.clone(), + state_schema_generation: expected.state_schema_generation.clone(), + state_contract_sha256: expected.state_contract_sha256.clone(), + runtime_id: expected.runtime_id.clone(), + runtime_instance_id: self.activation.runtime_instance_id.clone(), + bound_endpoint: expected + .route + .as_ref() + .map(|route| route.candidate_endpoint.clone()), + capabilities: expected + .capabilities + .iter() + .map(|capability| GameCultRuntimeCapability { + capability: capability.capability.clone(), + schema: capability.schema.clone(), + compatibility: capability.compatibility.clone(), + capacity: capability.minimum_capacity, + }) + .collect(), + health_contract: expected.health_contract.clone(), + state: state.into(), + detail: detail.into(), + write_lease_sha256, + signer_identity_id: self.provider_signer.entry().identity_id.clone(), + publisher_sequence: self.sequence, + observed_at_unix_millis: now_millis, + signature_algorithm: "ed25519".into(), + signature: Vec::new(), + activation_signer_identity_id: self.activation.activation_signer_identity_id.clone(), + activation_signature: Vec::new(), + }; + let payload = record.canonical_proof_payload()?; + record.signature = self + .provider_signer + .sign::(&payload) + .signature; + record.activation_signature = self.activation_signer.sign_presence_proof(&record)?; + record.validate()?; + Ok(record) + } +} + +fn read_single_record(path: &Path, record_type: &str, schema: &str) -> Result<(String, Vec)> { + let entries = SingleFileMessagePackBackingStore::new(path) + .pull_all_read_only_snapshot() + .context("reading a runtime bundle record")?; + let [envelope] = entries.as_slice() else { + bail!("runtime bundle record does not hold exactly one entry"); + }; + ensure!( + envelope.r#type == record_type && envelope.schema_id.as_deref() == Some(schema), + "runtime bundle record has the wrong type or schema" + ); + Ok((envelope.key.clone(), envelope.payload.clone())) +} + +/// The two authority descriptors systemd opens for the unit, in the order and +/// under the names Idunn's launch contract fixes. +pub fn inherited_authority_files() -> Result<(File, File)> { + use std::os::fd::{AsRawFd, FromRawFd}; + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let fds = std::env::var("LISTEN_FDS").context("systemd did not bind authority FDs")?; + let names = std::env::var("LISTEN_FDNAMES").context("systemd did not name authority FDs")?; + ensure!( + fds == "2" + && names + == format!( + "{IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME}:{RUNTIME_PRESENCE_IDENTITY_FD_NAME}" + ), + "systemd authority FDs differ from the launch contract" + ); + for fd in [3, 4] { + // SAFETY: F_GETFD observes one integer descriptor and transfers nothing. + ensure!( + unsafe { libc::fcntl(fd, libc::F_GETFD) } != -1, + "an authority descriptor is not open" + ); + } + // SAFETY: the LISTEN_* contract above assigns these two distinct, open + // descriptors to this process; this is their first and only consumer. + let (activation, provider) = unsafe { (File::from_raw_fd(3), File::from_raw_fd(4)) }; + for (file, exact) in [(&activation, Some(32_u64)), (&provider, None)] { + // OpenFile descriptors arrive without close-on-exec; the child must + // never inherit a signer. + // SAFETY: F_SETFD on a descriptor `file` owns. + ensure!( + unsafe { libc::fcntl(file.as_raw_fd(), libc::F_SETFD, libc::FD_CLOEXEC) } != -1, + "an authority descriptor could not be sealed" + ); + let metadata = file.metadata()?; + ensure!( + metadata.is_file() + && metadata.uid() == 0 + && metadata.permissions().mode() & 0o777 == 0o400 + && metadata.nlink() == 1 + && exact.map_or(metadata.len() > 0, |size| metadata.len() == size), + "an authority descriptor is not one root-owned 0400 file" + ); + } + Ok((activation, provider)) +} + +// ---------------------------------------------------------------- ports + +pub trait Clock: Send + Sync { + /// Wall-clock time, for values signed or published to another party. No + /// decision about freshness, deadlines or schedules reads it. + fn now_millis(&self) -> u64; + /// Monotonic time: never steps back, unaffected by wall-clock changes. Every + /// age, deadline and schedule in the launcher reads this one source. + fn monotonic_millis(&self) -> u64; + fn sleep(&self, duration: Duration); +} + +pub struct SystemClock; + +impl Clock for SystemClock { + fn now_millis(&self) -> u64 { + u64::try_from(chrono::Utc::now().timestamp_millis()).unwrap_or(0) + } + + fn monotonic_millis(&self) -> u64 { + static START: std::sync::OnceLock = std::sync::OnceLock::new(); + u64::try_from(START.get_or_init(std::time::Instant::now).elapsed().as_millis()) + .unwrap_or(u64::MAX) + } + + fn sleep(&self, duration: Duration) { + thread::sleep(duration); + } +} + +/// Systemd credentials as Idunn binds them: the files `NAME` under the unit's +/// credentials directory. +pub trait CredentialSource: Send + Sync { + fn contents(&self, name: &str) -> Result>; + fn path(&self, name: &str) -> Result; +} + +/// Resolves a credential only as a file directly under the credentials +/// directory systemd gives the unit (`$CREDENTIALS_DIRECTORY`, which Idunn +/// pins to `/run/credentials/`). The process environment is never a +/// credential namespace: a name is a plain file name, never a variable. +pub struct CredentialsDirectory(Option); + +impl CredentialsDirectory { + pub fn new(directory: PathBuf) -> Self { + Self(Some(directory)) + } + + /// The directory systemd exported; no directory means no credential is bound. + pub fn from_environment() -> Self { + Self(std::env::var_os(SYSTEMD_CREDENTIALS_DIRECTORY_ENVIRONMENT).map(PathBuf::from)) + } +} + +impl CredentialSource for CredentialsDirectory { + fn contents(&self, name: &str) -> Result> { + fs::read(self.path(name)?).map_err(|_| anyhow!("a credential could not be read")) + } + + fn path(&self, name: &str) -> Result { + ensure!(valid_credential_name(name), "credential name is invalid"); + let path = self + .0 + .as_ref() + .ok_or_else(|| anyhow!("a credential is not bound"))? + .join(name); + ensure!(fs::symlink_metadata(&path).is_ok(), "a credential is not bound"); + Ok(path) + } +} + +/// Where signed presence goes (Odin). +pub trait PresenceSink: Send + Sync { + fn put(&self, record: &GameCultRuntimePresenceHealthRecord) -> Result<()>; +} + +/// The lease as the launcher sees it: the current record, and a shared lock on +/// its store that Idunn's revocation cannot take while the child lives. +pub trait LeaseReader: Send + Sync { + fn read(&self) -> Result>; + fn hold_shared(&self) -> Result>; +} + +/// The shared lock on the lease store, held until dropped. Its descriptor is +/// handed to the child (`ChildSpawner::spawn`), so the lock lives as long as +/// the launcher or any process that inherited it. +pub trait LeaseLock: Send { + fn descriptor(&self) -> Option; +} + +pub trait HealthProbe: Send + Sync { + fn healthy(&self) -> bool; +} + +pub trait ChildHandle: Send { + /// The exit status once the child has exited. + fn exited(&mut self) -> Option; + /// SIGTERM, then SIGKILL after the grace period; returns once it is gone. + fn terminate(&mut self, grace: Duration); +} + +pub trait ChildSpawner: Send { + /// `inherit` is a descriptor the child keeps open across exec (the lease + /// lock); every other descriptor of the launcher is close-on-exec. + fn spawn( + &self, + program: &Path, + args: &[String], + inherit: Option, + ) -> Result>; +} + +// ---------------------------------------------------------------- production ports + +pub struct DriverLeaseReader { + driver: CultCacheWriteLeaseDriver, +} + +impl DriverLeaseReader { + pub fn new(target: &str, record_path: &Path) -> Self { + Self { + driver: CultCacheWriteLeaseDriver::new(target, record_path), + } + } +} + +struct SharedLockFile(File); + +impl LeaseLock for SharedLockFile { + fn descriptor(&self) -> Option { + Some(self.0.as_raw_fd()) + } +} + +impl LeaseReader for DriverLeaseReader { + fn read(&self) -> Result> { + self.driver + .current_for_holder() + .map(|current| current.map(|(_, lease)| lease)) + .map_err(|_| anyhow!("write-lease store refused custody, envelope or target")) + } + + /// LOCK_SH on `.lock`, the lock Idunn's revocation respects. The + /// descriptor is a plain file, so flock's lock belongs to its open file + /// description: a child that inherits it keeps the lock after the launcher + /// dies. + fn hold_shared(&self) -> Result> { + let file = File::open(authority_lock_path(&self.driver.record_path)).map_err(|_| anyhow!("write-lease lock could not be held"))?; + // SAFETY: flock on a descriptor this function owns. + ensure!( + unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_SH) } == 0, + "write-lease lock could not be held" + ); + Ok(Box::new(SharedLockFile(file))) + } +} + +/// Odin takes presence over RUDP at the endpoint Expected's Odin dependency +/// names. +pub struct OdinRudpSink { + endpoint: SocketAddr, +} + +impl OdinRudpSink { + pub fn from_expected(expected: &IdunnExpectedIncarnationRecord) -> Result { + let dependency = expected + .dependencies + .iter() + .find(|dependency| dependency.capability == ODIN_CAPABILITY) + .context("Expected has no Odin dependency")?; + let endpoint = dependency + .provider_endpoint + .as_deref() + .context("Expected Odin dependency has no endpoint")?; + let endpoint = endpoint.strip_prefix("rudp://").unwrap_or(endpoint); + Ok(Self { + endpoint: endpoint + .parse() + .map_err(|_| anyhow!("Expected Odin endpoint is not a socket address"))?, + }) + } +} + +impl PresenceSink for OdinRudpSink { + fn put(&self, record: &GameCultRuntimePresenceHealthRecord) -> Result<()> { + let message = CultNetMessage::DocumentPutRaw { + message_id: format!( + "runtime-presence:{}:{}:{}", + record.target, record.runtime_instance_id, record.publisher_sequence + ), + document: presence_document( + record, + Some(vec![ + "cultnet.transport.rudp.v0".into(), + "runtime-presence".into(), + ]), + )?, + }; + let bind = if self.endpoint.is_ipv4() { + "0.0.0.0:0" + } else { + "[::]:0" + }; + let socket = UdpSocket::bind(bind)?; + socket.set_read_timeout(Some(Duration::from_millis(100)))?; + let mut transport = CultNetRudpSocketTransportConnection::new( + CultNetRudpSocketTransportOptions::client( + &record.runtime_id, + socket, + self.endpoint, + ODIN_CULTMESH_CATALOG_CONNECTION_ID, + ), + )?; + transport.connect(Vec::new())?; + let deadline = Instant::now() + Duration::from_millis(500); + while !transport.connected() { + let _ = transport.receive_once()?; + transport.poll_resends()?; + ensure!(Instant::now() < deadline, "Odin did not accept the connection"); + } + let receipt = transport.send_reliable( + "schema", + encode_cultnet_message_to_vec(&message, CultNetWireContract::CultNetSchemaV0)?, + )?; + let deadline = Instant::now() + Duration::from_secs(2); + loop { + match transport.reliable_send_status(&receipt) { + CultNetRudpReliableSendStatus::Acknowledged => return Ok(()), + CultNetRudpReliableSendStatus::Invalidated => { + bail!("Odin invalidated the presence acknowledgement") + } + CultNetRudpReliableSendStatus::Pending => {} + } + let _ = transport.receive_once()?; + transport.poll_resends()?; + ensure!(Instant::now() < deadline, "Odin did not acknowledge presence"); + } + } +} + +fn presence_document( + record: &GameCultRuntimePresenceHealthRecord, + tags: Option>, +) -> Result { + let payload = rmp_serde::to_vec(record).context("encoding runtime presence")?; + Ok(CultNetRawDocumentRecord { + schema_id: GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + record_key: record.target.clone(), + stored_at: chrono::DateTime::from_timestamp_millis( + record.observed_at_unix_millis.try_into()?, + ) + .context("runtime presence timestamp is invalid")? + .to_rfc3339(), + payload_encoding: CultNetRawPayloadEncoding::Messagepack, + payload, + source_runtime_id: Some(record.runtime_id.clone()), + source_agent_id: Some(record.signer_identity_id.clone()), + source_role: Some("runtime-presence-publisher".into()), + tags, + }) +} + +/// `GET ` over TCP at the child's member bind; healthy on a 200. One +/// deadline covers connect, write and read, so a full backlog or a silent child +/// answers false within `HEALTH_PROBE_TIMEOUT`. +pub struct HttpHealthProbe { + pub member: SocketAddr, + pub path: String, +} + +impl HealthProbe for HttpHealthProbe { + fn healthy(&self) -> bool { + let deadline = Instant::now() + HEALTH_PROBE_TIMEOUT; + let Ok(mut stream) = TcpStream::connect_timeout(&self.member, HEALTH_PROBE_TIMEOUT) else { + return false; + }; + let Some(remaining) = deadline + .checked_duration_since(Instant::now()) + .filter(|left| !left.is_zero()) + else { + return false; + }; + let request = format!( + "GET {} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + self.path + ); + let mut head = [0_u8; 12]; + stream.set_write_timeout(Some(remaining)).is_ok() + && stream.write_all(request.as_bytes()).is_ok() + && read_by(&mut stream, &mut head, deadline).is_ok() + && head.starts_with(b"HTTP/1.") + && &head[8..12] == b" 200" + } +} + +pub struct ProcessSpawner; + +struct ProcessChild(Child); + +impl ChildSpawner for ProcessSpawner { + fn spawn( + &self, + program: &Path, + args: &[String], + inherit: Option, + ) -> Result> { + let mut command = Command::new(program); + // The child gets a bare environment: no Idunn variable, no candidate + // bind, no credential path leaks into it. + command.args(args).env_clear().stdin(Stdio::null()); + for name in ["PATH", "TMPDIR", "LANG", "LC_ALL"] { + if let Some(value) = std::env::var_os(name) { + command.env(name, value); + } + } + if let Some(descriptor) = inherit { + // SAFETY: runs in the forked child before exec; fcntl is + // async-signal-safe and the parent's descriptor stays close-on-exec. + unsafe { + command.pre_exec(move || { + if libc::fcntl(descriptor, libc::F_SETFD, 0) == -1 { + return Err(io::Error::last_os_error()); + } + Ok(()) + }); + } + } + let child = command.spawn().map_err(|_| anyhow!("the child could not be spawned"))?; + Ok(Box::new(ProcessChild(child))) + } +} + +impl ChildHandle for ProcessChild { + fn exited(&mut self) -> Option { + match self.0.try_wait() { + Ok(Some(status)) => Some(status.code().unwrap_or(1)), + Ok(None) => None, + Err(_) => Some(1), + } + } + + fn terminate(&mut self, grace: Duration) { + if self.exited().is_some() { + return; + } + // SAFETY: the pid is our own live child. + unsafe { libc::kill(self.0.id() as libc::pid_t, libc::SIGTERM) }; + let deadline = Instant::now() + grace; + while Instant::now() < deadline { + if self.exited().is_some() { + return; + } + thread::sleep(Duration::from_millis(20)); + } + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +// ---------------------------------------------------------------- config + +/// The closed placeholder set (decision D6). Any other `{{...}}` refuses. +pub struct ConfigRenderer<'a> { + pub credentials: &'a dyn CredentialSource, + /// The member bind the child serves on. + pub listen: SocketAddr, + pub state_root: Option<&'a Path>, +} + +impl ConfigRenderer<'_> { + pub fn render(&self, template: &str) -> Result { + let mut rendered = String::with_capacity(template.len()); + let mut rest = template; + while let Some(open) = rest.find("{{") { + rendered.push_str(&rest[..open]); + let after = &rest[open + 2..]; + let close = after + .find("}}") + .context("config template has an unterminated placeholder")?; + rendered.push_str(&self.expand(&after[..close])?); + rest = &after[close + 2..]; + } + rendered.push_str(rest); + Ok(rendered) + } + + fn expand(&self, key: &str) -> Result { + match key { + "child.listen" => return Ok(self.listen.to_string()), + "child.listen_host" => return Ok(self.listen.ip().to_string()), + "child.listen_port" => return Ok(self.listen.port().to_string()), + _ => {} + } + if key == "state_root" { + return Ok(self + .state_root + .context("config template needs a state root the launch did not give")? + .display() + .to_string()); + } + if let Some(name) = key.strip_prefix("credential:") { + ensure!(valid_credential_name(name), "credential placeholder name is invalid"); + let bytes = self.credentials.contents(name)?; + let text = String::from_utf8(bytes) + .map_err(|_| anyhow!("a credential is not UTF-8 text"))?; + return Ok(text.strip_suffix('\n').unwrap_or(&text).to_owned()); + } + if let Some(name) = key.strip_prefix("credential_path:") { + ensure!(valid_credential_name(name), "credential placeholder name is invalid"); + return Ok(self.credentials.path(name)?.display().to_string()); + } + bail!("config template uses a placeholder outside the closed set") + } +} + +fn valid_credential_name(name: &str) -> bool { + !name.is_empty() + && name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_' || byte == b'-') +} + +fn write_private_file(directory: &Path, name: &str, contents: &str) -> Result { + let path = directory.join(name); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&path) + .map_err(|_| anyhow!("the rendered config could not be created"))?; + file.write_all(contents.as_bytes())?; + Ok(path) +} + +// ---------------------------------------------------------------- lease + +/// A write lease this launcher holds: the record binds this Expected, +/// activation and one of the launcher's own Warming proofs, and its store is +/// share-locked until the hold drops. +pub struct LeaseHold { + reader: Arc, + lease: IdunnProcessWriteLeaseRecord, + sha256: String, + lock: Box, +} + +impl LeaseHold { + pub fn acquire( + reader: Arc, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + warming_proofs: &[String], + ) -> Result { + let lease = reader.read()?.context("no write lease is granted")?; + lease.validate()?; + lease_binds_candidate(expected, activation, &lease, warming_proofs)?; + let lock = reader.hold_shared()?; + ensure!( + reader.read()?.as_ref() == Some(&lease), + "write lease changed while it was being held" + ); + Ok(Self { + sha256: lease.canonical_sha256()?, + reader, + lease, + lock, + }) + } + + pub fn sha256(&self) -> &str { + &self.sha256 + } + + /// The locked descriptor, for the child to inherit. + pub fn lock_descriptor(&self) -> Option { + self.lock.descriptor() + } + + pub fn still_current(&self) -> bool { + matches!(self.reader.read(), Ok(Some(current)) if current == self.lease) + } +} + +// ---------------------------------------------------------------- route + +#[derive(Clone, Debug, PartialEq, Eq)] +enum Phase { + Warming, + Active(Option), + Failed, +} + +/// What the launcher currently is, shared with the presence acceptor. +struct Shared { + authority: Mutex, + phase: Mutex, + lease: Mutex>, + lease_required: bool, + health: Arc, + /// The last probe result and the monotonic time it started. Only the lifecycle + /// probes; a challenge reads this and never waits on the child. + probed: Mutex>, + clock: Arc, +} + +impl Shared { + /// Probes the child now and keeps the result for presence to answer from. + fn refresh_health(&self) -> bool { + let started = self.clock.monotonic_millis(); + let healthy = self.health.healthy(); + if let Ok(mut probed) = self.probed.lock() { + *probed = Some((healthy, started)); + } + healthy + } + + /// Active only while the last probe found the child healthy and is recent. + /// A child never probed, wedged, or probed too long ago reads degraded. A + /// dead presence acceptor does not degrade presence: it fails the unit. + fn serving_state(&self) -> &'static str { + let stale = u64::try_from(HEALTH_STALE.as_millis()).unwrap_or(u64::MAX); + let now = self.clock.monotonic_millis(); + // A probe can finish between this thread reading the clock and locking + // the result, so a stamp later than `now` is a result fresher than the + // reading, age zero: monotonic time does not run backwards, so the stamp + // is not wrong. + let healthy = self + .probed + .lock() + .ok() + .and_then(|probed| *probed) + .is_some_and(|(healthy, at)| healthy && now.saturating_sub(at) <= stale); + if healthy { "active" } else { "degraded" } + } + + fn sign(&self, state: &str, detail: &str, lease: Option) -> Result { + self.authority + .lock() + .map_err(|_| anyhow!("authority lock poisoned"))? + .sign(state, detail, lease, self.clock.now_millis()) + } + + fn lease_current(&self) -> bool { + !self.lease_required + || self + .lease + .lock() + .is_ok_and(|hold| hold.as_ref().is_some_and(LeaseHold::still_current)) + } + + /// The answer to a route observation: active only if the child is healthy + /// and the lease is still ours. + fn observe(&self, request: CultNetMessage) -> Result { + let CultNetMessage::SnapshotRequest { message_id, .. } = request else { + bail!("route challenge is not a snapshot request"); + }; + let detail = format!("route-observation:{message_id}"); + let phase = self.phase.lock().map_err(|_| anyhow!("phase lock poisoned"))?.clone(); + let record = match phase { + Phase::Warming => self.sign("warming", &detail, None)?, + Phase::Failed => self.sign("failed", &detail, None)?, + Phase::Active(lease) => { + let state = if self.lease_current() { + self.serving_state() + } else { + "degraded" + }; + self.sign(state, &detail, lease)? + } + }; + Ok(CultNetMessage::SnapshotResponseRaw { + message_id, + documents: vec![presence_document(&record, None)?], + }) + } +} + +type Responder = Arc Result + Send + Sync>; + +/// Owns the candidate endpoint and answers presence there and nowhere else. +/// Members never reach it: Idunn routes them to the child's member bind. +/// +/// The endpoint is public, so it spends bounded resources on a peer: one +/// connection at a time, so a connection holds at most one descriptor; a total +/// deadline for the whole exchange; and any first byte but a challenge's closes +/// the connection unanswered. No accept error ends it: it backs off and accepts +/// again, and only `halt` stops it. +pub struct PresenceAcceptor { + listener: TcpListener, + responder: Responder, +} + +impl PresenceAcceptor { + pub fn new(listener: TcpListener, responder: Responder) -> Self { + Self { listener, responder } + } + + pub fn serve(self, halt: Arc) { + self.serve_accepting(halt, |listener| listener.accept()); + } + + fn serve_accepting( + self, + halt: Arc, + mut accept: impl FnMut(&TcpListener) -> io::Result<(TcpStream, SocketAddr)>, + ) { + // A listener that cannot be made non-blocking could never observe + // `halt`; ending here is a dead acceptor, which fails the unit. + if self.listener.set_nonblocking(true).is_err() { + return; + } + while !halt.load(Ordering::SeqCst) { + match accept(&self.listener) { + Ok((connection, _)) => { + let _ = connection.set_nonblocking(false); + let _ = self.answer(connection); + } + // An idle non-blocking accept is an error too; every one waits + // and tries again. + Err(_) => thread::sleep(ACCEPT_BACKOFF), + } + } + } + + fn answer(&self, mut connection: TcpStream) -> Result<()> { + let deadline = Instant::now() + PRESENCE_DEADLINE; + let mut first = [0_u8; 1]; + read_by(&mut connection, &mut first, deadline)?; + ensure!(first[0] == CHALLENGE_FIRST_BYTE, "not a challenge frame"); + let mut rest = [0_u8; 3]; + read_by(&mut connection, &mut rest, deadline)?; + let length = u32::from_be_bytes([CHALLENGE_FIRST_BYTE, rest[0], rest[1], rest[2]]) as usize; + ensure!( + (1..=CHALLENGE_MAX_BYTES).contains(&length), + "challenge frame is outside its bound" + ); + let mut payload = vec![0_u8; length]; + read_by(&mut connection, &mut payload, deadline)?; + let request = + decode_cultnet_message_from_slice(&payload, CultNetWireContract::CultNetSchemaV0)?; + let response = (self.responder)(request)?; + let frame = encode_frame(&encode_cultnet_message_to_vec( + &response, + CultNetWireContract::CultNetSchemaV0, + )?)?; + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|left| !left.is_zero()) + .context("the answer was not ready in time")?; + connection.set_write_timeout(Some(remaining))?; + connection.write_all(&frame)?; + connection.flush()?; + Ok(()) + } +} + +/// Fills `buffer` or fails when `deadline` passes, however the peer paces it. +fn read_by(connection: &mut TcpStream, buffer: &mut [u8], deadline: Instant) -> Result<()> { + let mut filled = 0; + while filled < buffer.len() { + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|left| !left.is_zero()) + .context("the peer did not send in time")?; + connection.set_read_timeout(Some(remaining))?; + match connection.read(&mut buffer[filled..]) { + Ok(0) => bail!("the peer closed early"), + Ok(count) => filled += count, + Err(error) if error.kind() == io::ErrorKind::Interrupted => {} + Err(_) => bail!("the peer did not send in time"), + } + } + Ok(()) +} + +// ---------------------------------------------------------------- lifecycle + +pub struct Ports { + pub credentials: Arc, + pub clock: Arc, + pub sink: Arc, + pub lease: Arc, + pub spawner: Box, + /// Built from the member bind the child serves on. + pub health: Box Arc + Send>, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Outcome { + /// SIGTERM: the child was stopped and waited for. + Stopped, + /// The unit must die so continuity sees it; the code names why. + Failed(&'static str), +} + +impl Outcome { + pub fn exit_code(&self) -> i32 { + match self { + Self::Stopped => 0, + Self::Failed(_) => 1, + } + } +} + +/// The member bind Idunn hands the unit: a socket address on a loopback IP, +/// other than the candidate bind. Errors name the variable, never its value. +pub fn parse_member_bind(raw: &str, candidate_bind: &str) -> Result { + let member: SocketAddr = raw + .parse() + .map_err(|_| anyhow!("the Idunn member bind is not a socket address"))?; + ensure!( + member.ip().is_loopback(), + "the Idunn member bind is not on a loopback address" + ); + let candidate: SocketAddr = candidate_bind + .parse() + .map_err(|_| anyhow!("the Idunn candidate bind is not a socket address"))?; + ensure!(member != candidate, "the Idunn member bind equals the candidate bind"); + Ok(member) +} + +/// The monotonic reading `span` after `now`; a span too long to represent is a +/// deadline that never arrives. +fn after(now: u64, span: Duration) -> u64 { + now.saturating_add(u64::try_from(span.as_millis()).unwrap_or(u64::MAX)) +} + +/// The unit's whole life: Warming, lease, child, Active, until the child or the +/// lease ends or SIGTERM arrives. `route` is the already-bound candidate +/// listener, answered for presence only; `member` is the address the child +/// serves members on; `tmp` is the unit's private temporary directory. +pub fn run( + contract: &LaunchContract, + authority: RuntimeAuthority, + route: TcpListener, + member: SocketAddr, + tmp: &Path, + ports: Ports, + stop: &AtomicBool, +) -> Outcome { + let expected = authority.expected().clone(); + let activation = authority.activation().clone(); + if expected.health_contract != contract.health_contract { + return Outcome::Failed("health-contract-mismatch"); + } + let health = (ports.health)(member); + let shared = Arc::new(Shared { + authority: Mutex::new(authority), + phase: Mutex::new(Phase::Warming), + lease: Mutex::new(None), + lease_required: expected.write_lease_required, + health: Arc::clone(&health), + probed: Mutex::new(None), + clock: Arc::clone(&ports.clock), + }); + let halt = Arc::new(AtomicBool::new(false)); + let responder: Responder = { + let shared = Arc::clone(&shared); + Arc::new(move |request| shared.observe(request)) + }; + let acceptor = PresenceAcceptor::new(route, responder); + let serving = { + let halt = Arc::clone(&halt); + thread::Builder::new() + .name("presence-acceptor".into()) + .spawn(move || acceptor.serve(halt)) + }; + let Ok(serving) = serving else { + return Outcome::Failed("presence-acceptor"); + }; + let outcome = lifecycle(contract, &expected, &activation, member, tmp, &ports, &shared, &serving, stop); + halt.store(true, Ordering::SeqCst); + let _ = serving.join(); + outcome +} + +#[allow(clippy::too_many_arguments)] +fn lifecycle( + contract: &LaunchContract, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + member: SocketAddr, + tmp: &Path, + ports: &Ports, + shared: &Shared, + presence: &thread::JoinHandle<()>, + stop: &AtomicBool, +) -> Outcome { + let clock = &*ports.clock; + let publish = |state: &str, detail: &str, lease: Option| -> Option { + let record = shared.sign(state, detail, lease).ok()?; + let sha = record.canonical_sha256().ok()?; + // A refused or lost publish is retried by the heartbeat; Odin may + // refuse Warming for a while after Idunn starts the unit. + let _ = ports.sink.put(&record); + Some(sha) + }; + let set_phase = |phase: Phase| { + if let Ok(mut current) = shared.phase.lock() { + *current = phase; + } + }; + let fail = |child: Option<&mut Box>, code: &'static str| { + if let Some(child) = child { + child.terminate(TERMINATE_GRACE); + } + set_phase(Phase::Failed); + publish("failed", code, None); + Outcome::Failed(code) + }; + + // 1. Warming reaches Odin before any child exists. + let mut proofs: VecDeque = VecDeque::new(); + let remember = |sha: Option, proofs: &mut VecDeque| { + if let Some(sha) = sha { + proofs.push_back(sha); + while proofs.len() > MAX_RECENT_WARMING_PROOFS { + proofs.pop_front(); + } + } + }; + remember(publish("warming", "launcher-started", None), &mut proofs); + let heartbeat = contract.heartbeat; + let mut next_heartbeat = after(clock.monotonic_millis(), heartbeat); + + // 2. A stateful child waits for the lease; a stateless one does not. + let mut lease_sha = None; + let mut lease_fd = None; + if expected.write_lease_required { + let deadline = after(clock.monotonic_millis(), contract.lease_wait); + loop { + if stop.load(Ordering::SeqCst) { + return Outcome::Stopped; + } + if presence.is_finished() { + return fail(None, "presence-acceptor"); + } + let held = LeaseHold::acquire( + Arc::clone(&ports.lease), + expected, + activation, + proofs.make_contiguous(), + ); + if let Ok(hold) = held { + lease_sha = Some(hold.sha256().to_owned()); + lease_fd = hold.lock_descriptor(); + if let Ok(mut slot) = shared.lease.lock() { + *slot = Some(hold); + } + break; + } + let now = clock.monotonic_millis(); + if now >= deadline { + return fail(None, "lease-timeout"); + } + if now >= next_heartbeat { + remember(publish("warming", "write-lease-pending", None), &mut proofs); + next_heartbeat = after(now, heartbeat); + } + clock.sleep(POLL); + } + } + + // 3. Render, spawn, and wait for health. + let Some(mut child) = start_child(contract, member, tmp, ports, lease_fd) else { + return fail(None, "child-start-refused"); + }; + let startup_deadline = after(clock.monotonic_millis(), contract.startup); + loop { + if stop.load(Ordering::SeqCst) { + child.terminate(TERMINATE_GRACE); + return Outcome::Stopped; + } + if child.exited().is_some() { + return fail(None, "child-exited"); + } + if presence.is_finished() { + return fail(Some(&mut child), "presence-acceptor"); + } + if shared.refresh_health() { + break; + } + if clock.monotonic_millis() >= startup_deadline { + return fail(Some(&mut child), "startup-timeout"); + } + clock.sleep(POLL); + } + + // 4. Active, and heartbeats while it holds. + set_phase(Phase::Active(lease_sha.clone())); + publish(shared.serving_state(), "child-healthy", lease_sha.clone()); + let mut next_probe = after(clock.monotonic_millis(), HEALTH_REFRESH); + next_heartbeat = after(clock.monotonic_millis(), heartbeat); + loop { + if stop.load(Ordering::SeqCst) { + child.terminate(TERMINATE_GRACE); + return Outcome::Stopped; + } + if child.exited().is_some() { + return fail(None, "child-exited"); + } + if presence.is_finished() { + return fail(Some(&mut child), "presence-acceptor"); + } + if !shared.lease_current() { + return fail(Some(&mut child), "lease-lost"); + } + let now = clock.monotonic_millis(); + if now >= next_probe { + shared.refresh_health(); + next_probe = after(now, HEALTH_REFRESH); + } + if now >= next_heartbeat { + publish(shared.serving_state(), "heartbeat", lease_sha.clone()); + next_heartbeat = after(now, heartbeat); + } + clock.sleep(POLL); + } +} + +fn start_child( + contract: &LaunchContract, + member: SocketAddr, + tmp: &Path, + ports: &Ports, + lease_fd: Option, +) -> Option> { + let template = fs::read_to_string(contract.release_root.join(&contract.template)).ok()?; + let rendered = ConfigRenderer { + credentials: &*ports.credentials, + listen: member, + state_root: contract.state_root.as_deref(), + } + .render(&template) + .ok()?; + let config = write_private_file(tmp, "child-config", &rendered).ok()?; + let mut args = contract.child_args.clone(); + if let Some(flag) = &contract.config_flag { + args.push(flag.clone()); + args.push(config.display().to_string()); + } + ports + .spawner + .spawn(&contract.release_root.join(&contract.child), &args, lease_fd) + .ok() +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::os::unix::fs::PermissionsExt; + use std::sync::atomic::{AtomicI64, AtomicU64, AtomicUsize}; + + use cultcache_rs::{CultCacheEnvelope, TryCompareExchangeSnapshotOutcome}; + use cultnet_rs::{ + IdunnExpectedCapability, IdunnExpectedRoute, IdunnRuntimeActivationLaunch, + IdunnServiceIdentity, RuntimePresenceAuthenticationContext, VerifiedRuntimeAuthority, + authenticate_runtime_presence_claim, enroll_service_identity_at, verify_runtime_authority, + }; + + use super::*; + use crate::drivers::{WriteLeasePort, exact_route_snapshot_response, request_tcp_snapshot}; + + /// The member bind Idunn would hand these fixtures' unit. + fn member() -> SocketAddr { + "127.0.0.1:18845".parse().unwrap() + } + + fn digest(byte: char) -> String { + format!("sha256-{}", byte.to_string().repeat(64)) + } + + struct Fixture { + dir: tempfile::TempDir, + expected: IdunnExpectedIncarnationRecord, + activation: IdunnRuntimeActivationRecord, + activation_credential: Vec, + provider_credential: Vec, + verified: VerifiedRuntimeAuthority, + } + + fn write_record(path: &Path, key: &str, kind: &str, schema: &str, payload: Vec) { + let envelope = CultCacheEnvelope { + key: key.into(), + r#type: kind.into(), + payload, + stored_at: chrono::Utc::now().to_rfc3339(), + schema_id: Some(schema.into()), + }; + let outcome = SingleFileMessagePackBackingStore::new(path) + .try_compare_exchange_snapshot(&[], &[envelope]) + .unwrap(); + assert!(matches!( + outcome, + TryCompareExchangeSnapshotOutcome::Exchanged + )); + } + + fn fixture(lease_required: bool) -> Fixture { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + let provider_path = root.join("provider.cc"); + let provider = + enroll_service_identity_at::(&provider_path).unwrap(); + let idunn = + enroll_service_identity_at::(&root.join("idunn.cc")).unwrap(); + let expected = IdunnExpectedIncarnationRecord { + schema_version: IDUNN_EXPECTED_INCARNATION_SCHEMA.into(), + target: "synapse".into(), + plan_id: digest('1'), + incarnation_id: "incarnation-1".into(), + sealed_release_id: digest('2'), + source_repository: "github.com/GameCult/gamecult-chat".into(), + source_revision: "3".repeat(40), + recipe_sha256: digest('4'), + runtime_id: "synapse-yggdrasil".into(), + expected_signer_identity_id: provider.entry().identity_id.clone(), + health_contract: "gamecult.chat.synapse.presence-launcher.v1".into(), + artifact_sha256: digest('5'), + state_schema_generation: lease_required.then(|| "state-v1".into()), + state_contract_sha256: lease_required.then(|| digest('6')), + write_lease_required: lease_required, + route: Some(IdunnExpectedRoute { + route_id: "synapse-tcp".into(), + transport: "tcp".into(), + stable_endpoint: "tcp://127.0.0.1:18834".into(), + candidate_endpoint: "tcp://127.0.0.1:18835".into(), + }), + capabilities: vec![IdunnExpectedCapability { + capability: "gamecult.chat.homeserver".into(), + schema: "matrix.client.v3".into(), + compatibility: "v3".into(), + minimum_capacity: 1, + }], + dependencies: Vec::new(), + }; + let launch = + IdunnRuntimeActivationLaunch::issue(&expected, digest('7'), 100, &idunn).unwrap(); + let activation = launch.activation().clone(); + let mut activation_credential = Vec::new(); + launch.write_credential(&mut activation_credential).unwrap(); + let verified = verify_runtime_authority( + &expected, + &activation, + &idunn.trust_anchor().unwrap(), + &provider.entry().public_key, + ) + .unwrap(); + let bundle = root.join("bundle"); + fs::create_dir(&bundle).unwrap(); + write_record( + &bundle.join("expected.cc"), + &expected.target, + IdunnExpectedIncarnationRecord::TYPE, + IDUNN_EXPECTED_INCARNATION_SCHEMA, + expected.canonical_bytes().unwrap(), + ); + write_record( + &bundle.join("activation.cc"), + &expected.target, + IdunnRuntimeActivationRecord::TYPE, + IDUNN_RUNTIME_ACTIVATION_SCHEMA, + activation.canonical_bytes().unwrap(), + ); + let provider_credential = fs::read(&provider_path).unwrap(); + for directory in ["release/bin", "tmp"] { + fs::create_dir_all(root.join(directory)).unwrap(); + } + Fixture { + dir, + expected, + activation, + activation_credential, + provider_credential, + verified, + } + } + + impl Fixture { + fn authority(&self) -> RuntimeAuthority { + RuntimeAuthority::open( + &self.dir.path().join("bundle"), + &self.activation_credential[..], + &self.provider_credential[..], + ) + .unwrap() + } + + fn lease(&self, warming_sha256: &str) -> IdunnProcessWriteLeaseRecord { + IdunnProcessWriteLeaseRecord { + schema_version: cultnet_rs::IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into(), + target: self.expected.target.clone(), + expected_projection_sha256: self.expected.canonical_sha256().unwrap(), + plan_id: self.expected.plan_id.clone(), + incarnation_id: self.expected.incarnation_id.clone(), + sealed_release_id: self.expected.sealed_release_id.clone(), + activation_witness_sha256: self.activation.canonical_sha256().unwrap(), + state_schema_generation: self.expected.state_schema_generation.clone().unwrap(), + state_contract_sha256: self.expected.state_contract_sha256.clone().unwrap(), + runtime_id: self.expected.runtime_id.clone(), + runtime_instance_id: self.activation.runtime_instance_id.clone(), + warming_presence_sha256: warming_sha256.into(), + lease_epoch: 1, + issued_at_unix_millis: 200, + } + } + + fn contract(&self, template: &str) -> LaunchContract { + fs::write(self.dir.path().join("release/config.tmpl"), template).unwrap(); + LaunchContract { + release_root: self.dir.path().join("release"), + child: PathBuf::from("bin/child"), + child_args: vec!["--serve".into()], + config_flag: Some("--config".into()), + template: PathBuf::from("config.tmpl"), + health_path: "/health".into(), + health_contract: self.expected.health_contract.clone(), + state_root: Some(PathBuf::from("/var/lib/synapse")), + lease_wait: Duration::from_secs(3600), + startup: Duration::from_secs(3600), + heartbeat: Duration::from_secs(5), + } + } + } + + const WALL_EPOCH_MILLIS: i64 = 1_760_000_000_000; + + /// Virtual time runs a hundred times faster than the wall clock. + struct FastClock { + monotonic: AtomicU64, + wall_step: AtomicI64, + } + + impl FastClock { + /// The wall clock starts a Unix-epoch-sized distance from the monotonic + /// one, as the real pair does, so a deadline or schedule that reads the + /// wrong source is off by that distance and no test can pass by it. + fn new(start: u64) -> Self { + Self { monotonic: AtomicU64::new(start), wall_step: AtomicI64::new(WALL_EPOCH_MILLIS) } + } + + /// Steps the wall clock alone, as timesyncd or a VM restore would. + fn step_wall(&self, millis: i64) { + self.wall_step.fetch_add(millis, Ordering::SeqCst); + } + } + + impl Clock for FastClock { + fn now_millis(&self) -> u64 { + let wall = self.monotonic.load(Ordering::SeqCst) as i64 + self.wall_step.load(Ordering::SeqCst); + u64::try_from(wall).unwrap_or(0) + } + + fn monotonic_millis(&self) -> u64 { + self.monotonic.load(Ordering::SeqCst) + } + + fn sleep(&self, duration: Duration) { + self.monotonic + .fetch_add(duration.as_millis() as u64, Ordering::SeqCst); + thread::sleep(Duration::from_millis(1)); + } + } + + #[derive(Default)] + struct Sink(Mutex>); + + impl PresenceSink for Sink { + fn put(&self, record: &GameCultRuntimePresenceHealthRecord) -> Result<()> { + self.0.lock().unwrap().push(record.clone()); + Ok(()) + } + } + + impl Sink { + fn states(&self) -> Vec { + self.0 + .lock() + .unwrap() + .iter() + .map(|record| record.state.clone()) + .collect() + } + + fn first(&self, state: &str) -> Option { + self.0 + .lock() + .unwrap() + .iter() + .find(|record| record.state == state) + .cloned() + } + } + + #[derive(Default)] + struct FakeLease(Mutex>); + + impl LeaseReader for FakeLease { + fn read(&self) -> Result> { + Ok(self.0.lock().unwrap().clone()) + } + + fn hold_shared(&self) -> Result> { + Ok(Box::new(FakeLock)) + } + } + + /// The descriptor number a fake lock claims; the launcher must hand it on. + const FAKE_LOCK_FD: RawFd = 1234; + + struct FakeLock; + + impl LeaseLock for FakeLock { + fn descriptor(&self) -> Option { + Some(FAKE_LOCK_FD) + } + } + + struct Health(AtomicBool); + + impl HealthProbe for Health { + fn healthy(&self) -> bool { + self.0.load(Ordering::SeqCst) + } + } + + #[derive(Default)] + struct Child { + spawns: AtomicUsize, + exit: Mutex>, + terminated: AtomicBool, + args: Mutex>>, + inherited: Mutex>>, + } + + struct FakeChild(Arc); + + impl ChildHandle for FakeChild { + fn exited(&mut self) -> Option { + *self.0.exit.lock().unwrap() + } + + fn terminate(&mut self, _grace: Duration) { + self.0.terminated.store(true, Ordering::SeqCst); + *self.0.exit.lock().unwrap() = Some(143); + } + } + + struct FakeSpawner(Arc); + + impl ChildSpawner for FakeSpawner { + fn spawn( + &self, + _program: &Path, + args: &[String], + inherit: Option, + ) -> Result> { + self.0.inherited.lock().unwrap().push(inherit); + self.0.spawns.fetch_add(1, Ordering::SeqCst); + self.0.args.lock().unwrap().push(args.to_vec()); + Ok(Box::new(FakeChild(Arc::clone(&self.0)))) + } + } + + struct Credentials(BTreeMap>); + + impl CredentialSource for Credentials { + fn contents(&self, name: &str) -> Result> { + self.0.get(name).cloned().ok_or_else(|| anyhow!("unbound")) + } + + fn path(&self, name: &str) -> Result { + self.0 + .contains_key(name) + .then(|| PathBuf::from(format!("/run/credentials/unit/{name}"))) + .ok_or_else(|| anyhow!("unbound")) + } + } + + struct Run { + sink: Arc, + lease: Arc, + child: Arc, + stop: Arc, + outcome: Option>, + } + + fn start(fx: &Fixture, contract: LaunchContract, healthy: bool) -> Run { + let sink = Arc::new(Sink::default()); + let lease = Arc::new(FakeLease::default()); + let health = Arc::new(Health(AtomicBool::new(healthy))); + let child = Arc::new(Child::default()); + let stop = Arc::new(AtomicBool::new(false)); + let mut secrets = BTreeMap::new(); + secrets.insert("SYNAPSE_KEY".to_owned(), b"s3cret-value\n".to_vec()); + let ports = Ports { + credentials: Arc::new(Credentials(secrets)), + clock: Arc::new(FastClock::new(1_000_000)), + sink: sink.clone(), + lease: lease.clone(), + spawner: Box::new(FakeSpawner(child.clone())), + health: Box::new(move |_| health.clone()), + }; + let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let tmp = fx.dir.path().join("tmp"); + let flag = stop.clone(); + let outcome = + thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); + Run { + sink, + lease, + child, + stop, + outcome: Some(outcome), + } + } + + impl Run { + fn finish(&mut self) -> Outcome { + self.outcome.take().unwrap().join().unwrap() + } + + fn stop(&mut self) -> Outcome { + self.stop.store(true, Ordering::SeqCst); + self.finish() + } + } + + fn wait_until(what: &str, mut condition: impl FnMut() -> bool) { + for _ in 0..4000 { + if condition() { + return; + } + thread::sleep(Duration::from_millis(5)); + } + panic!("timed out waiting for {what}"); + } + + #[test] + fn stateful_child_never_starts_before_its_lease() { + let fx = fixture(true); + let mut launch = start(&fx, fx.contract("listen {{child.listen}}"), true); + wait_until("two warming heartbeats", || { + launch.sink.states().iter().filter(|s| *s == "warming").count() >= 2 + }); + assert_eq!(launch.child.spawns.load(Ordering::SeqCst), 0); + let warming = launch.sink.first("warming").unwrap(); + assert!(warming.write_lease_sha256.is_none()); + + // The lease binds the launcher's own Warming proof; only then it starts. + let lease = fx.lease(&warming.canonical_sha256().unwrap()); + *launch.lease.0.lock().unwrap() = Some(lease.clone()); + wait_until("child spawn", || { + launch.child.spawns.load(Ordering::SeqCst) == 1 + }); + wait_until("active", || launch.sink.first("active").is_some()); + let active = launch.sink.first("active").unwrap(); + assert_eq!( + active.write_lease_sha256.as_deref(), + Some(lease.canonical_sha256().unwrap().as_str()) + ); + assert_eq!(launch.stop(), Outcome::Stopped); + assert!(launch.child.terminated.load(Ordering::SeqCst)); + } + + #[test] + fn a_lease_for_another_activation_or_expected_is_not_accepted() { + let fx = fixture(true); + let foreign: [(&str, fn(&mut IdunnProcessWriteLeaseRecord)); 3] = [ + ("another runtime instance", |lease| { + lease.runtime_instance_id = "other".into() + }), + ("another Expected", |lease| { + lease.expected_projection_sha256 = digest('9') + }), + ("another warming proof", |lease| { + lease.warming_presence_sha256 = digest('9') + }), + ]; + for (name, edit) in foreign { + let mut contract = fx.contract("x"); + contract.lease_wait = Duration::from_secs(5); + let mut launch = start(&fx, contract, true); + // Installed at once; the launcher's own proofs are never digest('8'). + let mut lease = fx.lease(&digest('8')); + edit(&mut lease); + *launch.lease.0.lock().unwrap() = Some(lease); + let (child, sink) = (launch.child.clone(), launch.sink.clone()); + // Bounded: a lease deadline that never arrives fails here, not at the + // suite's outer timeout. + let outcome = within(Duration::from_secs(20), move || launch.finish()); + assert_eq!(outcome, Outcome::Failed("lease-timeout"), "{name}"); + assert_eq!(child.spawns.load(Ordering::SeqCst), 0, "{name}"); + assert_eq!(sink.states().last().unwrap(), "failed", "{name}"); + } + } + + #[test] + fn lease_loss_stops_the_child() { + let fx = fixture(true); + let mut launch = start(&fx, fx.contract("x"), true); + wait_until("warming", || launch.sink.first("warming").is_some()); + let warming = launch.sink.first("warming").unwrap(); + *launch.lease.0.lock().unwrap() = Some(fx.lease(&warming.canonical_sha256().unwrap())); + wait_until("active", || launch.sink.first("active").is_some()); + *launch.lease.0.lock().unwrap() = None; + let outcome = launch.finish(); + assert_eq!(outcome, Outcome::Failed("lease-lost")); + assert_ne!(outcome.exit_code(), 0); + assert!(launch.child.terminated.load(Ordering::SeqCst)); + assert_eq!(launch.sink.states().last().unwrap(), "failed"); + } + + #[test] + fn stateless_child_starts_without_a_lease() { + let fx = fixture(false); + let mut launch = start(&fx, fx.contract("x"), true); + wait_until("active", || launch.sink.first("active").is_some()); + assert_eq!(launch.child.spawns.load(Ordering::SeqCst), 1); + assert_eq!(launch.sink.states().first().unwrap(), "warming"); + assert!( + launch + .sink + .first("active") + .unwrap() + .write_lease_sha256 + .is_none() + ); + launch.stop(); + } + + #[test] + fn child_exit_ends_the_launcher() { + for code in [0, 3] { + let fx = fixture(false); + let mut launch = start(&fx, fx.contract("x"), true); + wait_until("active", || launch.sink.first("active").is_some()); + *launch.child.exit.lock().unwrap() = Some(code); + let outcome = launch.finish(); + assert_eq!(outcome, Outcome::Failed("child-exited"), "exit {code}"); + assert_eq!(outcome.exit_code(), 1); + assert_eq!(launch.sink.states().last().unwrap(), "failed"); + } + } + + #[test] + fn closed_placeholder_set_renders_and_refuses() { + let fx = fixture(false); + let template = "listen={{child.listen}}\nhost={{child.listen_host}}\nport={{child.listen_port}}\nroot={{state_root}}\nkey={{credential:SYNAPSE_KEY}}\npath={{credential_path:SYNAPSE_KEY}}\n"; + let mut launch = start(&fx, fx.contract(template), true); + wait_until("spawn", || launch.child.spawns.load(Ordering::SeqCst) == 1); + let args = launch.child.args.lock().unwrap()[0].clone(); + assert_eq!(args[..2], ["--serve", "--config"]); + let config = PathBuf::from(&args[2]); + assert_eq!(config.parent().unwrap(), fx.dir.path().join("tmp")); + assert_eq!( + fs::metadata(&config).unwrap().permissions().mode() & 0o777, + 0o600 + ); + let rendered = fs::read_to_string(&config).unwrap(); + assert_eq!( + rendered, + "listen=127.0.0.1:18845\nhost=127.0.0.1\nport=18845\nroot=/var/lib/synapse\nkey=s3cret-value\npath=/run/credentials/unit/SYNAPSE_KEY\n" + ); + assert!( + !rendered.contains("18835"), + "the candidate bind never reaches the child" + ); + launch.stop(); + + // Any other placeholder refuses to start, and the refusal names no input. + let fx = fixture(false); + let mut launch = start(&fx, fx.contract("a={{canary_bind}}"), true); + assert_eq!(launch.finish(), Outcome::Failed("child-start-refused")); + assert_eq!(launch.child.spawns.load(Ordering::SeqCst), 0); + let credentials = Credentials(BTreeMap::new()); + let renderer = ConfigRenderer { + credentials: &credentials, + listen: member(), + state_root: None, + }; + for template in [ + "{{canary_bind}}", + "{{credential:CANARY-MISSING}}", + "{{credential:canary b}}", + "{{child.listen}} {{canary", + "{{state_root}}", + ] { + let error = format!("{:#}", renderer.render(template).unwrap_err()); + assert!(!error.to_lowercase().contains("canary"), "{error}"); + } + } + + fn shared(fx: &Fixture, health: Arc) -> Arc { + shared_on(fx, health, Arc::new(SystemClock)) + } + + fn shared_on(fx: &Fixture, health: Arc, clock: Arc) -> Arc { + Arc::new(Shared { + authority: Mutex::new(fx.authority()), + phase: Mutex::new(Phase::Active(None)), + lease: Mutex::new(None), + lease_required: false, + health, + probed: Mutex::new(None), + clock, + }) + } + + #[test] + fn route_answer_degrades_with_the_child() { + let fx = fixture(false); + let health = Arc::new(Health(AtomicBool::new(true))); + let shared = shared(&fx, health.clone()); + let state_of = |shared: &Shared| { + let request = CultNetMessage::SnapshotRequest { + message_id: "challenge-1".into(), + schema_ids: None, + record_keys: None, + }; + let CultNetMessage::SnapshotResponseRaw { documents, .. } = + shared.observe(request).unwrap() + else { + panic!("not a raw snapshot response") + }; + let record: GameCultRuntimePresenceHealthRecord = + rmp_serde::from_slice(&documents[0].payload).unwrap(); + assert_eq!(record.detail, "route-observation:challenge-1"); + record.state + }; + // Never probed: nothing says the child is healthy. + assert_eq!(state_of(&shared), "degraded"); + shared.refresh_health(); + assert_eq!(state_of(&shared), "active"); + // A challenge answers from the last probe and never runs one. + health.0.store(false, Ordering::SeqCst); + assert_eq!(state_of(&shared), "active"); + shared.refresh_health(); + assert_eq!(state_of(&shared), "degraded"); + } + + #[test] + fn route_challenge_is_answered_with_both_signatures_through_idunns_client() { + let fx = fixture(false); + let shared = shared(&fx, Arc::new(Health(AtomicBool::new(true)))); + shared.refresh_health(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let responder: Responder = Arc::new(move |request| shared.observe(request)); + let halt = Arc::new(AtomicBool::new(false)); + let acceptor = PresenceAcceptor::new(listener, responder); + let serving = { + let halt = halt.clone(); + thread::spawn(move || acceptor.serve(halt)) + }; + let request = CultNetMessage::SnapshotRequest { + message_id: "challenge-7".into(), + schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), + record_keys: Some(vec![fx.expected.target.clone()]), + }; + let payload = + encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0).unwrap(); + let response = request_tcp_snapshot(address, &payload).ok().unwrap(); + let message = + decode_cultnet_message_from_slice(&response, CultNetWireContract::CultNetSchemaV0) + .unwrap(); + let answer = + exact_route_snapshot_response(message, "challenge-7", &fx.expected.target).unwrap(); + let record: GameCultRuntimePresenceHealthRecord = + rmp_serde::from_slice(&answer.canonical_presence).unwrap(); + let claim = authenticate_runtime_presence_claim( + &answer.canonical_presence, + &fx.verified, + RuntimePresenceAuthenticationContext { + trusted_received_at_unix_millis: record.observed_at_unix_millis, + maximum_age_millis: 10_000, + maximum_future_skew_millis: 10, + }, + ) + .unwrap(); + assert_eq!(claim.record().state, "active"); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap(); + } + + #[test] + fn contract_parsing_refuses_escapes_and_echoes_nothing() { + let base = |extra: &[&str]| -> Result { + let mut args: Vec = [ + "--child", + "bin/c", + "--template", + "t", + "--health-path", + "/health", + "--health-contract", + "h", + ] + .map(String::from) + .to_vec(); + args.extend(extra.iter().map(|value| value.to_string())); + LaunchContract::parse(PathBuf::from("/r"), args) + }; + assert!(base(&["--child-arg", "--flag-looking-value"]).is_ok()); + assert!(base(&["--child", "../escape"]).is_err()); + // The child serves members on Idunn's member bind: there is no + // listener to choose. + assert!(base(&["--listener", "loopback"]).is_err()); + assert!(base(&["--template", "/etc/passwd"]).is_err()); + let error = base(&["--canary-flag", "canary-value"]) + .unwrap_err() + .to_string(); + assert!(!error.contains("canary")); + } + + #[test] + fn real_lease_reader_holds_the_shared_lock_idunn_must_respect() { + let fx = fixture(true); + let path = fx.dir.path().join("lease.cc"); + let warming = digest('a'); + let lease = fx.lease(&warming); + write_record( + &path, + &lease.target, + IdunnProcessWriteLeaseRecord::TYPE, + cultnet_rs::IDUNN_PROCESS_WRITE_LEASE_SCHEMA, + lease.canonical_bytes().unwrap(), + ); + let reader: Arc = Arc::new(DriverLeaseReader::new(&lease.target, &path)); + let hold = LeaseHold::acquire(reader, &fx.expected, &fx.activation, &[warming]).unwrap(); + assert!(hold.still_current()); + let idunn = CultCacheWriteLeaseDriver::new(&lease.target, &path); + let denial = idunn.revoke_exact(Some(&lease)).unwrap_err(); + assert!(denial.to_string().contains("shared-lock consumer")); + drop(hold); + idunn.revoke_exact(Some(&lease)).unwrap(); + } + + #[test] + fn authority_binds_the_candidate_and_signs_strictly_increasing_sequences() { + let fx = fixture(false); + let mut authority = fx.authority(); + assert!(authority.require_candidate_bind("127.0.0.1:18835").is_ok()); + assert!(authority.require_candidate_bind("127.0.0.1:18836").is_err()); + let first = authority.sign("warming", "a", None, 5_000).unwrap(); + let second = authority.sign("warming", "b", None, 5_000).unwrap(); + assert_eq!(first.publisher_sequence, 5_000); + assert_eq!(second.publisher_sequence, 5_001); + assert_eq!( + first.bound_endpoint.as_deref(), + Some("tcp://127.0.0.1:18835") + ); + } + + #[test] + fn health_probe_is_healthy_only_on_a_200_from_the_child() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let member = listener.local_addr().unwrap(); + let probe = HttpHealthProbe { member, path: "/health".into() }; + let server = thread::spawn(move || { + for status in ["200 OK", "503 Service Unavailable"] { + let (mut stream, _) = listener.accept().unwrap(); + let mut line = String::new(); + io::BufRead::read_line(&mut io::BufReader::new(stream.try_clone().unwrap()), &mut line) + .unwrap(); + assert_eq!(line, "GET /health HTTP/1.1\r\n"); + write!(stream, "HTTP/1.1 {status}\r\nContent-Length: 2\r\n\r\nOK").unwrap(); + } + }); + assert!(probe.healthy()); + assert!(!probe.healthy()); + server.join().unwrap(); + assert!(!probe.healthy(), "no listener is not healthy"); + } + + #[test] + fn contract_parsing_reads_every_declared_field() { + let args: Vec = [ + "--child", "bin/c", "--child-arg", "-x", "--child-arg", "y", "--config-flag", "-c", + "--template", "t", "--health-path", "/h", + "--health-contract", "hc", "--state-root", "/s", "--lease-wait-secs", "11", + "--startup-secs", "22", "--heartbeat-secs", "33", + ] + .map(String::from) + .to_vec(); + let contract = LaunchContract::parse(PathBuf::from("/r"), args.clone()).unwrap(); + assert_eq!(contract.child_args, ["-x", "y"]); + assert_eq!(contract.config_flag.as_deref(), Some("-c")); + assert_eq!(contract.state_root, Some(PathBuf::from("/s"))); + assert_eq!( + (contract.lease_wait, contract.startup, contract.heartbeat), + (Duration::from_secs(11), Duration::from_secs(22), Duration::from_secs(33)) + ); + for zero in ["--lease-wait-secs", "--startup-secs", "--heartbeat-secs"] { + let mut bad = args.clone(); + bad.extend([zero.to_owned(), "0".to_owned()]); + assert!(LaunchContract::parse(PathBuf::from("/r"), bad).is_err(), "{zero}"); + } + } + + #[test] + fn credential_names_cannot_escape_or_be_empty() { + struct Any; + impl CredentialSource for Any { + fn contents(&self, _: &str) -> Result> { + Ok(b"v".to_vec()) + } + fn path(&self, name: &str) -> Result { + Ok(PathBuf::from(name)) + } + } + let renderer = ConfigRenderer { credentials: &Any, listen: member(), state_root: None }; + for ok in ["A", "a-b_9", "Z0"] { + assert_eq!(renderer.render(&format!("{{{{credential:{ok}}}}}")).unwrap(), "v"); + } + for bad in ["", "../x", "a/b", "a b", "a.b", "a:b", "é"] { + assert!(renderer.render(&format!("{{{{credential:{bad}}}}}")).is_err(), "{bad}"); + assert!(renderer.render(&format!("{{{{credential_path:{bad}}}}}")).is_err(), "{bad}"); + } + assert_eq!(renderer.render("a{{child.listen}}b{{child.listen}}c").unwrap(), "a127.0.0.1:18845b127.0.0.1:18845c"); + } + + #[test] + fn credentials_resolve_only_as_files_under_the_credentials_directory() { + let dir = tempfile::tempdir().unwrap(); + fs::write(dir.path().join("cred"), b"abc\n").unwrap(); + let credentials = CredentialsDirectory::new(dir.path().to_owned()); + assert_eq!(credentials.path("cred").unwrap(), dir.path().join("cred")); + assert_eq!(credentials.contents("cred").unwrap(), b"abc\n"); + assert!(credentials.path("unbound").is_err()); + for bad in ["", "..", "../cred", "a/b", ".", "a.b"] { + assert!(credentials.path(bad).is_err(), "{bad}"); + assert!(credentials.contents(bad).is_err(), "{bad}"); + } + assert!(CredentialsDirectory(None).path("cred").is_err()); + } + + #[test] + fn credential_placeholders_never_read_the_process_environment() { + let dir = tempfile::tempdir().unwrap(); + let file = dir.path().join("elsewhere"); + fs::write(&file, b"canary-file\n").unwrap(); + // SAFETY: variables no other test reads. + unsafe { + std::env::set_var("GAMECULT_IDUNN_CANDIDATE_BIND", "127.0.0.1:18835"); + std::env::set_var("IDUNN_LAUNCH_TEST_ENV_CREDENTIAL", &file); + } + let credentials = CredentialsDirectory::new(dir.path().to_owned()); + let renderer = ConfigRenderer { credentials: &credentials, listen: member(), state_root: None }; + for name in [ + "GAMECULT_IDUNN_CANDIDATE_BIND", + "IDUNN_LAUNCH_TEST_ENV_CREDENTIAL", + "../elsewhere", + "a/b", + "..", + ] { + for kind in ["credential", "credential_path"] { + let error = renderer.render(&format!("x={{{{{kind}:{name}}}}}")); + assert!(error.is_err(), "{kind}:{name}"); + assert!(!format!("{:#}", error.unwrap_err()).contains("18835")); + } + } + // The same name resolves when it is a credential file, never otherwise. + fs::write(dir.path().join("GAMECULT_IDUNN_CANDIDATE_BIND"), b"file-value\n").unwrap(); + assert_eq!( + renderer.render("{{credential:GAMECULT_IDUNN_CANDIDATE_BIND}}").unwrap(), + "file-value" + ); + } + + #[test] + fn odin_sink_takes_its_endpoint_only_from_the_odin_dependency() { + let fx = fixture(false); + let dependency = |capability: &str, endpoint: &str| cultnet_rs::IdunnExpectedDependency { + kind: "shared-infrastructure".into(), + capability: capability.into(), + schema: "s".into(), + compatibility: "v1".into(), + minimum_capacity: 1, + startup: "before-promotion".into(), + provider_id: None, + provider_authority: None, + provider_expected_projection_sha256: None, + provider_endpoint: Some(endpoint.into()), + }; + let mut expected = fx.expected.clone(); + assert!(OdinRudpSink::from_expected(&expected).is_err()); + expected.dependencies = vec![dependency("other.thing", "rudp://127.0.0.1:1")]; + assert!(OdinRudpSink::from_expected(&expected).is_err()); + expected.dependencies = vec![ + dependency("other.thing", "rudp://127.0.0.1:1"), + dependency(ODIN_CAPABILITY, "rudp://127.0.0.1:4100"), + ]; + assert_eq!( + OdinRudpSink::from_expected(&expected).unwrap().endpoint, + "127.0.0.1:4100".parse().unwrap() + ); + expected.dependencies = vec![dependency(ODIN_CAPABILITY, "rudp://not-an-address")]; + assert!(OdinRudpSink::from_expected(&expected).is_err()); + } + + + #[test] + fn a_probe_result_is_evidence_only_while_it_is_recent() { + let fx = fixture(false); + let clock = Arc::new(FastClock::new(1_000_000)); + let shared = shared_on(&fx, Arc::new(Health(AtomicBool::new(true))), clock.clone()); + shared.refresh_health(); + let stale = HEALTH_STALE.as_millis() as u64; + clock.monotonic.fetch_add(stale, Ordering::SeqCst); + assert_eq!(shared.serving_state(), "active", "a result exactly at the bound is still recent"); + clock.monotonic.fetch_add(1, Ordering::SeqCst); + assert_eq!(shared.serving_state(), "degraded", "a result past the bound is not evidence"); + shared.refresh_health(); + assert_eq!(shared.serving_state(), "active"); + } + + /// A lease reader that grants, at the first read once `after` Warming + /// records exist, a lease bound to the Warming record `back` places before + /// the newest. The launcher reads it from its own thread between publishes, + /// so the record it names is at a known place in the retained window. + struct WindowLease { + sink: Arc, + template: IdunnProcessWriteLeaseRecord, + after: usize, + back: usize, + granted: Mutex>, + } + + impl LeaseReader for WindowLease { + fn read(&self) -> Result> { + let mut granted = self.granted.lock().unwrap(); + if granted.is_none() { + let records = self.sink.0.lock().unwrap(); + let warming: Vec<_> = records.iter().filter(|r| r.state == "warming").collect(); + if warming.len() >= self.after { + let mut lease = self.template.clone(); + lease.warming_presence_sha256 = + warming[warming.len() - self.back].canonical_sha256().unwrap(); + *granted = Some(lease); + } + } + Ok(granted.clone()) + } + + fn hold_shared(&self) -> Result> { + Ok(Box::new(FakeLock)) + } + } + + /// Starts a stateful unit waiting on a `WindowLease` and reports whether it + /// accepted the lease (spawned the child) or retried well past the grant. + fn warming_window_accepts(back: usize) -> bool { + let fx = fixture(true); + let sink = Arc::new(Sink::default()); + let child = Arc::new(Child::default()); + let lease = Arc::new(WindowLease { + sink: sink.clone(), + template: fx.lease(""), + after: MAX_RECENT_WARMING_PROOFS + 6, + back, + granted: Mutex::new(None), + }); + let ports = Ports { + credentials: Arc::new(Credentials(BTreeMap::new())), + clock: Arc::new(FastClock::new(1_000_000)), + sink: sink.clone(), + lease: lease.clone(), + spawner: Box::new(FakeSpawner(child.clone())), + health: Box::new(|_| Arc::new(Health(AtomicBool::new(true)))), + }; + let mut contract = fx.contract("x"); + contract.heartbeat = POLL; + let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let tmp = fx.dir.path().join("tmp"); + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let unit = thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); + wait_until("the lease is granted", || lease.granted.lock().unwrap().is_some()); + let warming = || sink.states().iter().filter(|s| *s == "warming").count(); + let granted_at = warming(); + let mut spawned = false; + for _ in 0..4000 { + spawned = child.spawns.load(Ordering::SeqCst) == 1; + if spawned || warming() >= granted_at + 10 { + break; + } + thread::sleep(Duration::from_millis(5)); + } + stop.store(true, Ordering::SeqCst); + // A mutant may wedge the unit; only a unit that took the lease is joined. + if spawned { + assert_eq!(join_within(Duration::from_secs(10), unit), Outcome::Stopped); + } + spawned + } + + #[test] + fn the_warming_proof_window_keeps_exactly_the_cap() { + assert!( + warming_window_accepts(MAX_RECENT_WARMING_PROOFS), + "the oldest of the retained Warming proofs is accepted" + ); + assert!( + !warming_window_accepts(MAX_RECENT_WARMING_PROOFS + 1), + "a Warming proof past the cap is no longer accepted" + ); + } + + /// The child is healthy, then the wall clock steps by `step_millis`, then + /// the child wedges. A challenge must keep reading the truth throughout. + fn a_wall_clock_step_never_decides_freshness(step_millis: i64) { + struct Toggle { + healthy: AtomicBool, + probes: AtomicUsize, + } + impl HealthProbe for Toggle { + fn healthy(&self) -> bool { + self.probes.fetch_add(1, Ordering::SeqCst); + self.healthy.load(Ordering::SeqCst) + } + } + let fx = fixture(false); + let sink = Arc::new(Sink::default()); + let clock = Arc::new(FastClock::new(100_000_000)); + let probe = Arc::new(Toggle { healthy: AtomicBool::new(true), probes: AtomicUsize::new(0) }); + let given = probe.clone(); + let ports = Ports { + credentials: Arc::new(Credentials(BTreeMap::new())), + clock: clock.clone(), + sink: sink.clone(), + lease: Arc::new(FakeLease::default()), + spawner: Box::new(FakeSpawner(Arc::new(Child::default()))), + health: Box::new(move |_| given.clone()), + }; + let contract = fx.contract("x"); + let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = route.local_addr().unwrap(); + let tmp = fx.dir.path().join("tmp"); + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let unit = thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); + wait_until("active", || sink.first("active").is_some()); + assert_eq!(challenge_state(address, &fx).as_deref(), Some("active")); + + clock.step_wall(step_millis); + for _ in 0..10 { + assert_eq!( + challenge_state(address, &fx).as_deref(), + Some("active"), + "a healthy child reads active across a wall-clock step of {step_millis} ms" + ); + thread::sleep(Duration::from_millis(20)); + } + + probe.healthy.store(false, Ordering::SeqCst); + let probed = probe.probes.load(Ordering::SeqCst); + let published = sink.0.lock().unwrap().len(); + let started = Instant::now(); + let mut state = None; + while started.elapsed() < Duration::from_secs(5) { + state = challenge_state(address, &fx); + if state.as_deref() == Some("degraded") { + break; + } + thread::sleep(Duration::from_millis(20)); + } + assert_eq!( + state.as_deref(), + Some("degraded"), + "a wedged child reads degraded within the freshness window after a wall-clock step of {step_millis} ms" + ); + assert!(probe.probes.load(Ordering::SeqCst) > probed, "probing continued"); + wait_until("a heartbeat after the step", || sink.0.lock().unwrap().len() > published); + stop.store(true, Ordering::SeqCst); + assert_eq!(join_within(Duration::from_secs(10), unit), Outcome::Stopped); + } + + #[test] + fn a_backward_wall_clock_step_does_not_keep_a_wedged_child_active() { + a_wall_clock_step_never_decides_freshness(-600_000); + } + + #[test] + fn a_forward_wall_clock_step_does_not_make_a_healthy_child_degraded() { + a_wall_clock_step_never_decides_freshness(600_000); + } + + #[test] + fn the_cached_result_ages_on_the_monotonic_clock_alone() { + let fx = fixture(false); + let clock = Arc::new(FastClock::new(100_000_000)); + let shared = shared_on(&fx, Arc::new(Health(AtomicBool::new(true))), clock.clone()); + shared.refresh_health(); + for step in [-600_000, 1_200_000] { + clock.step_wall(step); + assert_eq!(shared.serving_state(), "active", "wall step {step}"); + } + clock.monotonic.fetch_add(HEALTH_STALE.as_millis() as u64 + 1, Ordering::SeqCst); + assert_eq!(shared.serving_state(), "degraded", "monotonic age decides, whatever the wall says"); + } + + /// A monotonic clock whose read, on a thread that marks itself, returns the + /// current value and then waits to be released: the presence thread + /// descheduled right after it reads the time. + struct HeldClock { + monotonic: AtomicU64, + held: Mutex>>, + release: Mutex>>, + } + + thread_local! { + static HOLD_THIS_READ: std::cell::Cell = const { std::cell::Cell::new(false) }; + } + + impl Clock for HeldClock { + fn now_millis(&self) -> u64 { + 0 + } + + fn monotonic_millis(&self) -> u64 { + let value = self.monotonic.load(Ordering::SeqCst); + if HOLD_THIS_READ.with(std::cell::Cell::get) { + self.held.lock().unwrap().take().unwrap().send(()).unwrap(); + self.release.lock().unwrap().take().unwrap().recv().unwrap(); + } + value + } + + fn sleep(&self, _: Duration) {} + } + + #[test] + fn a_presence_read_descheduled_across_a_probe_still_sees_a_healthy_child() { + let fx = fixture(false); + let (held_tx, held_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let clock = Arc::new(HeldClock { + monotonic: AtomicU64::new(1_000), + held: Mutex::new(Some(held_tx)), + release: Mutex::new(Some(release_rx)), + }); + let shared = shared_on(&fx, Arc::new(Health(AtomicBool::new(true))), clock.clone()); + shared.refresh_health(); + let reader = { + let shared = shared.clone(); + thread::spawn(move || { + HOLD_THIS_READ.with(|hold| hold.set(true)); + shared.serving_state() + }) + }; + // The reader is descheduled in its clock read. The lifecycle probes a + // millisecond later and stores a result stamped after the reader's time. + held_rx.recv().unwrap(); + clock.monotonic.store(1_001, Ordering::SeqCst); + assert!(shared.refresh_health()); + release_tx.send(()).unwrap(); + assert_eq!(reader.join().unwrap(), "active"); + } + + #[test] + fn the_declared_durations_are_bounded_at_parse() { + let base: Vec = [ + "--child", "bin/c", "--template", "t", "--health-path", "/h", "--health-contract", "hc", + ] + .map(String::from) + .to_vec(); + for flag in ["--lease-wait-secs", "--startup-secs", "--heartbeat-secs"] { + for (value, accepted) in [ + ("86400".to_owned(), true), + ("86401".to_owned(), false), + (u64::MAX.to_string(), false), + ] { + let mut args = base.clone(); + args.extend([flag.to_owned(), value.clone()]); + let parsed = LaunchContract::parse(PathBuf::from("/r"), args); + assert_eq!(parsed.is_ok(), accepted, "{flag} {value}"); + if let Err(error) = parsed { + assert!(!format!("{error:#}").contains(&value), "the refusal echoes its input"); + } + } + } + } + + #[test] + fn a_span_too_long_for_the_clock_is_a_deadline_that_never_arrives() { + assert_eq!(after(10, Duration::from_millis(5)), 15); + assert_eq!(after(u64::MAX - 1, Duration::from_secs(1)), u64::MAX); + assert_eq!(after(10, Duration::MAX), u64::MAX); + } + + /// Runs `work` and returns its result, failing the test when it has not + /// finished within `limit`, so a mutant that makes it wait forever fails + /// instead of hanging the suite. + fn within(limit: Duration, work: impl FnOnce() -> T + Send + 'static) -> T { + join_within(limit, thread::spawn(work)) + } + + fn join_within(limit: Duration, handle: thread::JoinHandle) -> T { + let started = Instant::now(); + while !handle.is_finished() { + assert!(started.elapsed() < limit, "still running after {limit:?}"); + thread::sleep(Duration::from_millis(10)); + } + handle.join().unwrap() + } + + /// The state the presence endpoint signs in answer to a challenge, or None + /// when it is not answered promptly. + fn challenge_state(address: SocketAddr, fx: &Fixture) -> Option { + let request = CultNetMessage::SnapshotRequest { + message_id: "wedge".into(), + schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), + record_keys: Some(vec![fx.expected.target.clone()]), + }; + let payload = + encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0).unwrap(); + let started = Instant::now(); + let answer = request_tcp_snapshot(address, &payload).ok()?; + if started.elapsed() > Duration::from_millis(750) { + return None; + } + let CultNetMessage::SnapshotResponseRaw { documents, .. } = + decode_cultnet_message_from_slice(&answer, CultNetWireContract::CultNetSchemaV0).ok()? + else { + return None; + }; + let record: GameCultRuntimePresenceHealthRecord = + rmp_serde::from_slice(&documents.first()?.payload).ok()?; + Some(record.state) + } + + #[derive(Clone, Copy)] + enum Wedge { + /// Accepts and never answers. + Silent, + /// Never accepts; the kernel backlog fills. + FullBacklog, + /// Accepts and answers 200 after 1.1 s, past the probe's deadline. + Slow, + } + + type WedgeSwitch = Arc>>; + + /// A child that answers 200 until wedged. Returns the member address, the + /// switch, and the thread to join after `halt`. + fn serving_child(halt: Arc) -> (SocketAddr, WedgeSwitch, thread::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let address = listener.local_addr().unwrap(); + let wedge: WedgeSwitch = Arc::default(); + let mode = wedge.clone(); + let serving = thread::spawn(move || { + while !halt.load(Ordering::SeqCst) { + let current = *mode.lock().unwrap(); + if matches!(current, Some(Wedge::FullBacklog)) { + thread::sleep(Duration::from_millis(5)); + continue; + } + let Ok((mut stream, _)) = listener.accept() else { + thread::sleep(Duration::from_millis(5)); + continue; + }; + stream.set_nonblocking(false).unwrap(); + match current { + None => { + let mut request = [0_u8; 256]; + let _ = stream.set_read_timeout(Some(Duration::from_millis(200))); + let _ = stream.read(&mut request); + let _ = stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"); + } + Some(Wedge::Slow) => { + thread::spawn(move || { + thread::sleep(Duration::from_millis(1100)); + let _ = stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"); + }); + } + _ => { + thread::spawn(move || { + thread::sleep(Duration::from_secs(8)); + drop(stream); + }); + } + } + } + }); + (address, wedge, serving) + } + + /// Whatever wedges the child, every challenge is still answered promptly, + /// and the answer turns to a signed degraded within a few probes. + fn a_wedged_child_is_answered_degraded(wedge: Wedge) { + let fx = fixture(false); + let sink = Arc::new(Sink::default()); + let halt = Arc::new(AtomicBool::new(false)); + let (member, mode, child) = serving_child(halt.clone()); + let ports = Ports { + credentials: Arc::new(Credentials(BTreeMap::new())), + clock: Arc::new(SystemClock), + sink: sink.clone(), + lease: Arc::new(FakeLease::default()), + spawner: Box::new(FakeSpawner(Arc::new(Child::default()))), + health: Box::new(|member| Arc::new(HttpHealthProbe { member, path: "/health".into() })), + }; + let contract = fx.contract("x"); + let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = route.local_addr().unwrap(); + let tmp = fx.dir.path().join("tmp"); + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let unit = thread::spawn(move || run(&contract, authority, route, member, &tmp, ports, &flag)); + wait_until("active", || sink.first("active").is_some()); + assert_eq!(challenge_state(address, &fx).as_deref(), Some("active")); + + *mode.lock().unwrap() = Some(wedge); + // A backlog fills only once clients have connected without being + // accepted. + let mut queued = Vec::new(); + if matches!(wedge, Wedge::FullBacklog) { + for _ in 0..4096 { + match TcpStream::connect_timeout(&member, Duration::from_millis(20)) { + Ok(stream) => queued.push(stream), + Err(_) => break, + } + } + } + let started = Instant::now(); + let mut degraded = false; + while started.elapsed() < Duration::from_secs(6) { + let state = challenge_state(address, &fx); + assert!(state.is_some(), "a challenge went unanswered after {:?}", started.elapsed()); + if state.as_deref() == Some("degraded") { + degraded = true; + break; + } + thread::sleep(Duration::from_millis(100)); + } + assert!(degraded, "the wedged child was never reported degraded"); + stop.store(true, Ordering::SeqCst); + assert_eq!(join_within(Duration::from_secs(10), unit), Outcome::Stopped); + halt.store(true, Ordering::SeqCst); + join_within(Duration::from_secs(10), child); + drop(queued); + } + + #[test] + fn a_silent_child_is_answered_degraded_never_silent() { + a_wedged_child_is_answered_degraded(Wedge::Silent); + } + + #[test] + fn a_child_with_a_full_backlog_is_answered_degraded_never_silent() { + a_wedged_child_is_answered_degraded(Wedge::FullBacklog); + } + + #[test] + fn a_child_answering_after_the_probe_deadline_is_answered_degraded_never_silent() { + a_wedged_child_is_answered_degraded(Wedge::Slow); + } + + /// The virtual time (milliseconds) of each record that has the given + /// detail. + fn times_of(sink: &Sink, detail: &str) -> Vec { + sink.0 + .lock() + .unwrap() + .iter() + .filter(|record| record.detail == detail) + .map(|record| record.observed_at_unix_millis) + .collect() + } + + #[test] + fn a_child_that_never_becomes_healthy_fails_the_unit_at_the_startup_timeout() { + let fx = fixture(false); + let mut contract = fx.contract("x"); + contract.startup = Duration::from_secs(5); + let mut launch = start(&fx, contract, false); + let sink = launch.sink.clone(); + let child = launch.child.clone(); + let outcome = within(Duration::from_secs(20), move || launch.finish()); + assert_eq!(outcome, Outcome::Failed("startup-timeout")); + assert!(child.terminated.load(Ordering::SeqCst), "the child outlived the unit"); + let started = times_of(&sink, "launcher-started")[0]; + let failed = times_of(&sink, "startup-timeout")[0]; + // Virtual time: the deadline, found within one poll of it. + let waited = failed - started; + assert!( + (5_000..=5_000 + POLL.as_millis() as u64).contains(&waited), + "failed after {waited} ms of virtual time" + ); + } + + #[test] + fn heartbeats_follow_the_contracts_cadence_while_waiting_for_the_lease_and_while_active() { + let fx = fixture(true); + let mut contract = fx.contract("x"); + contract.heartbeat = Duration::from_secs(1); + let mut launch = start(&fx, contract, true); + let cadence = 1_000..=1_000 + POLL.as_millis() as u64; + wait_until("lease-wait heartbeats", || { + times_of(&launch.sink, "write-lease-pending").len() >= 4 + }); + let warming = launch.sink.first("warming").unwrap(); + let mut beats = vec![warming.observed_at_unix_millis]; + beats.extend(times_of(&launch.sink, "write-lease-pending").into_iter().take(4)); + for pair in beats.windows(2) { + assert!(cadence.contains(&(pair[1] - pair[0])), "lease-wait beats {beats:?}"); + } + *launch.lease.0.lock().unwrap() = Some(fx.lease(&warming.canonical_sha256().unwrap())); + wait_until("active heartbeats", || times_of(&launch.sink, "heartbeat").len() >= 4); + let mut beats = vec![times_of(&launch.sink, "child-healthy")[0]]; + beats.extend(times_of(&launch.sink, "heartbeat").into_iter().take(4)); + for pair in beats.windows(2) { + assert!(cadence.contains(&(pair[1] - pair[0])), "active beats {beats:?}"); + } + assert_eq!(launch.stop().exit_code(), 0); + } + + + /// Healthy, and notes the virtual time of every probe. + struct RecordingProbe { + clock: Arc, + at: Mutex>, + } + + impl HealthProbe for RecordingProbe { + fn healthy(&self) -> bool { + self.at.lock().unwrap().push(self.clock.monotonic_millis()); + true + } + } + + #[test] + fn an_active_child_is_probed_at_the_refresh_cadence() { + let fx = fixture(false); + let sink = Arc::new(Sink::default()); + let clock = Arc::new(FastClock::new(1_000_000)); + let probe = Arc::new(RecordingProbe { clock: clock.clone(), at: Mutex::new(Vec::new()) }); + let seen = probe.clone(); + let ports = Ports { + credentials: Arc::new(Credentials(BTreeMap::new())), + clock, + sink: sink.clone(), + lease: Arc::new(FakeLease::default()), + spawner: Box::new(FakeSpawner(Arc::new(Child::default()))), + health: Box::new(move |_| probe.clone()), + }; + let contract = fx.contract("x"); + let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let tmp = fx.dir.path().join("tmp"); + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let unit = thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); + wait_until("five probes", || seen.at.lock().unwrap().len() >= 5); + stop.store(true, Ordering::SeqCst); + assert_eq!(join_within(Duration::from_secs(10), unit), Outcome::Stopped); + let at = seen.at.lock().unwrap().clone(); + // The first probe ends startup; every later one follows the cadence. + let cadence = HEALTH_REFRESH.as_millis() as u64..=HEALTH_REFRESH.as_millis() as u64 + POLL.as_millis() as u64; + for pair in at.windows(2) { + assert!(cadence.contains(&(pair[1] - pair[0])), "probes at {at:?}"); + } + } + + #[test] + fn process_spawner_escalates_to_kill_when_the_child_ignores_sigterm() { + let mut stubborn = ProcessSpawner + .spawn(Path::new("/bin/sh"), &["-c".into(), "trap '' TERM; while :; do :; done".into()], None) + .unwrap(); + thread::sleep(Duration::from_millis(200)); + let started = Instant::now(); + stubborn.terminate(Duration::from_millis(400)); + assert!(started.elapsed() >= Duration::from_millis(400), "SIGTERM gets its grace period"); + assert!(stubborn.exited().is_some(), "then SIGKILL ends it"); + } + + #[test] + fn process_spawner_reports_exit_codes_and_terminates_a_running_child() { + let mut gone = ProcessSpawner + .spawn(Path::new("/bin/sh"), &["-c".into(), "exit 3".into()], None) + .unwrap(); + wait_until("child exit", || gone.exited().is_some()); + assert_eq!(gone.exited(), Some(3)); + let mut running = ProcessSpawner + .spawn(Path::new("/bin/sh"), &["-c".into(), "sleep 30".into()], None) + .unwrap(); + assert_eq!(running.exited(), None); + running.terminate(Duration::from_secs(5)); + assert!(running.exited().is_some()); + } + + // ------------------------------------------------------------ presence acceptor + + type Accepted = io::Result<(TcpStream, SocketAddr)>; + + fn serve_presence_accepting( + fx: &Fixture, + responder: Responder, + accept: impl FnMut(&TcpListener) -> Accepted + Send + 'static, + ) -> (SocketAddr, Arc, thread::JoinHandle<()>) { + let _ = fx; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let acceptor = PresenceAcceptor::new(listener, responder); + let halt = Arc::new(AtomicBool::new(false)); + let flag = halt.clone(); + (address, halt, thread::spawn(move || acceptor.serve_accepting(flag, accept))) + } + + fn observing(fx: &Fixture) -> Responder { + let shared = shared(fx, Arc::new(Health(AtomicBool::new(true)))); + shared.refresh_health(); + Arc::new(move |request| shared.observe(request)) + } + + fn serve_presence(fx: &Fixture) -> (SocketAddr, Arc, thread::JoinHandle<()>) { + serve_presence_accepting(fx, observing(fx), |listener| listener.accept()) + } + + /// True when the peer closed (EOF or reset) within `within`, having sent + /// nothing. + fn closed_within(stream: &mut TcpStream, within: Duration) -> bool { + stream.set_read_timeout(Some(within)).unwrap(); + let mut byte = [0_u8; 64]; + match stream.read(&mut byte) { + Ok(0) => true, + Ok(_) => false, + Err(error) => !matches!(error.kind(), io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut), + } + } + + fn challenge_answered(address: SocketAddr, fx: &Fixture) -> bool { + let request = CultNetMessage::SnapshotRequest { + message_id: "bounded".into(), + schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), + record_keys: Some(vec![fx.expected.target.clone()]), + }; + let payload = + encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0).unwrap(); + request_tcp_snapshot(address, &payload).is_ok() + } + + #[test] + fn only_challenge_frames_are_answered_and_anything_else_is_closed_unanswered() { + let fx = fixture(false); + let reached = Arc::new(AtomicUsize::new(0)); + let responder: Responder = { + let (inner, reached) = (observing(&fx), reached.clone()); + Arc::new(move |request| { + reached.fetch_add(1, Ordering::SeqCst); + inner(request) + }) + }; + let (address, halt, serving) = + serve_presence_accepting(&fx, responder, |listener| listener.accept()); + // Member traffic is not presence: an HTTP request gets no byte back. + let mut member = TcpStream::connect(address).unwrap(); + member + .write_all(b"POST /_matrix/x HTTP/1.1\r\nContent-Length: 2\r\n\r\n\x00\x01") + .unwrap(); + assert!(closed_within(&mut member, Duration::from_secs(3)), "member bytes were answered"); + // A leading 0x00 with a frame length outside its bound never reaches + // the responder. + let mut malformed = TcpStream::connect(address).unwrap(); + malformed.write_all(&[0x00, 0xff, 0xff, 0xff]).unwrap(); + assert!(closed_within(&mut malformed, Duration::from_secs(3)), "a malformed frame was answered"); + assert_eq!(reached.load(Ordering::SeqCst), 0); + // A real challenge is answered, and is the only thing that reaches it. + assert!(challenge_answered(address, &fx)); + assert_eq!(reached.load(Ordering::SeqCst), 1); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap(); + } + + #[test] + fn a_challenge_up_to_the_frame_bound_is_answered_and_one_past_it_is_not() { + let fx = fixture(false); + let (address, halt, serving) = serve_presence(&fx); + // A request far larger than a few KiB, well inside the 1 MiB bound. + let request = CultNetMessage::SnapshotRequest { + message_id: "large".into(), + schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), + record_keys: Some((0..300).map(|index| format!("{index:0>40}")).collect()), + }; + let payload = + encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0).unwrap(); + assert!(payload.len() > 10_000 && payload.len() < CHALLENGE_MAX_BYTES); + assert!(request_tcp_snapshot(address, &payload).is_ok()); + // One byte past the bound is closed unanswered. + let past = (CHALLENGE_MAX_BYTES as u32 + 1).to_be_bytes(); + assert_eq!(past[0], CHALLENGE_FIRST_BYTE); + let mut oversized = TcpStream::connect(address).unwrap(); + oversized.write_all(&past).unwrap(); + assert!(closed_within(&mut oversized, Duration::from_secs(3)), "an oversized frame was read"); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap(); + } + + #[test] + fn an_idle_peer_holds_the_acceptor_for_the_presence_deadline_and_no_longer() { + let fx = fixture(false); + let (address, halt, serving) = serve_presence(&fx); + let mut idle = TcpStream::connect(address).unwrap(); + let started = Instant::now(); + // Idunn waits 3 s for a challenge answer; one stalled peer ahead of it + // must not cost it that. + assert!(challenge_answered(address, &fx)); + let waited = started.elapsed(); + assert!(waited >= Duration::from_millis(800), "the idle peer was cut early: {waited:?}"); + assert!(waited < Duration::from_millis(2200), "the idle peer held the acceptor: {waited:?}"); + assert!(closed_within(&mut idle, Duration::from_secs(1)), "the idle peer stayed connected"); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap(); + } + + #[test] + fn a_challenge_trickled_a_byte_at_a_time_is_cut_off_at_the_presence_deadline() { + let fx = fixture(false); + let (address, halt, serving) = serve_presence(&fx); + let mut slow = TcpStream::connect(address).unwrap(); + slow.write_all(&[0x00, 0x00, 0x01, 0x00]).unwrap(); + let started = Instant::now(); + let mut closed = false; + while started.elapsed() < Duration::from_secs(4) { + if closed_within(&mut slow, Duration::from_millis(150)) || slow.write_all(&[0x01]).is_err() { + closed = true; + break; + } + } + assert!(closed, "the trickle was never cut off"); + let elapsed = started.elapsed(); + assert!(elapsed >= Duration::from_millis(800), "cut before its deadline: {elapsed:?}"); + assert!(elapsed < Duration::from_millis(2200), "cut after its deadline: {elapsed:?}"); + assert!(challenge_answered(address, &fx)); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap(); + } + + #[test] + fn accept_errors_never_end_the_presence_acceptor() { + let fx = fixture(false); + // Descriptor exhaustion: the first accepts fail, then descriptors free. + let failures = Arc::new(AtomicUsize::new(0)); + let (address, halt, serving) = { + let failures = failures.clone(); + serve_presence_accepting(&fx, observing(&fx), move |listener| { + if failures.fetch_add(1, Ordering::SeqCst) < 3 { + Err(io::Error::from_raw_os_error(libc::EMFILE)) + } else { + listener.accept() + } + }) + }; + assert!(challenge_answered(address, &fx), "the acceptor did not recover"); + assert!(failures.load(Ordering::SeqCst) >= 4); + assert!(!serving.is_finished()); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap(); + } + + /// A listener whose accept always fails: a datagram socket's descriptor. + fn failing_listener() -> TcpListener { + use std::os::fd::{FromRawFd, IntoRawFd}; + let datagram = UdpSocket::bind("127.0.0.1:0").unwrap(); + // SAFETY: the descriptor is owned by the new listener alone. + unsafe { TcpListener::from_raw_fd(datagram.into_raw_fd()) } + } + + #[test] + fn an_acceptor_whose_every_accept_fails_runs_until_halted() { + let fx = fixture(false); + let acceptor = PresenceAcceptor::new(failing_listener(), observing(&fx)); + let halt = Arc::new(AtomicBool::new(false)); + let flag = halt.clone(); + let serving = thread::spawn(move || acceptor.serve(flag)); + thread::sleep(Duration::from_millis(400)); + assert!(!serving.is_finished(), "the acceptor ended on an accept error"); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap(); + } + + /// A clock that kills the presence thread when it reads the time: the + /// stand-in for a presence acceptor that dies while the child is fine. + /// Answering a challenge reads the cached result's age, which reads the + /// monotonic clock. + struct PresenceTripwire(FastClock); + + impl Clock for PresenceTripwire { + fn now_millis(&self) -> u64 { + self.0.now_millis() + } + + fn monotonic_millis(&self) -> u64 { + assert_ne!(thread::current().name(), Some("presence-acceptor"), "presence thread killed"); + self.0.monotonic_millis() + } + + fn sleep(&self, duration: Duration) { + self.0.sleep(duration); + } + } + + #[test] + fn a_dead_presence_acceptor_fails_the_unit() { + let fx = fixture(false); + let sink = Arc::new(Sink::default()); + let child = Arc::new(Child::default()); + let ports = Ports { + credentials: Arc::new(Credentials(BTreeMap::new())), + clock: Arc::new(PresenceTripwire(FastClock::new(1_000_000))), + sink: sink.clone(), + lease: Arc::new(FakeLease::default()), + spawner: Box::new(FakeSpawner(child.clone())), + health: Box::new(|_| Arc::new(Health(AtomicBool::new(true)))), + }; + let contract = fx.contract("x"); + let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = route.local_addr().unwrap(); + let tmp = fx.dir.path().join("tmp"); + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let handle = + thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); + wait_until("active", || sink.first("active").is_some()); + // The first challenge reaches the tripwire on the presence thread. + assert!(!challenge_answered(address, &fx)); + assert_eq!(join_within(Duration::from_secs(20), handle), Outcome::Failed("presence-acceptor")); + assert_eq!(sink.first("failed").unwrap().detail, "presence-acceptor"); + assert!(child.terminated.load(Ordering::SeqCst), "the child outlived the unit"); + } + + #[test] + fn a_live_acceptor_keeps_presence_active() { + let fx = fixture(false); + let mut launch = start(&fx, fx.contract("x"), true); + wait_until("two active heartbeats", || { + launch.sink.states().iter().filter(|state| *state == "active").count() >= 2 + }); + assert!(launch.sink.first("degraded").is_none()); + launch.stop(); + } + + /// A store with a lease for `fx`, and Idunn's own driver over it. + fn lease_store( + fx: &Fixture, + name: &str, + ) -> (DriverLeaseReader, CultCacheWriteLeaseDriver, IdunnProcessWriteLeaseRecord) { + let dir = fx.dir.path().join(name); + fs::create_dir(&dir).unwrap(); + let path = dir.join("lease.cc"); + let lease = fx.lease(&digest('a')); + write_record( + &path, + &lease.target, + IdunnProcessWriteLeaseRecord::TYPE, + cultnet_rs::IDUNN_PROCESS_WRITE_LEASE_SCHEMA, + lease.canonical_bytes().unwrap(), + ); + fs::write(dir.join("lease.cc.lock"), b"").unwrap(); + ( + DriverLeaseReader::new(&lease.target, &path), + CultCacheWriteLeaseDriver::new(&lease.target, &path), + lease, + ) + } + + #[test] + fn a_child_that_inherits_the_lease_lock_holds_it_after_the_launcher_is_gone() { + let fx = fixture(true); + let (reader, idunn, lease) = lease_store(&fx, "inherited"); + let guard = reader.hold_shared().unwrap(); + let mut child = ProcessSpawner + .spawn(Path::new("/bin/sh"), &["-c".into(), "exec sleep 30".into()], guard.descriptor()) + .unwrap(); + // The launcher dies: its own descriptor goes with it. + drop(guard); + // Held for a full second: a fork of some other test could pin the lock + // for an instant, only the child under test keeps it that long. + for _ in 0..10 { + let denial = idunn.revoke_exact(Some(&lease)).unwrap_err(); + assert!(denial.to_string().contains("shared-lock consumer"), "the child does not hold the lock"); + thread::sleep(Duration::from_millis(100)); + } + child.terminate(Duration::from_secs(5)); + wait_until("child gone", || child.exited().is_some()); + idunn.revoke_exact(Some(&lease)).unwrap(); + } + + #[test] + fn a_child_spawned_without_the_descriptor_leaves_the_lock_with_the_launcher() { + let fx = fixture(true); + let (reader, idunn, lease) = lease_store(&fx, "uninherited"); + let guard = reader.hold_shared().unwrap(); + let mut child = ProcessSpawner + .spawn(Path::new("/bin/sh"), &["-c".into(), "exec sleep 30".into()], None) + .unwrap(); + drop(guard); + // Another test's fork may hold a copy for the instant before its exec; + // the child under test would hold it for thirty seconds. + wait_until("lock released", || idunn.revoke_exact(Some(&lease)).is_ok()); + child.terminate(Duration::from_secs(5)); + } + + #[test] + fn the_launcher_hands_its_lease_descriptor_to_the_child_it_spawns() { + let fx = fixture(true); + let mut launch = start(&fx, fx.contract("x"), true); + wait_until("warming", || launch.sink.first("warming").is_some()); + let warming = launch.sink.first("warming").unwrap(); + *launch.lease.0.lock().unwrap() = Some(fx.lease(&warming.canonical_sha256().unwrap())); + wait_until("active", || launch.sink.first("active").is_some()); + assert_eq!(*launch.child.inherited.lock().unwrap(), [Some(FAKE_LOCK_FD)]); + launch.stop(); + + let fx = fixture(false); + let mut launch = start(&fx, fx.contract("x"), true); + wait_until("active", || launch.sink.first("active").is_some()); + assert_eq!(*launch.child.inherited.lock().unwrap(), [None]); + launch.stop(); + } + + /// Unhealthy for its first `probes` calls, healthy from then on. + struct LateHealth { + calls: AtomicUsize, + probes: usize, + } + + impl HealthProbe for LateHealth { + fn healthy(&self) -> bool { + self.calls.fetch_add(1, Ordering::SeqCst) >= self.probes + } + } + + #[test] + fn a_child_that_becomes_healthy_late_is_not_killed_at_its_first_probe() { + let fx = fixture(false); + let mut contract = fx.contract("x"); + contract.startup = Duration::from_secs(60); + let sink = Arc::new(Sink::default()); + let child = Arc::new(Child::default()); + let health = Arc::new(LateHealth { calls: AtomicUsize::new(0), probes: 5 }); + let ports = Ports { + credentials: Arc::new(Credentials(BTreeMap::new())), + clock: Arc::new(FastClock::new(1_000_000)), + sink: sink.clone(), + lease: Arc::new(FakeLease::default()), + spawner: Box::new(FakeSpawner(child.clone())), + health: Box::new(move |_| health.clone()), + }; + let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let tmp = fx.dir.path().join("tmp"); + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let handle = + thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); + wait_until("active", || sink.first("active").is_some()); + assert!(!child.terminated.load(Ordering::SeqCst), "killed while it was still starting"); + assert_eq!(child.spawns.load(Ordering::SeqCst), 1); + assert!(sink.first("failed").is_none()); + stop.store(true, Ordering::SeqCst); + assert_eq!(handle.join().unwrap(), Outcome::Stopped); + } + + #[test] + fn a_silent_child_fails_the_probe_within_its_timeout() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let member = listener.local_addr().unwrap(); + // Accepts and never answers. + let held = thread::spawn(move || { + let _connection = listener.accept().unwrap(); + thread::sleep(Duration::from_secs(4)); + }); + let probe = HttpHealthProbe { member, path: "/health".into() }; + let started = Instant::now(); + assert!(!within(Duration::from_secs(5), move || probe.healthy())); + let elapsed = started.elapsed(); + assert!(elapsed >= Duration::from_millis(900), "gave up before its timeout: {elapsed:?}"); + assert!(elapsed < HEALTH_PROBE_TIMEOUT + Duration::from_millis(200), "outlived its timeout: {elapsed:?}"); + drop(held); + } + + #[test] + fn the_probe_deadline_covers_the_whole_response() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let member = listener.local_addr().unwrap(); + // Every byte arrives inside any per-read timeout, but the head takes + // three seconds in all. + let dribbler = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + for byte in b"HTTP/1.1 200 OK" { + if stream.write_all(&[*byte]).is_err() { + return; + } + thread::sleep(Duration::from_millis(250)); + } + }); + let probe = HttpHealthProbe { member, path: "/health".into() }; + let started = Instant::now(); + assert!(!within(Duration::from_secs(5), move || probe.healthy())); + let elapsed = started.elapsed(); + assert!(elapsed < HEALTH_PROBE_TIMEOUT + Duration::from_millis(200), "outlived its deadline: {elapsed:?}"); + join_within(Duration::from_secs(10), dribbler); + } + + #[test] + fn a_process_spawner_refuses_an_invalid_lease_descriptor_and_runs_no_child() { + let dir = tempfile::tempdir().unwrap(); + let marker = dir.path().join("ran"); + let script = format!("touch '{}'", marker.display()); + for invalid in [-1, 9999] { + let refused = ProcessSpawner.spawn( + Path::new("/bin/sh"), + &["-c".to_owned(), script.clone()], + Some(invalid), + ); + assert!(refused.is_err(), "descriptor {invalid} was accepted"); + } + thread::sleep(Duration::from_millis(300)); + assert!(!marker.exists(), "a child ran past an invalid descriptor"); + // Control: the same spawn without a descriptor runs the child. + let mut child = ProcessSpawner + .spawn(Path::new("/bin/sh"), &["-c".to_owned(), script], None) + .unwrap(); + wait_until("the control child to exit", || child.exited().is_some()); + assert!(marker.exists()); + } + + #[test] + fn the_member_bind_is_a_loopback_address_other_than_the_candidate() { + let candidate = "127.0.0.1:18835"; + assert_eq!( + parse_member_bind("127.0.0.1:18845", candidate).unwrap(), + "127.0.0.1:18845".parse().unwrap() + ); + for bad in [ + "127.0.0.1:18835", + "10.0.0.5:18845", + "0.0.0.0:18845", + "canary-not-an-address", + "", + ] { + let error = parse_member_bind(bad, candidate).unwrap_err().to_string(); + assert!(!error.contains("canary") && !error.contains("10.0.0.5"), "{error}"); + } + assert!(parse_member_bind("127.0.0.1:18845", "not-an-address").is_err()); + } +} diff --git a/src/lib.rs b/src/lib.rs index 7b3fd60..b5499a6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,4 +4,10 @@ pub mod deployment_plan; pub mod drivers; pub mod host; pub mod host_actuator; +#[cfg(unix)] +pub mod launcher; +mod incident; +mod runner_evidence; pub mod provisioning; +#[cfg(unix)] +mod steward_orders; diff --git a/src/runner_evidence.rs b/src/runner_evidence.rs new file mode 100644 index 0000000..30dd1d2 --- /dev/null +++ b/src/runner_evidence.rs @@ -0,0 +1,805 @@ +//! The output tail of a failed runner step. +//! +//! One file, `runner-evidence.cc`, beside `control.cc`, holds one type: the +//! last bytes of a failed runner container's stdout and stderr, one record per +//! transaction. The engine's pre-fence failure path is its only writer, and +//! `idunn status --command` its only reader. It is never `control.cc`, which +//! refuses any type it does not know, and never `incidents.cc`, which is +//! published and carries no free text. +//! +//! The tail is raw runner output. It is kept unredacted and readable by root +//! alone (mode 0600): root can already read every secret the runner was given. +//! It never enters an error, a transaction record or the journal. The status +//! reader is a terminal, so everything it prints from this store goes through +//! one allowlist sink (`emit`): printable ASCII passes, all else is escaped. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, bail, ensure}; +use cultcache_rs::{ + CultCacheEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, + TryCompareExchangeSnapshotOutcome, +}; +use serde::{Deserialize, Serialize}; + +use crate::control_plane::{decode_record, require_id, typed_envelope}; +use crate::drivers::{ + LockContended, RUNNER_OUTPUT_TAIL_BYTES, RunnerStepFailure, RunnerSubject, + exchange_behind_private_lock, runner_status, +}; + +const RUNNER_FAILURE_SCHEMA: &str = "idunn.runner_failure.v1"; + +/// Records kept; the writer drops the oldest past this, so the store needs no +/// other owner. +pub(crate) const RUNNER_EVIDENCE_RECORDS: usize = 32; + +/// `runner-evidence.cc`, beside the control store. +pub(crate) fn runner_evidence_store_path(state_store: &Path) -> PathBuf { + state_store.with_file_name("runner-evidence.cc") +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum SubjectKind { + Step, + ExternalInput, +} + +#[derive(Clone, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "idunn.runner_failure", schema = "idunn.runner_failure.v1")] +pub(crate) struct RunnerFailureRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) transaction_id: String, + #[cultcache(key = 2)] + pub(crate) command_id: String, + #[cultcache(key = 3)] + pub(crate) target: String, + #[cultcache(key = 4)] + pub(crate) subject_kind: SubjectKind, + #[cultcache(key = 5)] + pub(crate) subject_id: String, + #[cultcache(key = 6)] + pub(crate) runner: String, + #[cultcache(key = 7)] + pub(crate) exit_code: Option, + #[cultcache(key = 8)] + pub(crate) signal: Option, + #[cultcache(key = 9)] + pub(crate) stdout_tail: Vec, + #[cultcache(key = 10)] + pub(crate) stdout_total_bytes: u64, + #[cultcache(key = 11)] + pub(crate) stderr_tail: Vec, + #[cultcache(key = 12)] + pub(crate) stderr_total_bytes: u64, + #[cultcache(key = 13)] + pub(crate) recorded_at_unix_millis: u64, +} + +/// Tails stay out of `Debug`, so a failed assertion or a logged record never +/// prints runner output. +impl std::fmt::Debug for RunnerFailureRecord { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RunnerFailureRecord") + .field("transaction_id", &self.transaction_id) + .field("subject_id", &self.subject_id) + .finish_non_exhaustive() + } +} + +impl RunnerFailureRecord { + pub(crate) fn from_failure( + transaction_id: &str, + command_id: &str, + target: &str, + failure: &RunnerStepFailure, + now: u64, + ) -> Self { + let (subject_kind, subject_id) = match &failure.subject { + RunnerSubject::Step(id) => (SubjectKind::Step, id), + RunnerSubject::ExternalInput(id) => (SubjectKind::ExternalInput, id), + }; + Self { + schema_version: RUNNER_FAILURE_SCHEMA.to_owned(), + transaction_id: transaction_id.to_owned(), + command_id: command_id.to_owned(), + target: target.to_owned(), + subject_kind, + subject_id: subject_id.clone(), + runner: failure.runner.clone(), + exit_code: failure.exit_code, + signal: failure.signal, + stdout_tail: failure.stdout_tail.clone(), + stdout_total_bytes: failure.stdout_total_bytes, + stderr_tail: failure.stderr_tail.clone(), + stderr_total_bytes: failure.stderr_total_bytes, + recorded_at_unix_millis: now, + } + } + + pub(crate) fn validate(&self) -> Result<()> { + ensure!( + self.schema_version == RUNNER_FAILURE_SCHEMA, + "runner failure schema is unsupported" + ); + require_id(&self.transaction_id, "runner failure transaction")?; + require_id(&self.command_id, "runner failure command")?; + require_id(&self.target, "runner failure target")?; + require_id(&self.subject_id, "runner failure subject")?; + require_id(&self.runner, "runner failure runner")?; + for (tail, total, stream) in [ + (&self.stdout_tail, self.stdout_total_bytes, "stdout"), + (&self.stderr_tail, self.stderr_total_bytes, "stderr"), + ] { + ensure!( + tail.len() <= RUNNER_OUTPUT_TAIL_BYTES, + "runner failure {stream} tail exceeds its bound" + ); + ensure!( + tail.len() as u64 <= total, + "runner failure {stream} tail exceeds its stream" + ); + } + Ok(()) + } + + fn envelope(&self) -> Result { + self.validate()?; + typed_envelope( + &self.transaction_id, + Self::TYPE, + RUNNER_FAILURE_SCHEMA, + self, + self.recorded_at_unix_millis, + ) + } +} + +/// One read of the store: the envelopes exactly as stored, and the records +/// decoded from them in the same order. +fn snapshot(path: &Path) -> Result<(Vec, Vec)> { + let envelopes = SingleFileMessagePackBackingStore::new(path) + .pull_all_read_only_snapshot() + .context("reading the runner evidence store")?; + let mut records = Vec::new(); + for envelope in &envelopes { + ensure!( + envelope.r#type == RunnerFailureRecord::TYPE + && envelope.schema_id.as_deref() == Some(RUNNER_FAILURE_SCHEMA), + "the runner evidence store holds a document that is not a runner failure" + ); + let record: RunnerFailureRecord = decode_record(envelope)?; + record.validate()?; + ensure!( + envelope.key == record.transaction_id, + "runner evidence store key differs from the record's transaction" + ); + records.push(record); + } + Ok((envelopes, records)) +} + +/// How a write left the store. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Kept { + Stored, + /// The store held bytes that do not decode; it was moved to + /// `unreadable_store_path` and the record opened a fresh one. `replaced` + /// is the last-written time (unix milliseconds) of an earlier set-aside + /// store this one displaced, which is gone: one slot, and it says so. + SetAside { replaced: Option }, +} + +/// Where an undecodable store is kept. One slot: the next one replaces it, so +/// the damage a bad store can cost is one extra file. +pub(crate) fn unreadable_store_path(path: &Path) -> PathBuf { + path.with_file_name("runner-evidence.unreadable.cc") +} + +/// Moves the store over the previous set-aside one, root-only, and reports +/// what it displaced. Nothing of the store's content is named: a decode error +/// can quote stored values, and none is carried here. +fn set_aside_unreadable(path: &Path) -> Result> { + let aside = unreadable_store_path(path); + let replaced = std::fs::symlink_metadata(&aside).ok().map(|earlier| { + earlier + .modified() + .ok() + .and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok()) + .map_or(0, |since| since.as_millis() as u64) + }); + std::fs::rename(path, &aside) + .context("the runner evidence store is undecodable and could not be set aside")?; + keep_private(&aside)?; + Ok(replaced) +} + +/// How the writer reads the store's bytes; tests inject an I/O error here. +type ReadStore<'a> = &'a dyn Fn(&Path) -> std::io::Result>; + +fn unreadable_fault(error: &std::io::Error) -> anyhow::Error { + anyhow::anyhow!("the runner evidence store could not be read: {}", error.kind()) +} + +/// The one writer. Upsert by transaction id; past `RUNNER_EVIDENCE_RECORDS` the +/// oldest records go in the same exchange. The file ends at mode 0600. +/// +/// A store is set aside only when bytes that were read do not decode. An I/O +/// error leaves the store in place, keeps no evidence for that failure, and +/// the error names its kind for the journal. +pub(crate) fn record(path: &Path, record: &RunnerFailureRecord) -> Result { + record_reading(path, record, &|path| std::fs::read(path)) +} + +fn record_reading(path: &Path, record: &RunnerFailureRecord, read: ReadStore) -> Result { + let first = match read(path) { + Ok(bytes) => Some(bytes), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(unreadable_fault(&error)), + }; + let (envelopes, records, how) = match snapshot(path) { + Ok((envelopes, records)) => (envelopes, records, Kept::Stored), + Err(_) => { + let Some(first) = first else { + bail!("the runner evidence store could not be decoded"); + }; + // Undecodable only if the same bytes are still there to read: a + // second read that fails, or finds other bytes, is not a verdict. + match read(path) { + Ok(second) if second == first => {} + Ok(_) => bail!("the runner evidence store changed while it was being read"), + Err(error) => return Err(unreadable_fault(&error)), + } + let replaced = set_aside_unreadable(path)?; + (Vec::new(), Vec::new(), Kept::SetAside { replaced }) + } + }; + let mut kept: Vec<(u64, CultCacheEnvelope)> = envelopes + .iter() + .zip(&records) + .filter(|(_, stored)| stored.transaction_id != record.transaction_id) + .map(|(envelope, stored)| (stored.recorded_at_unix_millis, envelope.clone())) + .collect(); + kept.push((record.recorded_at_unix_millis, record.envelope()?)); + // Stable: records of one instant keep their stored order, so the record + // being written survives a tie. + kept.sort_by_key(|(recorded_at, _)| *recorded_at); + let excess = kept.len().saturating_sub(RUNNER_EVIDENCE_RECORDS); + let replacements: Vec<_> = kept + .into_iter() + .skip(excess) + .map(|(_, envelope)| envelope) + .collect(); + match exchange_behind_private_lock(path, &envelopes, &replacements)? { + TryCompareExchangeSnapshotOutcome::Exchanged => {} + TryCompareExchangeSnapshotOutcome::Mismatch => { + bail!("the runner evidence store changed while it was being written") + } + TryCompareExchangeSnapshotOutcome::LockContended => { + return Err(LockContended { + store: "runner evidence store", + } + .into()); + } + } + keep_private(path)?; + Ok(how) +} + +#[cfg(unix)] +fn keep_private(path: &Path) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + .context("making the runner evidence store private") +} + +#[cfg(not(unix))] +fn keep_private(_path: &Path) -> Result<()> { + Ok(()) +} + +/// The one reader. An absent store holds nothing. +pub(crate) fn for_transaction( + path: &Path, + transaction_id: &str, +) -> Result> { + Ok(snapshot(path)? + .1 + .into_iter() + .find(|stored| stored.transaction_id == transaction_id)) +} + +/// The record as `idunn status --command` shows it, as raw lines. They are +/// not text yet: `show_status` is the only caller, and `emit` the only +/// writer. +fn render(record: &RunnerFailureRecord) -> Vec> { + let noun = match record.subject_kind { + SubjectKind::Step => "step", + SubjectKind::ExternalInput => "external input", + }; + let mut lines: Vec> = vec![ + format!(" failed {noun} {}", record.subject_id).into_bytes(), + format!(" runner {}", record.runner).into_bytes(), + format!(" status {}", runner_status(record.exit_code, record.signal)).into_bytes(), + ]; + for (stream, tail, total) in [ + ("stdout", &record.stdout_tail, record.stdout_total_bytes), + ("stderr", &record.stderr_tail, record.stderr_total_bytes), + ] { + lines.push(format!(" {stream} (last {} of {total} bytes)", tail.len()).into_bytes()); + let text = tail.strip_suffix(b"\n").unwrap_or(tail); + if !text.is_empty() { + lines.extend(text.split(|byte| *byte == b'\n').map(|line| { + let mut shown = b" | ".to_vec(); + shown.extend_from_slice(line); + shown + })); + } + } + lines +} + +/// What `idunn status --command` prints for a transaction's runner evidence: +/// its record, or one line naming why the store is unreadable. Nothing is +/// written when the transaction has no record. +pub(crate) fn show_status( + out: &mut impl std::io::Write, + store: &Path, + transaction_id: &str, +) -> std::io::Result<()> { + let lines = match for_transaction(store, transaction_id) { + Ok(Some(record)) => render(&record), + Ok(None) => return Ok(()), + Err(error) => { + vec![format!(" runner evidence unreadable: {}", error.root_cause()).into_bytes()] + } + }; + emit(out, &lines) +} + +/// The one sink: every byte the status read prints from the evidence store, +/// or from an error about it, is written here and nowhere else. An allowlist: +/// printable ASCII and tab pass; every other code point is a visible +/// `\u{..}` escape and every invalid UTF-8 byte a `\x..` escape. A newline +/// inside a line is escaped too: this function writes the only ones. +pub(crate) fn emit(out: &mut impl std::io::Write, lines: &[Vec]) -> std::io::Result<()> { + for line in lines { + out.write_all(escape(line).as_bytes())?; + out.write_all(b"\n")?; + } + Ok(()) +} + +pub(crate) fn escape(bytes: &[u8]) -> String { + let mut shown = String::with_capacity(bytes.len()); + for chunk in bytes.utf8_chunks() { + for character in chunk.valid().chars() { + match character { + ' '..='~' | '\t' => shown.push(character), + _ => shown.push_str(&format!("\\u{{{:x}}}", u32::from(character))), + } + } + for byte in chunk.invalid() { + shown.push_str(&format!("\\x{byte:02x}")); + } + } + shown +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use std::os::unix::fs::PermissionsExt; + + fn failure(subject: RunnerSubject, stdout: &[u8], stderr: &[u8]) -> RunnerStepFailure { + RunnerStepFailure { + subject, + runner: "rust".into(), + exit_code: Some(3), + signal: None, + stdout_tail: stdout.to_vec(), + stdout_total_bytes: stdout.len() as u64, + stderr_tail: stderr.to_vec(), + stderr_total_bytes: stderr.len() as u64, + } + } + + /// What status writes for a record, through the one sink. + fn shown(record: &RunnerFailureRecord) -> String { + let mut written = Vec::new(); + emit(&mut written, &render(record)).expect("a Vec takes any write"); + String::from_utf8(written).expect("the sink writes ASCII only") + } + + fn shown_lines(record: &RunnerFailureRecord) -> Vec { + shown(record).lines().map(str::to_owned).collect() + } + + fn record_at(transaction: &str, now: u64) -> RunnerFailureRecord { + RunnerFailureRecord::from_failure( + transaction, + "up-service", + "service", + &failure(RunnerSubject::Step("build".into()), b"out\n", b"err\n"), + now, + ) + } + + #[test] + fn the_evidence_store_keeps_the_newest_32() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + // The literal, not the constant: the spec's bound is 32. + for index in 0..=32 { + record(&path, &record_at(&format!("tx-{index:02}"), 1000 + index as u64))?; + } + let kept: Vec<_> = snapshot(&path)?.1.into_iter().map(|stored| stored.transaction_id).collect(); + assert_eq!(kept.len(), 32); + assert!(!kept.contains(&"tx-00".to_owned()), "the oldest survived"); + assert!(kept.contains(&"tx-32".to_owned()), "the newest was evicted"); + assert!(kept.contains(&"tx-01".to_owned()), "the second oldest was evicted early"); + // Rewriting a resident transaction replaces it and drops nothing. + let mut again = record_at("tx-05", 5000); + again.stdout_tail = b"rewritten\n".to_vec(); + again.stdout_total_bytes = 10; + record(&path, &again)?; + let kept = snapshot(&path)?.1; + assert_eq!(kept.len(), 32); + assert!(kept.iter().any(|stored| stored.transaction_id == "tx-01")); + let stored = for_transaction(&path, "tx-05")?.context("rewritten record is gone")?; + assert_eq!(stored.stdout_tail, b"rewritten\n"); + assert_eq!(for_transaction(&path, "tx-00")?, None); + Ok(()) + } + + #[test] + fn a_record_round_trips_byte_for_byte() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + let written = RunnerFailureRecord::from_failure( + "tx-a", + "up-service", + "service", + &failure(RunnerSubject::ExternalInput("toolchain".into()), &[0, 1, 255, 10], b""), + 77, + ); + record(&path, &written)?; + assert_eq!(for_transaction(&path, "tx-a")?, Some(written)); + assert_eq!(for_transaction(&path, "tx-b")?, None); + Ok(()) + } + + #[test] + fn the_evidence_store_is_private() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + // Whatever the umask, the mode is the store's own rule, on every write. + for index in 0..2 { + record(&path, &record_at(&format!("tx-{index}"), 10 + index))?; + assert_eq!(std::fs::metadata(&path)?.permissions().mode() & 0o777, 0o600); + } + Ok(()) + } + + #[test] + fn a_tail_over_its_bound_or_its_stream_is_refused() { + let mut oversize = record_at("tx-a", 1); + oversize.stdout_tail = vec![b'x'; RUNNER_OUTPUT_TAIL_BYTES + 1]; + oversize.stdout_total_bytes = u64::MAX; + assert!(oversize.validate().is_err()); + let mut impossible = record_at("tx-a", 1); + impossible.stderr_total_bytes = 1; + assert!(impossible.validate().is_err()); + let mut exact = record_at("tx-a", 1); + exact.stdout_tail = vec![b'x'; RUNNER_OUTPUT_TAIL_BYTES]; + exact.stdout_total_bytes = RUNNER_OUTPUT_TAIL_BYTES as u64; + assert!(exact.validate().is_ok()); + } + + #[test] + fn status_renders_the_tail_with_control_bytes_escaped() { + let mut stored = record_at("tx-a", 1); + stored.stdout_tail = b"first\n\x1b[2Jsecond\x00end\tTab\nlast\n".to_vec(); + stored.stdout_total_bytes = 5000; + stored.stderr_tail = b"bad\r\x07".to_vec(); + stored.stderr_total_bytes = 6; + let lines = shown_lines(&stored); + let text = lines.join("\n"); + assert!(!text.contains('\u{1b}') && !text.contains('\0') && !text.contains('\r')); + assert!(!text.contains('\u{7}')); + assert_eq!( + lines, + vec![ + " failed step build", + " runner rust", + " status exit status 3", + " stdout (last 30 of 5000 bytes)", + " | first", + " | \\u{1b}[2Jsecond\\u{0}end\tTab", + " | last", + " stderr (last 5 of 6 bytes)", + " | bad\\u{d}\\u{7}", + ] + ); + } + + #[test] + fn debug_names_the_record_and_prints_no_tail() { + let mut stored = record_at("tx-a", 1); + stored.stdout_tail = b"SECRET-OUTPUT".to_vec(); + stored.stdout_total_bytes = 13; + let debug = format!("{stored:?}"); + assert!(debug.contains("tx-a") && !debug.contains("SECRET"), "{debug}"); + } + + #[test] + fn an_empty_tail_renders_only_its_header() { + let mut stored = record_at("tx-a", 1); + stored.stdout_tail = Vec::new(); + stored.stdout_total_bytes = 0; + let lines = shown_lines(&stored); + assert!(lines.contains(&" stdout (last 0 of 0 bytes)".to_owned())); + assert!(!lines.iter().any(|line| line == " | ")); + } + + #[test] + fn the_reader_names_an_unreadable_store_as_an_error() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + std::fs::write(&path, b"\xc1 not a store")?; + assert!(for_transaction(&path, "tx-a").is_err()); + Ok(()) + } + + fn files_beside(path: &Path) -> Result> { + let mut names: Vec = std::fs::read_dir(path.parent().context("no parent")?)? + .map(|entry| Ok(entry?.file_name().to_string_lossy().into_owned())) + .collect::>()?; + names.retain(|name| name.starts_with("runner-evidence") && !name.contains("lock")); + names.sort(); + Ok(names) + } + + /// A store a newer Idunn wrote: well-formed, but a schema this one does not know. + fn write_future_store(path: &Path) -> Result<()> { + let mut future = record_at("tx-future", 1); + future.schema_version = "idunn.runner_failure.v2".into(); + let envelope = typed_envelope( + &future.transaction_id, + RunnerFailureRecord::TYPE, + "idunn.runner_failure.v2", + &future, + 1, + )?; + let outcome = exchange_behind_private_lock(path, &[], &[envelope])?; + ensure!( + matches!(outcome, TryCompareExchangeSnapshotOutcome::Exchanged), + "the future store was not written" + ); + Ok(()) + } + + #[test] + fn a_corrupt_store_is_set_aside_and_the_next_failure_keeps_its_evidence() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + let damage = b"\xc1 not a store \x00 damage"; + std::fs::write(&path, damage)?; + let kept = record(&path, &record_at("tx-a", 5))?; + assert_eq!(kept, Kept::SetAside { replaced: None }); + // Kept, not deleted: the damage is where the journal says, root-only. + let aside = unreadable_store_path(&path); + assert_eq!(std::fs::read(&aside)?, damage); + assert_eq!(std::fs::metadata(&aside)?.permissions().mode() & 0o777, 0o600); + // The failure's evidence is in a fresh, private store. + assert_eq!( + for_transaction(&path, "tx-a")?.map(|stored| stored.transaction_id), + Some("tx-a".to_owned()) + ); + assert_eq!(std::fs::metadata(&path)?.permissions().mode() & 0o777, 0o600); + // A healthy store is not set aside again. + assert_eq!(record(&path, &record_at("tx-b", 6))?, Kept::Stored); + assert_eq!(std::fs::read(&aside)?, damage); + Ok(()) + } + + #[test] + fn a_future_version_store_is_set_aside_and_the_next_failure_keeps_its_evidence() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + write_future_store(&path)?; + let before = std::fs::read(&path)?; + assert!(for_transaction(&path, "tx-future").is_err()); + assert_eq!( + record(&path, &record_at("tx-a", 5))?, + Kept::SetAside { replaced: None } + ); + assert_eq!(std::fs::read(unreadable_store_path(&path))?, before); + assert!(for_transaction(&path, "tx-a")?.is_some()); + assert_eq!(for_transaction(&path, "tx-future")?, None); + Ok(()) + } + + #[test] + fn at_most_one_unreadable_store_is_kept() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + let damages: [&[u8]; 3] = [b"\xc1 first", b"\xc1 second", b"\xc1 third"]; + for (index, damage) in damages.into_iter().enumerate() { + std::fs::write(&path, damage)?; + record(&path, &record_at(&format!("tx-{index}"), 5 + index as u64))?; + assert_eq!( + std::fs::read(unreadable_store_path(&path))?, + damage, + "the newest damage is the one kept" + ); + } + assert_eq!( + files_beside(&path)?, + vec![ + "runner-evidence.cc".to_owned(), + "runner-evidence.unreadable.cc".to_owned() + ] + ); + Ok(()) + } + + #[test] + fn a_store_that_cannot_be_read_is_left_in_place_and_the_write_fails() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + // A directory where the store should be: unreadable, not undecodable. + std::fs::create_dir(&path)?; + assert!(record(&path, &record_at("tx-a", 1)).is_err()); + assert!(path.is_dir()); + assert!(!unreadable_store_path(&path).exists()); + Ok(()) + } + + #[test] + fn every_code_point_but_printable_ascii_is_a_visible_escape() { + // The allowlist, exhaustively: no class list decides anything. + for code in 0..=0x10ffffu32 { + let Some(character) = char::from_u32(code) else { + continue; + }; + let text = character.to_string(); + let shown = escape(text.as_bytes()); + if matches!(character, ' '..='~' | '\t') { + assert_eq!(shown, text, "U+{code:04X} should pass"); + } else { + assert_eq!(shown, format!("\\u{{{code:x}}}"), "U+{code:04X}"); + } + } + // Invalid UTF-8 is escaped byte by byte; valid text round it passes. + assert_eq!(escape(b"a\xc1\xffb\xe2\x82"), "a\\xc1\\xffb\\xe2\\x82"); + assert_eq!(escape(b"ok \xe2\x82\xac"), "ok \\u{20ac}"); + } + + #[test] + fn a_newline_inside_a_line_cannot_forge_a_line() { + let mut stored = record_at("tx-a", 1); + stored.subject_id = "build\n status exit status 0".into(); + let lines = shown_lines(&stored); + assert_eq!(lines[0], " failed step build\\u{a} status exit status 0"); + assert_eq!( + lines.iter().filter(|line| line.starts_with(" status ")).count(), + 1 + ); + } + + /// A store whose first record is well-formed except that its subject kind + /// holds `marker`: the decode error quotes it. + fn store_with_hostile_subject_kind(path: &Path, marker: &[u8; 4]) -> Result<()> { + record(path, &record_at("tx-a", 1))?; + let mut bytes = std::fs::read(path)?; + let at = bytes + .windows(4) + .position(|window| window == b"step") + .context("no step marker")?; + bytes[at..at + 4].copy_from_slice(marker); + std::fs::write(path, bytes)?; + Ok(()) + } + + #[test] + fn the_unreadable_store_line_is_escaped_with_the_rest() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + store_with_hostile_subject_kind(&path, b"\x1b[2J")?; + let mut written = Vec::new(); + show_status(&mut written, &path, "tx-a")?; + assert!(!written.contains(&0x1b), "an escape byte reached the terminal"); + assert!(written.iter().all(|byte| matches!(byte, b' '..=b'~' | b'\t' | b'\n'))); + let text = String::from_utf8(written)?; + assert!(text.starts_with(" runner evidence unreadable: "), "{text}"); + assert!(text.contains("\\u{1b}[2J"), "the quoted value was not shown escaped: {text}"); + Ok(()) + } + + #[test] + fn a_failed_set_aside_names_itself_keeps_the_store_and_echoes_nothing() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + store_with_hostile_subject_kind(&path, b"CNRY")?; + let damage = std::fs::read(&path)?; + // Sanity: this damage decodes to an error that quotes the canary. + assert!(format!("{:#}", snapshot(&path).unwrap_err()).contains("CNRY")); + // A non-empty directory where the aside goes: rename fails even as root. + let aside = unreadable_store_path(&path); + std::fs::create_dir(&aside)?; + std::fs::write(aside.join("occupied"), b"x")?; + let error = record(&path, &record_at("tx-b", 2)).unwrap_err(); + for text in [format!("{error:#}"), format!("{error:?}")] { + assert!(text.contains("could not be set aside"), "{text}"); + assert!(!text.contains("CNRY"), "stored content reached the error: {text}"); + } + // The failure cost nothing: the store is as it was, with no evidence + // written over it. + assert_eq!(std::fs::read(&path)?, damage); + assert!(aside.join("occupied").exists()); + Ok(()) + } + + #[test] + fn an_io_error_leaves_the_store_in_place_and_keeps_no_evidence() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + record(&path, &record_at("tx-a", 1))?; + let before = std::fs::read(&path)?; + let denied = || std::io::Error::from(std::io::ErrorKind::PermissionDenied); + // The first read fails on a healthy store. + let error = record_reading(&path, &record_at("tx-b", 2), &|_| Err(denied())).unwrap_err(); + assert!(format!("{error:#}").contains("permission denied"), "{error:#}"); + assert_eq!(std::fs::read(&path)?, before, "the store changed"); + assert!(!unreadable_store_path(&path).exists()); + assert!(for_transaction(&path, "tx-b")?.is_none()); + // The second read, the one that confirms the bytes do not decode, fails. + std::fs::write(&path, b"\xc1 not a store")?; + let calls = std::cell::Cell::new(0usize); + let read = |path: &Path| { + calls.set(calls.get() + 1); + if calls.get() == 2 { + return Err(denied()); + } + std::fs::read(path) + }; + let error = record_reading(&path, &record_at("tx-b", 2), &read).unwrap_err(); + assert!(format!("{error:#}").contains("permission denied"), "{error:#}"); + assert_eq!(std::fs::read(&path)?, b"\xc1 not a store"); + assert!(!unreadable_store_path(&path).exists()); + // Bytes that changed between the reads are no verdict either. + let calls = std::cell::Cell::new(0usize); + let read = |path: &Path| -> std::io::Result> { + calls.set(calls.get() + 1); + let mut bytes = std::fs::read(path)?; + bytes.push(calls.get() as u8); + Ok(bytes) + }; + assert!(record_reading(&path, &record_at("tx-b", 2), &read).is_err()); + assert_eq!(std::fs::read(&path)?, b"\xc1 not a store"); + assert!(!unreadable_store_path(&path).exists()); + Ok(()) + } + + #[test] + fn a_second_set_aside_says_it_replaced_the_first() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + std::fs::write(&path, b"\xc1 first")?; + assert_eq!( + record(&path, &record_at("tx-a", 1))?, + Kept::SetAside { replaced: None } + ); + std::fs::write(&path, b"\xc1 second")?; + let Kept::SetAside { replaced: Some(_) } = record(&path, &record_at("tx-b", 2))? else { + bail!("a displaced set-aside store went unreported"); + }; + assert_eq!(std::fs::read(unreadable_store_path(&path))?, b"\xc1 second"); + Ok(()) + } +} diff --git a/src/steward_orders.rs b/src/steward_orders.rs new file mode 100644 index 0000000..d4be1ee --- /dev/null +++ b/src/steward_orders.rs @@ -0,0 +1,4079 @@ +//! Typed orders from Idunn's steward. +//! +//! `idunn orders serve` is a second root unit from the same binary as +//! `idunn serve`, with its own store (`orders.cc`) and socket. It alone mints +//! order ids, decides acceptance and refusal, actuates, and records outcomes. +//! The survival daemon takes no part: `control.cc` never learns an order type. +//! +//! An order is a closed enum, never a command string: install packages, act on +//! a systemd unit, or deploy a target through the same commit primitive +//! `idunn up` uses. Every unit is allowed except the always-refused set, which +//! is compiled in here and read from nowhere: no binding field, record or flag +//! holds it. The `#operator` post is a retried projection of the store. +//! +//! Lifecycle, one row per step: ISSUE a frame on the socket; AUTHENTICATE the +//! peer's uid; VALIDATE the grammar (a malformed order is a typed refusal and +//! no record); ACCEPT a `steward_order`; SCOPE the always-refused check; BRAKE +//! `mind-control check`; START record before actuation, at most once; RUN; +//! RESULT outcome; POST accepted and result. +//! +//! No refusal or error here carries a value the issuer supplied: it names the +//! field and a code. A refused unit that a record names was resolved by +//! systemd and passed the unit grammar. + +use std::fmt; +use std::io::{Read, Write}; +use std::os::unix::fs::{MetadataExt, PermissionsExt}; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::os::unix::process::ExitStatusExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::Arc; +use std::sync::mpsc; +use std::thread; +use std::time::Duration; + +use anyhow::{Context, Result, anyhow, bail, ensure}; +use cultcache_rs::{ + CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, +}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::control_plane::{ + command_state, commit_deployment_command, decode_record, default_state_store, now_millis, + path_value, require_id, require_selector, string_value, typed_envelope, +}; +use crate::runner_evidence::{emit, escape}; + +const ORDER_SCHEMA: &str = "idunn.steward_order.v1"; +const START_SCHEMA: &str = "idunn.steward_order_start.v1"; +const OUTCOME_SCHEMA: &str = "idunn.steward_order_outcome.v1"; +const POST_SCHEMA: &str = "idunn.steward_order_post.v1"; + +const REQUEST_FRAME_LIMIT: usize = 16 * 1024; +const RESPONSE_FRAME_LIMIT: usize = 256 * 1024; +const TAIL_BYTES: usize = 8 * 1024; +const REASON_LIMIT: usize = 300; +const PACKAGE_LIMIT: usize = 16; +const UNIT_LIMIT: usize = 200; +const LIST_LIMIT: usize = 50; +const UNIT_SUFFIXES: [&str; 5] = [".service", ".socket", ".timer", ".path", ".mount"]; + +/// The always-refused set: Idunn's own units and the minds' own. Matched on +/// every name a unit answers to, ASCII case-insensitively (systemd units are +/// case-sensitive, so the fold only widens the refusal). +fn is_refused_name(name: &str) -> bool { + let name = name.to_ascii_lowercase(); + ["idunn-", "mind-self@", "mind-persona@"] + .iter() + .any(|prefix| name.starts_with(prefix)) +} + +/// Units whose binaries suspend, hibernate or open recovery: no property +/// reveals it, so they are refused by name, on every name they answer to. +const HOST_NAMES: [&str; 6] = [ + "systemd-suspend.service", + "systemd-hibernate.service", + "systemd-hybrid-sleep.service", + "systemd-suspend-then-hibernate.service", + "rescue.service", + "emergency.service", +]; + +fn is_host_name(name: &str) -> bool { + HOST_NAMES.iter().any(|host| name.eq_ignore_ascii_case(host)) +} + +// ---------------------------------------------------------------- grammar -- + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Code { + Empty, + TooLong, + TooMany, + BadCharacter, + BadShape, + Malformed, + Unauthorized, + Unknown, + Unavailable, +} + +/// A refusal names the field and a code. It never carries the offending value. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct Refusal { + pub(crate) field: String, + pub(crate) code: Code, +} + +impl Refusal { + fn new(field: &'static str, code: Code) -> Self { + Self { + field: field.to_owned(), + code, + } + } +} + +impl fmt::Display for Refusal { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{} refused: {:?}", self.field, self.code) + } +} + +impl std::error::Error for Refusal {} + +fn validate_package(spec: &str) -> Result<(), Refusal> { + const FIELD: &str = "package"; + if spec.is_empty() { + return Err(Refusal::new(FIELD, Code::Empty)); + } + if spec.len() > UNIT_LIMIT { + return Err(Refusal::new(FIELD, Code::TooLong)); + } + let (name, version) = match spec.split_once('=') { + Some((name, version)) => (name, Some(version)), + None => (spec, None), + }; + let bytes = name.as_bytes(); + if bytes.len() < 2 { + return Err(Refusal::new(FIELD, Code::BadShape)); + } + let head = bytes[0].is_ascii_lowercase() || bytes[0].is_ascii_digit(); + let rest = bytes[1..].iter().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'+' | b'.' | b'-') + }); + if !head || !rest { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + // apt reads `name-` as a removal and `name.deb` as a local file. + if name.ends_with('-') || name.ends_with(".deb") { + return Err(Refusal::new(FIELD, Code::BadShape)); + } + if let Some(version) = version { + if version.is_empty() { + return Err(Refusal::new(FIELD, Code::BadShape)); + } + if !version.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'+' | b'~' | b':' | b'-') + }) { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + } + Ok(()) +} + +fn validate_unit(unit: &str) -> Result<(), Refusal> { + const FIELD: &str = "unit"; + if unit.is_empty() { + return Err(Refusal::new(FIELD, Code::Empty)); + } + if unit.len() > UNIT_LIMIT { + return Err(Refusal::new(FIELD, Code::TooLong)); + } + let allowed = unit.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b':' | b'_' | b'.' | b'@' | b'-') + }); + if !allowed || unit.starts_with('-') { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + match UNIT_SUFFIXES + .iter() + .find_map(|suffix| unit.strip_suffix(suffix)) + { + Some(stem) if !stem.is_empty() => Ok(()), + _ => Err(Refusal::new(FIELD, Code::BadShape)), + } +} + +fn validate_reason(reason: &str) -> Result<(), Refusal> { + const FIELD: &str = "reason"; + if reason.is_empty() { + return Err(Refusal::new(FIELD, Code::Empty)); + } + if reason.len() > REASON_LIMIT { + return Err(Refusal::new(FIELD, Code::TooLong)); + } + if !reason.bytes().all(|byte| (0x20..=0x7e).contains(&byte)) { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + Ok(()) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Verb { + Start, + Stop, + Restart, + Reload, +} + +impl Verb { + fn as_str(self) -> &'static str { + match self { + Verb::Start => "start", + Verb::Stop => "stop", + Verb::Restart => "restart", + Verb::Reload => "reload", + } + } + + fn parse(word: &str) -> Option { + [Verb::Start, Verb::Stop, Verb::Restart, Verb::Reload] + .into_iter() + .find(|verb| verb.as_str() == word) + } +} + +/// The closed set of things an order can ask for. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Operation { + PackageInstall { packages: Vec }, + UnitAction { verb: Verb, unit: String }, + Deploy { selector: String }, +} + +/// One fixed argv for `systemd-run`, built from the typed operation. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Step { + pub(crate) argv: Vec, + pub(crate) environment: Vec<(String, String)>, +} + +impl Operation { + fn validate(&self) -> Result<(), Refusal> { + match self { + Operation::PackageInstall { packages } => { + if packages.is_empty() { + return Err(Refusal::new("package", Code::Empty)); + } + if packages.len() > PACKAGE_LIMIT { + return Err(Refusal::new("package", Code::TooMany)); + } + packages.iter().try_for_each(|package| validate_package(package)) + } + Operation::UnitAction { unit, .. } => validate_unit(unit), + Operation::Deploy { selector } => require_selector(selector) + .map_err(|_| Refusal::new("selector", Code::BadShape)), + } + } + + fn steps(&self) -> Vec { + let words = |words: &[&str]| words.iter().map(|word| (*word).to_owned()).collect(); + let apt_environment = vec![("DEBIAN_FRONTEND".to_owned(), "noninteractive".to_owned())]; + match self { + Operation::PackageInstall { packages } => { + // Never a removal, never a pattern: each name selects exactly + // itself, and apt aborts rather than displace an installed package. + let mut install: Vec = + words(&["apt-get", "install", "-y", "-q", "--no-remove", "--"]); + let mut present: Vec = words(&["dpkg", "-s", "--"]); + for package in packages { + let (name, version) = package.split_once('=').unwrap_or((package, "")); + install.push(match version { + "" => format!("?exact-name({name})"), + version => format!("?exact-name({name})={version}"), + }); + present.push(name.to_owned()); + } + // An exact name that matches nothing installs nothing and exits 0, + // so the last step fails unless every named package is installed. + vec![ + Step { + argv: words(&["apt-get", "update", "-q"]), + environment: apt_environment.clone(), + }, + Step { + argv: install, + environment: apt_environment, + }, + Step { + argv: present, + environment: Vec::new(), + }, + ] + } + Operation::UnitAction { verb, unit } => vec![Step { + argv: vec![ + "systemctl".into(), + verb.as_str().into(), + "--job-mode=ignore-requirements".into(), + "--".into(), + unit.clone(), + ], + environment: Vec::new(), + }], + Operation::Deploy { .. } => Vec::new(), + } + } + + fn describe(&self) -> String { + match self { + Operation::PackageInstall { packages } => { + format!("package-install {}", packages.join(" ")) + } + Operation::UnitAction { verb, unit } => format!("unit {} {unit}", verb.as_str()), + Operation::Deploy { selector } => format!("deploy {selector}"), + } + } +} + +// ---------------------------------------------------------------- records -- + +#[derive(Clone, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "idunn.steward_order", schema = "idunn.steward_order.v1")] +pub(crate) struct OrderRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) order_id: String, + #[cultcache(key = 2)] + pub(crate) principal: String, + #[cultcache(key = 3)] + pub(crate) issuer_uid: u32, + #[cultcache(key = 4)] + pub(crate) issuer_unit: String, + #[cultcache(key = 5)] + pub(crate) issuer_invocation: String, + #[cultcache(key = 6)] + pub(crate) issued_at_unix_millis: u64, + #[cultcache(key = 7)] + pub(crate) operation: Operation, + #[cultcache(key = 8)] + pub(crate) reason: String, +} + +impl fmt::Debug for OrderRecord { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OrderRecord") + .field("order_id", &self.order_id) + .finish_non_exhaustive() + } +} + +#[derive(Clone, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.steward_order_start", + schema = "idunn.steward_order_start.v1" +)] +pub(crate) struct StartRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) order_id: String, + #[cultcache(key = 2)] + pub(crate) started_at_unix_millis: u64, + /// The transient unit name, or the deployment command id. + #[cultcache(key = 3)] + pub(crate) actuation: String, +} + +impl fmt::Debug for StartRecord { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("StartRecord") + .field("order_id", &self.order_id) + .finish_non_exhaustive() + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Via { + Name, + Alias, + Host, + OnEnd, + Bound, + Upholds, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Refused { + AlwaysRefused { unit: String, via: Via }, + /// The unit only wraps `member` (`ConsistsOf=`): a stop, restart or reload + /// of the wrapper exits 0 and leaves the member alone. + Wrapper { member: String }, + Braked, + /// The order's principal or issuing uid is not the steward bound now. + NotSteward, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Outcome { + Ran, + Failed { + exit_code: Option, + signal: Option, + }, + Refused(Refused), + Lost, + Submitted { command_id: String }, +} + +impl Outcome { + fn describe(&self) -> String { + match self { + Outcome::Ran => "ran (exit 0)".into(), + Outcome::Failed { + exit_code: Some(code), + .. + } => format!("failed (exit {code})"), + Outcome::Failed { + signal: Some(signal), + .. + } => format!("failed (signal {signal})"), + Outcome::Failed { .. } => "failed (not run to an exit)".into(), + Outcome::Refused(Refused::AlwaysRefused { unit, via }) => format!( + "refused: always-refused unit {} reached by {}", + escape(unit.as_bytes()), + match via { + Via::Name => "name", + Via::Alias => "alias", + Via::Host => "host", + Via::OnEnd => "on-end", + Via::Bound => "bound", + Via::Upholds => "upholds", + } + ), + Outcome::Refused(Refused::Wrapper { member }) => format!( + "refused: wrapper unit, it only wraps {}", + escape(member.as_bytes()) + ), + Outcome::Refused(Refused::Braked) => "refused: braked".into(), + Outcome::Refused(Refused::NotSteward) => { + "refused: the issuer is not the current steward".into() + } + Outcome::Lost => "lost: the orders service restarted mid-run".into(), + Outcome::Submitted { command_id } => { + format!("submitted deployment {}", escape(command_id.as_bytes())) + } + } + } +} + +#[derive(Clone, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.steward_order_outcome", + schema = "idunn.steward_order_outcome.v1" +)] +pub(crate) struct OutcomeRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) order_id: String, + #[cultcache(key = 2)] + pub(crate) ended_at_unix_millis: u64, + #[cultcache(key = 3)] + pub(crate) outcome: Outcome, + #[cultcache(key = 4)] + pub(crate) stdout_tail: Vec, + #[cultcache(key = 5)] + pub(crate) stderr_tail: Vec, +} + +/// Tails stay out of `Debug`: they are raw program output. +impl fmt::Debug for OutcomeRecord { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OutcomeRecord") + .field("order_id", &self.order_id) + .field("outcome", &self.outcome) + .finish_non_exhaustive() + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum PostKind { + Accepted, + Result, +} + +impl PostKind { + fn as_str(self) -> &'static str { + match self { + PostKind::Accepted => "accepted", + PostKind::Result => "result", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.steward_order_post", + schema = "idunn.steward_order_post.v1" +)] +pub(crate) struct PostRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + /// `:`. + #[cultcache(key = 1)] + pub(crate) post_key: String, + #[cultcache(key = 2)] + pub(crate) event_id: String, + #[cultcache(key = 3)] + pub(crate) posted_at_unix_millis: u64, +} + +// ------------------------------------------------------------------ store -- + +/// `orders.cc`: root-only, written by `idunn orders serve` alone. +pub(crate) struct OrderStore { + directory: PathBuf, +} + +#[derive(Default)] +struct Ledger { + orders: Vec, + starts: Vec, + outcomes: Vec, + posts: Vec, +} + +impl Ledger { + fn start_of(&self, order_id: &str) -> Option<&StartRecord> { + self.starts.iter().find(|start| start.order_id == order_id) + } + + fn outcome_of(&self, order_id: &str) -> Option<&OutcomeRecord> { + self.outcomes + .iter() + .find(|outcome| outcome.order_id == order_id) + } + + fn posted(&self, order_id: &str, kind: PostKind) -> bool { + let key = post_key(order_id, kind); + self.posts.iter().any(|post| post.post_key == key) + } +} + +fn post_key(order_id: &str, kind: PostKind) -> String { + format!("{order_id}:{}", kind.as_str()) +} + +impl OrderStore { + pub(crate) fn new(directory: &Path) -> Self { + Self { + directory: directory.to_owned(), + } + } + + fn path(&self) -> PathBuf { + self.directory.join("orders.cc") + } + + fn prepare(&self) -> Result<()> { + std::fs::create_dir_all(&self.directory).context("creating the orders directory")?; + std::fs::set_permissions(&self.directory, std::fs::Permissions::from_mode(0o700)) + .context("making the orders directory private") + } + + /// Insert a record whose (type, key) is absent. `false` means it was there. + fn put(&self, key: &str, schema: &str, value: &T) -> Result { + self.prepare()?; + let path = self.path(); + let now = now_millis()?; + let inserted = SingleFileMessagePackBackingStore::new(&path).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: T::TYPE.into(), + key: key.into(), + current: None, + }], + &[typed_envelope(key, T::TYPE, schema, value, now)?], + )?; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)) + .context("making the orders store private")?; + Ok(inserted) + } + + fn ledger(&self) -> Result { + let path = self.path(); + let mut ledger = Ledger::default(); + if !path.exists() { + return Ok(ledger); + } + let envelopes = SingleFileMessagePackBackingStore::new(&path) + .pull_all_read_only_snapshot() + .context("reading the orders store")?; + for envelope in &envelopes { + let schema = envelope.schema_id.as_deref(); + let key = envelope.key.as_str(); + match envelope.r#type.as_str() { + OrderRecord::TYPE if schema == Some(ORDER_SCHEMA) => { + let record: OrderRecord = decode_record(envelope)?; + ensure!(record.order_id == key, "an order is stored under another key"); + ledger.orders.push(record); + } + StartRecord::TYPE if schema == Some(START_SCHEMA) => { + let record: StartRecord = decode_record(envelope)?; + ensure!(record.order_id == key, "a start is stored under another key"); + ledger.starts.push(record); + } + OutcomeRecord::TYPE if schema == Some(OUTCOME_SCHEMA) => { + let record: OutcomeRecord = decode_record(envelope)?; + ensure!(record.order_id == key, "an outcome is stored under another key"); + ledger.outcomes.push(record); + } + PostRecord::TYPE if schema == Some(POST_SCHEMA) => { + let record: PostRecord = decode_record(envelope)?; + ensure!(record.post_key == key, "a post is stored under another key"); + ledger.posts.push(record); + } + _ => bail!("the orders store holds a document that is not an order record"), + } + } + ledger + .orders + .sort_by_key(|order| (order.issued_at_unix_millis, order.order_id.clone())); + Ok(ledger) + } +} + +// ------------------------------------------------------------------ ports -- + +/// The properties naming what systemd does when a unit ends or its job times +/// out: each holds an emergency action (`reboot`, `poweroff-force`, ...). +const ACTIONS: [&str; 4] = ["SuccessAction", "FailureAction", "StartLimitAction", "JobTimeoutAction"]; + +/// The properties naming units systemd starts when a unit ends. +const ON_END: [&str; 2] = ["OnFailure", "OnSuccess"]; + +/// What `systemctl show` says about one unit. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct UnitFacts { + pub(crate) id: String, + pub(crate) names: Vec, + pub(crate) not_found: bool, + /// The properties that make the unit act when it ends: each emergency + /// action other than `none`, and each `OnFailure=` and `OnSuccess=` unit. + pub(crate) on_end: Vec, + /// The units bound to this one at runtime (`BoundBy=`): systemd stops + /// them when this unit stops, whatever the job mode. + pub(crate) bound_by: Vec, + /// The units this one upholds (`Upholds=`): systemd starts them in a full + /// transaction whenever this unit is active, whatever the job mode. + pub(crate) upholds: Vec, + /// The units this one groups (`ConsistsOf=`): the verbs reach them only + /// through propagation, which the job mode removes. + pub(crate) consists_of: Vec, +} + +impl UnitFacts { + fn identities(&self) -> impl Iterator { + std::iter::once(&self.id).chain(self.names.iter()) + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct RunReport { + pub(crate) exit_code: Option, + pub(crate) signal: Option, + pub(crate) stdout_tail: Vec, + pub(crate) stderr_tail: Vec, +} + +pub(crate) trait Systemd: Send + Sync { + fn show(&self, unit: &str) -> Result; + fn invocation(&self, unit: &str) -> Option; + /// Run one step as the transient unit `unit`, to its end. + fn run(&self, unit: &str, step: &Step) -> Result; +} + +pub(crate) struct Peer { + pub(crate) uid: u32, + pub(crate) issuer_unit: String, +} + +pub(crate) trait PeerCredentials: Send + Sync { + fn peer(&self, stream: &UnixStream) -> Result; +} + +pub(crate) trait BrakeCheck: Send + Sync { + /// `true` only when the instance's brake check exits 0. + fn clear(&self, principal: &str) -> bool; +} + +pub(crate) trait Homeserver: Send + Sync { + /// Idempotent by `txn_id`; returns the event id. + fn post(&self, txn_id: &str, text: &str) -> Result; +} + +pub(crate) struct Ctx { + pub(crate) store: OrderStore, + pub(crate) state_store: PathBuf, + pub(crate) steward_uid: u32, + pub(crate) principal: String, + pub(crate) systemd: Arc, + pub(crate) peers: Arc, + pub(crate) brake: Arc, + pub(crate) homeserver: Arc, +} + +// ------------------------------------------------------------------ scope -- + +fn refused(unit: &str, via: Via) -> Refused { + Refused::AlwaysRefused { + unit: unit.to_owned(), + via, + } +} + +/// Would `verb` on `unit` be refused. systemd builds the transaction and the +/// order tells it to build one job (`--job-mode=ignore-requirements`), so this +/// reads only what that job mode cannot remove from the one named unit: the +/// name it answers to, what fires when it ends, who is bound to it, what it upholds and +/// what it only wraps. +fn refusal_for(systemd: &dyn Systemd, unit: &str, verb: Verb) -> Result> { + if is_refused_name(unit) { + return Ok(Some(refused(unit, Via::Name))); + } + let facts = systemd.show(unit)?; + ensure!(!facts.not_found, "systemd has no such unit to check"); + for name in facts.identities() { + if is_refused_name(name) { + return Ok(Some(refused(name, Via::Alias))); + } + if is_host_name(name) { + return Ok(Some(refused(name, Via::Host))); + } + } + if verb != Verb::Reload && !facts.on_end.is_empty() { + return Ok(Some(refused(unit, Via::OnEnd))); + } + if matches!(verb, Verb::Stop | Verb::Restart) + && let Some(bound) = facts.bound_by.first() + { + return Ok(Some(refused(bound, Via::Bound))); + } + if matches!(verb, Verb::Start | Verb::Restart) && !facts.upholds.is_empty() { + return Ok(Some(refused(unit, Via::Upholds))); + } + if verb != Verb::Start + && let Some(member) = facts.consists_of.first() + { + return Ok(Some(Refused::Wrapper { + member: member.clone(), + })); + } + Ok(None) +} + +// -------------------------------------------------------------- execution -- + +fn tail(bytes: &[u8]) -> Vec { + bytes[bytes.len().saturating_sub(TAIL_BYTES)..].to_vec() +} + +fn conclude( + ctx: &Ctx, + order_id: &str, + outcome: Outcome, + stdout_tail: &[u8], + stderr_tail: &[u8], +) -> Result<()> { + ctx.store.put( + order_id, + OUTCOME_SCHEMA, + &OutcomeRecord { + schema_version: OUTCOME_SCHEMA.into(), + order_id: order_id.into(), + ended_at_unix_millis: now_millis()?, + outcome, + stdout_tail: tail(stdout_tail), + stderr_tail: tail(stderr_tail), + }, + )?; + Ok(()) +} + +fn order_uuid(order_id: &str) -> &str { + order_id.strip_prefix("ord-").unwrap_or(order_id) +} + +pub(crate) fn requester(principal: &str, order_id: &str) -> String { + format!("steward:{principal} order {order_id}") +} + +/// Run one accepted order. A start record is written before anything is +/// actuated and an order that has one is never run again. +pub(crate) fn execute(ctx: &Ctx, order_id: &str) -> Result<()> { + let ledger = ctx.store.ledger()?; + let order = ledger + .orders + .iter() + .find(|order| order.order_id == order_id) + .context("the order is not in the store")?; + if ledger.start_of(order_id).is_some() || ledger.outcome_of(order_id).is_some() { + return Ok(()); + } + // One gate for a fresh order and a recovered one: who may order is what the + // binding says now, not what it said when the order was accepted. + if order.principal != ctx.principal || order.issuer_uid != ctx.steward_uid { + return conclude(ctx, order_id, Outcome::Refused(Refused::NotSteward), b"", b""); + } + if order.operation.validate().is_err() { + return conclude( + ctx, + order_id, + Outcome::Failed { + exit_code: None, + signal: None, + }, + b"", + b"the stored order no longer satisfies the grammar", + ); + } + if let Operation::UnitAction { verb, unit } = &order.operation { + match refusal_for(ctx.systemd.as_ref(), unit, *verb) { + Ok(None) => {} + Ok(Some(hit)) => return conclude(ctx, order_id, Outcome::Refused(hit), b"", b""), + Err(_) => { + return conclude( + ctx, + order_id, + Outcome::Failed { + exit_code: None, + signal: None, + }, + b"", + b"the unit could not be resolved, so it was not acted on", + ); + } + } + } + if !ctx.brake.clear(&ctx.principal) { + return conclude(ctx, order_id, Outcome::Refused(Refused::Braked), b"", b""); + } + let uuid = order_uuid(order_id); + let command_id = format!("up-{uuid}"); + let steps = order.operation.steps(); + let actuation = match &order.operation { + Operation::Deploy { .. } => command_id.clone(), + _ => format!("idunn-order-{uuid}"), + }; + let started = StartRecord { + schema_version: START_SCHEMA.into(), + order_id: order_id.into(), + started_at_unix_millis: now_millis()?, + actuation, + }; + if !ctx.store.put(order_id, START_SCHEMA, &started)? { + return Ok(()); + } + if let Operation::Deploy { selector } = &order.operation { + let outcome = match commit_deployment_command( + &ctx.state_store, + &command_id, + selector, + &requester(&order.principal, order_id), + ) { + Ok(_) => Outcome::Submitted { command_id }, + Err(_) => Outcome::Failed { + exit_code: None, + signal: None, + }, + }; + return conclude(ctx, order_id, outcome, b"", b""); + } + let (mut stdout, mut stderr) = (Vec::new(), Vec::new()); + let mut outcome = Outcome::Ran; + for (index, step) in steps.iter().enumerate() { + let unit = if steps.len() == 1 { + format!("idunn-order-{uuid}") + } else { + format!("idunn-order-{uuid}-{}", index + 1) + }; + match ctx.systemd.run(&unit, step) { + Ok(report) => { + stdout = report.stdout_tail; + stderr = report.stderr_tail; + if report.exit_code != Some(0) { + outcome = Outcome::Failed { + exit_code: report.exit_code, + signal: report.signal, + }; + break; + } + } + Err(_) => { + outcome = Outcome::Failed { + exit_code: None, + signal: None, + }; + stderr = b"the transient unit could not be started".to_vec(); + break; + } + } + } + conclude(ctx, order_id, outcome, &stdout, &stderr) +} + +/// On service start: an order with a start record and no outcome ran, or began +/// to, in a life that ended. It is Lost and never run again. Returns the +/// orders that never started, oldest first. +pub(crate) fn recover(ctx: &Ctx) -> Result> { + let ledger = ctx.store.ledger()?; + for start in &ledger.starts { + if ledger.outcome_of(&start.order_id).is_none() { + conclude(ctx, &start.order_id, Outcome::Lost, b"", b"")?; + } + } + Ok(ledger + .orders + .iter() + .filter(|order| { + ledger.start_of(&order.order_id).is_none() && ledger.outcome_of(&order.order_id).is_none() + }) + .map(|order| order.order_id.clone()) + .collect()) +} + +// ---------------------------------------------------------------- posting -- + +struct Due { + order_id: String, + kind: PostKind, + text: String, +} + +fn quoted(reason: &str) -> String { + format!("\"{}\"", escape(reason.as_bytes())) +} + +fn accepted_text(order: &OrderRecord) -> String { + format!( + "Steward order {} accepted: {}\nissuer {} (invocation {})\nreason: {}", + order.order_id, + escape(order.operation.describe().as_bytes()), + escape(order.issuer_unit.as_bytes()), + escape(order.issuer_invocation.as_bytes()), + quoted(&order.reason) + ) +} + +/// A deploy's command has finished once its state is neither queued nor running. +fn deploy_finished(state: &str) -> bool { + !matches!(state, "queued" | "running") +} + +/// What is owed to the room. `deploy_state` is a deployment command's derived +/// state and detail, read from the control store by the caller. +fn due_posts( + ledger: &Ledger, + deploy_state: &dyn Fn(&str) -> Result<(&'static str, String)>, +) -> Vec { + let mut due = Vec::new(); + for order in &ledger.orders { + let id = &order.order_id; + if !ledger.posted(id, PostKind::Accepted) { + due.push(Due { + order_id: id.clone(), + kind: PostKind::Accepted, + text: accepted_text(order), + }); + } + if ledger.posted(id, PostKind::Result) { + continue; + } + let Some(record) = ledger.outcome_of(id) else { + continue; + }; + let mut line = record.outcome.describe(); + if let Outcome::Submitted { command_id } = &record.outcome { + match deploy_state(command_id) { + Ok((state, detail)) if deploy_finished(state) => { + line = format!("{line}: {state} {}", escape(detail.as_bytes())); + } + _ => continue, + } + } + due.push(Due { + order_id: id.clone(), + kind: PostKind::Result, + text: format!( + "Steward order {id}: {line}\n{}", + escape(order.operation.describe().as_bytes()) + ), + }); + } + due +} + +/// Post what is due, in order, stopping at the first failure. Returns how many +/// went out. +pub(crate) fn post_pass(ctx: &Ctx) -> Result { + let ledger = ctx.store.ledger()?; + let mut sent = 0; + let deploy_state = |command_id: &str| command_state(&ctx.state_store, command_id); + for due in due_posts(&ledger, &deploy_state) { + let transaction = format!("idunn-{}-{}", due.order_id, due.kind.as_str()); + let event_id = ctx.homeserver.post(&transaction, &due.text)?; + ctx.store.put( + &post_key(&due.order_id, due.kind), + POST_SCHEMA, + &PostRecord { + schema_version: POST_SCHEMA.into(), + post_key: post_key(&due.order_id, due.kind), + event_id, + posted_at_unix_millis: now_millis()?, + }, + )?; + sent += 1; + } + Ok(sent) +} + +// ----------------------------------------------------------------- frames -- + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct OrderRequest { + pub(crate) operation: Operation, + pub(crate) reason: String, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ShowRequest { + pub(crate) id: String, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Request { + Order(OrderRequest), + Show(ShowRequest), + List, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct OrderView { + pub(crate) order: OrderRecord, + pub(crate) start: Option, + pub(crate) outcome: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct OrderLine { + pub(crate) order_id: String, + pub(crate) status: String, + pub(crate) summary: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Response { + Accepted { order_id: String }, + Refused(Refusal), + Show(Box), + List(Vec), +} + +fn write_frame(stream: &mut impl Write, value: &impl Serialize) -> Result<()> { + let bytes = rmp_serde::to_vec_named(value)?; + stream.write_all(&u32::try_from(bytes.len())?.to_be_bytes())?; + stream.write_all(&bytes)?; + Ok(stream.flush()?) +} + +fn read_frame(stream: &mut impl Read, limit: usize) -> Result> { + let mut length = [0u8; 4]; + stream.read_exact(&mut length)?; + let length = u32::from_be_bytes(length) as usize; + ensure!(length <= limit, "a frame exceeds its limit"); + let mut bytes = vec![0u8; length]; + stream.read_exact(&mut bytes)?; + Ok(bytes) +} + +fn status_word(view: &OrderView) -> &'static str { + match (&view.outcome, &view.start) { + (Some(record), _) => match record.outcome { + Outcome::Ran => "ran", + Outcome::Failed { .. } => "failed", + Outcome::Refused(_) => "refused", + Outcome::Lost => "lost", + Outcome::Submitted { .. } => "submitted", + }, + (None, Some(_)) => "running", + (None, None) => "accepted", + } +} + +fn view_of(ledger: &Ledger, order: &OrderRecord) -> OrderView { + OrderView { + order: order.clone(), + start: ledger.start_of(&order.order_id).cloned(), + outcome: ledger.outcome_of(&order.order_id).cloned(), + } +} + +fn answer(stream: &mut UnixStream, response: Response) -> Result<()> { + write_frame(stream, &response) +} + +fn refuse(stream: &mut UnixStream, field: &'static str, code: Code) -> Result<()> { + answer(stream, Response::Refused(Refusal::new(field, code))) +} + +/// Serve one connection. The peer is authenticated before its frame is read. +pub(crate) fn handle( + ctx: &Ctx, + stream: &mut UnixStream, + queue: &mpsc::Sender, +) -> Result<()> { + stream.set_read_timeout(Some(Duration::from_secs(5)))?; + stream.set_write_timeout(Some(Duration::from_secs(5)))?; + let peer = ctx.peers.peer(stream)?; + // The steward may order and read; root, the operator, may only read. + let steward = peer.uid == ctx.steward_uid; + if !steward && peer.uid != 0 { + eprintln!("steward socket: refused a peer that is not the steward"); + return refuse(stream, "peer", Code::Unauthorized); + } + let request = read_frame(stream, REQUEST_FRAME_LIMIT) + .ok() + .and_then(|bytes| rmp_serde::from_slice::(&bytes).ok()); + let Some(request) = request else { + eprintln!("steward socket: refused a malformed frame"); + return refuse(stream, "frame", Code::Malformed); + }; + match request { + Request::Order(_) if !steward => { + eprintln!("steward socket: refused an order from a peer that is not the steward"); + refuse(stream, "peer", Code::Unauthorized) + } + Request::Order(request) => { + let checked = + validate_reason(&request.reason).and_then(|()| request.operation.validate()); + if let Err(refusal) = checked { + eprintln!("steward socket: refused an order at the grammar"); + return answer(stream, Response::Refused(refusal)); + } + let order = OrderRecord { + schema_version: ORDER_SCHEMA.into(), + order_id: format!("ord-{}", Uuid::new_v4()), + principal: ctx.principal.clone(), + issuer_uid: peer.uid, + issuer_invocation: ctx.systemd.invocation(&peer.issuer_unit).unwrap_or_default(), + issuer_unit: peer.issuer_unit, + issued_at_unix_millis: now_millis()?, + operation: request.operation, + reason: request.reason, + }; + match ctx.store.put(&order.order_id, ORDER_SCHEMA, &order) { + Ok(true) => {} + _ => return refuse(stream, "store", Code::Unavailable), + } + queue + .send(order.order_id.clone()) + .map_err(|_| anyhow!("the executor has stopped"))?; + answer( + stream, + Response::Accepted { + order_id: order.order_id, + }, + ) + } + Request::Show(request) => { + if request.id.strip_prefix("ord-").map(Uuid::parse_str).is_none_or(|parsed| parsed.is_err()) + { + return refuse(stream, "id", Code::BadShape); + } + let ledger = ctx.store.ledger()?; + match ledger.orders.iter().find(|order| order.order_id == request.id) { + Some(order) => answer(stream, Response::Show(Box::new(view_of(&ledger, order)))), + None => refuse(stream, "id", Code::Unknown), + } + } + Request::List => { + let ledger = ctx.store.ledger()?; + let skip = ledger.orders.len().saturating_sub(LIST_LIMIT); + let lines = ledger + .orders + .iter() + .skip(skip) + .map(|order| { + let view = view_of(&ledger, order); + OrderLine { + order_id: order.order_id.clone(), + status: status_word(&view).into(), + summary: order.operation.describe(), + } + }) + .collect(); + answer(stream, Response::List(lines)) + } + } +} + +// ------------------------------------------------------------- real ports -- + +/// `systemctl` and `systemd-run`, found by name. +pub(crate) struct HostSystemd; + +/// The names in one `systemctl show` list value. systemd quotes a name that +/// holds a backslash and doubles the backslash inside the quotes. +fn parse_names(value: &str) -> Vec { + let mut names = Vec::new(); + let mut current = String::new(); + let (mut quoted, mut open, mut escaped) = (false, false, false); + for character in value.chars() { + match character { + _ if escaped => { + current.push(character); + escaped = false; + } + '\\' if quoted => escaped = true, + '"' => { + quoted = !quoted; + open = true; + } + _ if character.is_whitespace() && !quoted => { + if open { + names.push(std::mem::take(&mut current)); + open = false; + } + } + _ => { + current.push(character); + open = true; + } + } + } + if open { + names.push(current); + } + names +} + +/// The properties one `systemctl show` is asked for: identity, what fires at +/// the unit's end, who is bound to it, and what it upholds or groups. +fn show_properties() -> String { + ["Id", "Names", "LoadState"] + .into_iter() + .chain(ACTIONS) + .chain(ON_END) + .chain(["BoundBy", "Upholds", "ConsistsOf"]) + .collect::>() + .join(",") +} + +fn parse_show(output: &str) -> Result { + let mut facts = UnitFacts::default(); + for line in output.lines() { + let Some((key, value)) = line.split_once('=') else { + continue; + }; + match key { + "Id" => facts.id = value.to_owned(), + "Names" => facts.names = parse_names(value), + "LoadState" => facts.not_found = value == "not-found", + "BoundBy" => facts.bound_by = parse_names(value), + "Upholds" => facts.upholds = parse_names(value), + "ConsistsOf" => facts.consists_of = parse_names(value), + _ if ACTIONS.contains(&key) => { + if !matches!(value, "" | "none") { + facts.on_end.push(key.to_owned()); + } + } + _ if ON_END.contains(&key) => { + facts.on_end.extend(parse_names(value).into_iter().map(|_| key.to_owned())); + } + _ => {} + } + } + ensure!(!facts.id.is_empty(), "systemctl named no unit"); + Ok(facts) +} + +impl Systemd for HostSystemd { + fn show(&self, unit: &str) -> Result { + let output = Command::new("systemctl") + .args(["show", "--no-pager"]) + .arg(format!("--property={}", show_properties())) + .args(["--", unit]) + .stdin(Stdio::null()) + .output() + .context("running systemctl show")?; + ensure!(output.status.success(), "systemctl show failed"); + parse_show(&String::from_utf8_lossy(&output.stdout)) + } + + fn invocation(&self, unit: &str) -> Option { + let output = Command::new("systemctl") + .args(["show", "--no-pager", "--property=InvocationID", "--value", "--"]) + .arg(unit) + .stdin(Stdio::null()) + .output() + .ok()?; + let value = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + (output.status.success() && !value.is_empty()).then_some(value) + } + + fn run(&self, unit: &str, step: &Step) -> Result { + let mut command = Command::new("systemd-run"); + command.args(["--unit", unit, "--wait", "--pipe", "--collect"]); + command.args(["-p", "RuntimeMaxSec=15min"]); + for (name, value) in &step.environment { + command.arg(format!("--setenv={name}={value}")); + } + let output = command + .arg("--") + .args(&step.argv) + .stdin(Stdio::null()) + .output() + .context("running systemd-run")?; + Ok(RunReport { + exit_code: output.status.code(), + signal: output.status.signal(), + stdout_tail: tail(&output.stdout), + stderr_tail: tail(&output.stderr), + }) + } +} + +/// A system service or scope names the issuer: the first unit under +/// `system.slice` in the peer's cgroup path. A peer cannot choose it. Slices +/// above it are the manager's, and what a delegated subtree adds below it is +/// ignored. Any other path (a user manager's, a bare directory) is `unknown`. +fn unit_from_cgroup(text: &str) -> String { + let line = text + .lines() + .find(|line| line.starts_with("0::") || line.contains(":name=systemd:")) + .unwrap_or_default(); + let path = line.rsplit_once(':').map_or("", |(_, path)| path); + let unit = path.strip_prefix("/system.slice/").and_then(|under| { + for segment in under.split('/') { + if segment.ends_with(".slice") { + continue; + } + let unit = segment.ends_with(".service") || segment.ends_with(".scope"); + return (unit && segment.len() <= UNIT_LIMIT).then_some(segment); + } + None + }); + unit.map_or_else(|| "unknown".to_owned(), str::to_owned) +} + +/// Kernel peer credentials (`SO_PEERCRED`) and the peer's cgroup. +pub(crate) struct KernelPeers; + +impl PeerCredentials for KernelPeers { + fn peer(&self, stream: &UnixStream) -> Result { + use std::os::fd::AsRawFd; + let mut credentials = libc::ucred { + pid: 0, + uid: 0, + gid: 0, + }; + let mut length = std::mem::size_of::() as libc::socklen_t; + // SAFETY: the descriptor is a live socket and the pointer and length + // describe `credentials`, which outlives the call. + let status = unsafe { + libc::getsockopt( + stream.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_PEERCRED, + (&mut credentials as *mut libc::ucred).cast(), + &mut length, + ) + }; + ensure!( + status == 0 && length as usize == std::mem::size_of::(), + "peer credentials are unavailable" + ); + let cgroup = + std::fs::read_to_string(format!("/proc/{}/cgroup", credentials.pid)).unwrap_or_default(); + Ok(Peer { + uid: credentials.uid, + issuer_unit: unit_from_cgroup(&cgroup), + }) + } +} + +/// `mind-control --instance check`; a missing binary is a held brake. +pub(crate) struct MindControl { + pub(crate) program: PathBuf, +} + +impl BrakeCheck for MindControl { + fn clear(&self, principal: &str) -> bool { + Command::new(&self.program) + .args(["--instance", principal, "check"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok_and(|status| status.success()) + } +} + +/// The Synapse client API on loopback, over plain HTTP/1.0 so the reply is +/// never chunked. The token is read from its credential file for each post. +pub(crate) struct Synapse { + pub(crate) address: std::net::SocketAddr, + pub(crate) room: String, + pub(crate) token_file: PathBuf, + /// The whole wait for a reply, however slowly it drips. + pub(crate) reply_within: Duration, +} + +fn percent_encode(text: &str) -> String { + let mut encoded = String::new(); + for byte in text.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { + encoded.push(char::from(byte)); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded +} + +impl Homeserver for Synapse { + fn post(&self, txn_id: &str, text: &str) -> Result { + let token = std::fs::read_to_string(&self.token_file) + .context("the homeserver token is unreadable")?; + let token = token.trim(); + ensure!( + !token.is_empty() && token.bytes().all(|byte| byte.is_ascii_graphic()), + "the homeserver token is not a bearer token" + ); + let body = serde_json::to_vec(&serde_json::json!({"msgtype": "m.text", "body": text}))?; + let head = format!( + "PUT /_matrix/client/v3/rooms/{}/send/m.room.message/{txn_id} HTTP/1.0\r\n\ + Host: {}\r\nAuthorization: Bearer {token}\r\n\ + Content-Type: application/json\r\nContent-Length: {}\r\n\r\n", + percent_encode(&self.room), + self.address, + body.len() + ); + let mut socket = std::net::TcpStream::connect_timeout(&self.address, Duration::from_secs(5)) + .context("the homeserver is unreachable")?; + socket.set_write_timeout(Some(self.reply_within))?; + socket.write_all(head.as_bytes())?; + socket.write_all(&body)?; + let deadline = std::time::Instant::now() + self.reply_within; + let mut reply = Vec::new(); + let mut chunk = [0u8; 4096]; + while reply.len() < RESPONSE_FRAME_LIMIT { + let remaining = deadline + .checked_duration_since(std::time::Instant::now()) + .filter(|left| !left.is_zero()) + .context("the homeserver reply was too slow")?; + socket.set_read_timeout(Some(remaining))?; + let count = socket + .read(&mut chunk) + .context("the homeserver reply was cut short")?; + if count == 0 { + break; + } + reply.extend_from_slice(&chunk[..count]); + } + let split = reply + .windows(4) + .position(|window| window == b"\r\n\r\n") + .context("the homeserver reply has no body")?; + let status = String::from_utf8_lossy(&reply[..split]); + ensure!( + status.split_whitespace().nth(1) == Some("200"), + "the homeserver refused the post" + ); + #[derive(Deserialize)] + struct Sent { + event_id: String, + } + let sent: Sent = serde_json::from_slice(&reply[split + 4..]) + .map_err(|_| anyhow!("the homeserver reply has no event id"))?; + Ok(sent.event_id) + } +} + +// ---------------------------------------------------------------- binding -- + +/// `/etc/gamecult/idunn/steward.toml`: who the steward is and where to post. +/// It lists no unit of any kind, and any field outside this set is refused. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct StewardBinding { + pub(crate) principal: String, + pub(crate) user: String, + pub(crate) mind_control: PathBuf, + pub(crate) room: String, + pub(crate) homeserver: String, + pub(crate) token_credential: String, +} + +impl StewardBinding { + pub(crate) fn parse(text: &str) -> Result { + let binding: Self = toml::from_str(text) + .map_err(|error| anyhow!("the steward binding does not parse: {}", error.message()))?; + binding.validate()?; + Ok(binding) + } + + fn validate(&self) -> Result<()> { + require_id(&self.principal, "steward principal")?; + ensure!( + !self.principal.starts_with('-') && !self.principal.contains('/'), + "steward principal is invalid" + ); + let user_ok = !self.user.is_empty() + && self.user.len() <= 32 + && self.user.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'_' | b'-') + }) + && !self.user.starts_with(|first: char| first.is_ascii_digit() || first == '-'); + ensure!(user_ok && self.user != "root", "steward user is invalid or root"); + ensure!( + self.mind_control.is_absolute(), + "mind_control must be an absolute path" + ); + let room_ok = self.room.starts_with('!') + && self.room.len() <= 255 + && self.room.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'!' | b'.' | b'_' | b':' | b'=' | b'-') + }); + ensure!(room_ok, "room must be a Matrix room id"); + self.address()?; + let credential_ok = !self.token_credential.is_empty() + && !self.token_credential.starts_with('.') + && self.token_credential.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-') + }); + ensure!(credential_ok, "token_credential is not a credential name"); + Ok(()) + } + + fn address(&self) -> Result { + let address: std::net::SocketAddr = self + .homeserver + .strip_prefix("http://") + .and_then(|rest| rest.parse().ok()) + .context("homeserver must be http://:")?; + ensure!(address.ip().is_loopback(), "homeserver must be loopback"); + Ok(address) + } +} + +/// Root-admitted: owned by root and writable by no one else. +fn admitted(owner: u32, mode: u32) -> bool { + owner == 0 && mode & 0o022 == 0 +} + +fn load_binding(path: &Path) -> Result { + let metadata = std::fs::metadata(path).context("reading the steward binding")?; + ensure!( + metadata.is_file() && admitted(metadata.uid(), metadata.mode()), + "the steward binding is not a root-owned file closed to writes" + ); + let text = std::fs::read_to_string(path).context("reading the steward binding")?; + StewardBinding::parse(&text) +} + +fn user_uid(name: &str) -> Result { + let name = std::ffi::CString::new(name)?; + // SAFETY: `passwd` is plain data and getpwnam_r fills it, with strings in + // `buffer`, which outlives every read of them below (none are read). + let (status, found, uid) = unsafe { + let mut passwd: libc::passwd = std::mem::zeroed(); + let mut buffer = vec![0 as libc::c_char; 4096]; + let mut result: *mut libc::passwd = std::ptr::null_mut(); + let status = libc::getpwnam_r( + name.as_ptr(), + &mut passwd, + buffer.as_mut_ptr(), + buffer.len(), + &mut result, + ); + (status, !result.is_null(), passwd.pw_uid) + }; + ensure!(status == 0 && found, "the steward user is unknown"); + ensure!(uid != 0, "the steward user resolves to root"); + Ok(uid) +} + +// ------------------------------------------------------------------ serve -- + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct ServeOptions { + pub(crate) state_store: PathBuf, + pub(crate) store: PathBuf, + pub(crate) steward_binding: PathBuf, + pub(crate) socket: PathBuf, +} + +const DEFAULT_STORE: &str = "/var/lib/gamecult/idunn-orders"; +const DEFAULT_BINDING: &str = "/etc/gamecult/idunn/steward.toml"; +const DEFAULT_SOCKET: &str = "/run/idunn-orders/steward.sock"; + +pub(crate) fn parse_serve(mut args: impl Iterator) -> Result { + ensure!( + args.next().as_deref() == Some("serve"), + "idunn orders takes the subcommand serve" + ); + let mut options = ServeOptions { + state_store: default_state_store(), + store: DEFAULT_STORE.into(), + steward_binding: DEFAULT_BINDING.into(), + socket: DEFAULT_SOCKET.into(), + }; + while let Some(argument) = args.next() { + match argument.as_str() { + "--state-store" => options.state_store = path_value(&mut args, "--state-store")?, + "--store" => options.store = path_value(&mut args, "--store")?, + "--steward-binding" => { + options.steward_binding = path_value(&mut args, "--steward-binding")? + } + "--socket" => options.socket = path_value(&mut args, "--socket")?, + _ => bail!("an orders serve option is not part of the grammar"), + } + } + Ok(options) +} + +/// Seconds between post passes: two when the last pass went through, and +/// doubling for each failed pass in a row up to a minute. +fn post_pause(failures: u32) -> u64 { + if failures == 0 { + 2 + } else { + (2u64 << failures.min(5)).min(60) + } +} + +/// Run a thread whose end ends the service: systemd restarts it, and the +/// start recovery settles what it left. +fn vital(name: &'static str, work: impl FnOnce() + Send + 'static) { + struct ExitOnDrop(&'static str); + impl Drop for ExitOnDrop { + fn drop(&mut self) { + eprintln!("steward orders: the {} thread ended", self.0); + std::process::exit(1); + } + } + thread::Builder::new() + .name(name.into()) + .spawn(move || { + let _guard = ExitOnDrop(name); + work(); + }) + .expect("a thread can be spawned"); +} + +/// The socket belongs to the steward and no one else may open it; the peer +/// credential check on every connection stays the lock. +fn bind_socket(path: &Path, steward_uid: u32) -> Result { + if let Ok(metadata) = std::fs::symlink_metadata(path) { + use std::os::unix::fs::FileTypeExt; + ensure!( + metadata.file_type().is_socket(), + "the socket path holds something that is not a socket" + ); + std::fs::remove_file(path).context("removing a stale socket")?; + } + let listener = UnixListener::bind(path).context("binding the steward socket")?; + std::os::unix::fs::chown(path, Some(steward_uid), None) + .context("giving the steward socket to the steward")?; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + .context("setting the steward socket mode")?; + Ok(listener) +} + +/// Files this service creates in Idunn's own stores (a deploy order writes +/// `control.cc`) take Idunn's umask, whatever the unit that runs it sets. +const IDUNN_UMASK: libc::mode_t = 0o027; + +pub(crate) fn serve(options: ServeOptions) -> Result<()> { + // SAFETY: umask only sets the process file-mode mask; it cannot fail. + unsafe { libc::umask(IDUNN_UMASK) }; + let binding = load_binding(&options.steward_binding)?; + let credentials = std::env::var_os("CREDENTIALS_DIRECTORY") + .map(PathBuf::from) + .context("CREDENTIALS_DIRECTORY is unset; the token comes from LoadCredential")?; + let ctx = Arc::new(Ctx { + store: OrderStore::new(&options.store), + state_store: options.state_store, + steward_uid: user_uid(&binding.user)?, + principal: binding.principal.clone(), + systemd: Arc::new(HostSystemd), + peers: Arc::new(KernelPeers), + brake: Arc::new(MindControl { + program: binding.mind_control.clone(), + }), + homeserver: Arc::new(Synapse { + address: binding.address()?, + room: binding.room.clone(), + token_file: credentials.join(&binding.token_credential), + reply_within: Duration::from_secs(10), + }), + }); + ctx.store.prepare()?; + let pending = recover(&ctx)?; + let listener = bind_socket(&options.socket, ctx.steward_uid)?; + let (queue, orders) = mpsc::channel::(); + for order_id in pending { + queue.send(order_id)?; + } + let executor = Arc::clone(&ctx); + vital("executor", move || { + for order_id in orders { + if let Err(error) = execute(&executor, &order_id) { + eprintln!("steward orders: {order_id} was not run to a record: {error:#}"); + } + } + }); + let poster = Arc::clone(&ctx); + vital("poster", move || { + let mut failures = 0u32; + loop { + match post_pass(&poster) { + Ok(_) => failures = 0, + Err(error) => { + failures = failures.saturating_add(1); + eprintln!("steward orders: posting is delayed: {error:#}"); + } + } + thread::sleep(Duration::from_secs(post_pause(failures))); + } + }); + for stream in listener.incoming() { + match stream { + Ok(mut stream) => { + if let Err(error) = handle(&ctx, &mut stream, &queue) { + eprintln!("steward socket: a connection ended in error: {error:#}"); + } + } + Err(error) => eprintln!("steward socket: accept failed: {}", error.kind()), + } + } + bail!("the steward socket stopped accepting") +} + +// ----------------------------------------------------------------- client -- + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum OrderAction { + Issue { operation: Operation, reason: String }, + Show { id: String }, + List, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct OrderCommand { + pub(crate) socket: PathBuf, + pub(crate) action: OrderAction, +} + +pub(crate) fn parse_order(mut args: impl Iterator) -> Result { + let kind = args.next().context("idunn order requires an operation")?; + let mut socket = PathBuf::from(DEFAULT_SOCKET); + let mut reason = None; + let mut positional = Vec::new(); + while let Some(argument) = args.next() { + match argument.as_str() { + "--socket" => socket = path_value(&mut args, "--socket")?, + "--reason" => reason = Some(string_value(&mut args, "--reason")?), + word if word.starts_with('-') => bail!("an order option is not part of the grammar"), + _ => positional.push(argument), + } + } + let action = match kind.as_str() { + "list" => { + ensure!(positional.is_empty() && reason.is_none(), "list takes no arguments"); + OrderAction::List + } + "show" => { + ensure!(positional.len() == 1 && reason.is_none(), "show takes one order id"); + OrderAction::Show { + id: positional.remove(0), + } + } + "package-install" | "unit" | "deploy" => { + let operation = match kind.as_str() { + "package-install" => Operation::PackageInstall { + packages: positional, + }, + "unit" => { + ensure!(positional.len() == 2, "unit takes a verb and a unit"); + let verb = Verb::parse(&positional[0]) + .context("the unit verb is not start, stop, restart or reload")?; + Operation::UnitAction { + verb, + unit: positional.remove(1), + } + } + _ => { + ensure!(positional.len() == 1, "deploy takes one selector"); + Operation::Deploy { + selector: positional.remove(0), + } + } + }; + let reason = reason.context("an order requires --reason")?; + validate_reason(&reason)?; + operation.validate()?; + OrderAction::Issue { operation, reason } + } + _ => bail!("the order operation is not part of the grammar"), + }; + Ok(OrderCommand { socket, action }) +} + +fn call(socket: &Path, request: &Request) -> Result { + let mut stream = UnixStream::connect(socket).context("connecting to the steward socket")?; + stream.set_read_timeout(Some(Duration::from_secs(30)))?; + write_frame(&mut stream, request)?; + let bytes = read_frame(&mut stream, RESPONSE_FRAME_LIMIT)?; + rmp_serde::from_slice(&bytes).map_err(|_| anyhow!("the steward socket sent an unreadable reply")) +} + +fn render_tail(lines: &mut Vec>, stream: &str, bytes: &[u8]) { + if bytes.is_empty() { + return; + } + lines.push(format!(" {stream} (last {} bytes)", bytes.len()).into_bytes()); + let text = bytes.strip_suffix(b"\n").unwrap_or(bytes); + lines.extend(text.split(|byte| *byte == b'\n').map(|line| { + let mut shown = b" | ".to_vec(); + shown.extend_from_slice(line); + shown + })); +} + +/// Raw lines for the terminal; `emit` is their only way out. +fn render_view(view: &OrderView) -> Vec> { + let order = &view.order; + let mut lines: Vec> = vec![ + format!("order {}", order.order_id).into_bytes(), + format!("status {}", status_word(view)).into_bytes(), + format!("operation {}", order.operation.describe()).into_bytes(), + format!( + "issuer {} (invocation {}, uid {})", + order.issuer_unit, order.issuer_invocation, order.issuer_uid + ) + .into_bytes(), + format!("reason {}", quoted(&order.reason)).into_bytes(), + ]; + if let Some(start) = &view.start { + lines.push(format!("started {}", start.actuation).into_bytes()); + } + if let Some(record) = &view.outcome { + lines.push(format!("outcome {}", record.outcome.describe()).into_bytes()); + render_tail(&mut lines, "stdout", &record.stdout_tail); + render_tail(&mut lines, "stderr", &record.stderr_tail); + } + lines +} + +pub(crate) fn run_order(command: OrderCommand) -> Result<()> { + let request = match command.action { + OrderAction::Issue { operation, reason } => Request::Order(OrderRequest { operation, reason }), + OrderAction::Show { id } => Request::Show(ShowRequest { id }), + OrderAction::List => Request::List, + }; + let mut out = std::io::stdout().lock(); + match call(&command.socket, &request)? { + Response::Accepted { order_id } => writeln!(out, "{order_id}")?, + Response::Refused(refusal) => bail!("the steward refused the request: {refusal}"), + Response::Show(view) => emit(&mut out, &render_view(&view))?, + Response::List(lines) => { + let lines: Vec> = lines + .iter() + .map(|line| format!("{} {} {}", line.order_id, line.status, line.summary).into_bytes()) + .collect(); + emit(&mut out, &lines)? + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::sync::Mutex; + use std::sync::atomic::{AtomicBool, Ordering}; + + use tempfile::TempDir; + + use super::*; + + const STEWARD: u32 = 1000; + const CANARY: &str = "CANARY-7f3a91"; + + /// What `systemctl show` prints for a loaded unit that acts on nothing, + /// plus the given properties (a given key replaces its default). + fn show_text(id: &str, properties: &[(&str, &str)]) -> String { + let mut text = format!("Id={id}\n"); + for (key, default) in [("Names", id), ("LoadState", "loaded")] { + if !properties.iter().any(|(given, _)| *given == key) { + text += &format!("{key}={default}\n"); + } + } + for (key, value) in properties { + text += &format!("{key}={value}\n"); + } + text + } + + struct FakeSystemd { + units: Mutex>, + runs: Mutex>, + shows: Mutex>, + report: Mutex, + } + + impl Systemd for FakeSystemd { + fn show(&self, unit: &str) -> Result { + self.shows.lock().unwrap().push(unit.into()); + let known = self.units.lock().unwrap().get(unit).cloned(); + parse_show(&known.unwrap_or_else(|| show_text(unit, &[]))) + } + + fn invocation(&self, _unit: &str) -> Option { + Some("invocation-1".into()) + } + + fn run(&self, unit: &str, step: &Step) -> Result { + self.runs.lock().unwrap().push((unit.into(), step.clone())); + Ok(self.report.lock().unwrap().clone()) + } + } + + struct FakePeers(u32); + + impl PeerCredentials for FakePeers { + fn peer(&self, _stream: &UnixStream) -> Result { + Ok(Peer { + uid: self.0, + issuer_unit: "mind-eureka-self.service".into(), + }) + } + } + + /// Whether the brake is clear, and the principals it was asked about. + struct FakeBrake(AtomicBool, Mutex>); + + impl BrakeCheck for FakeBrake { + fn clear(&self, principal: &str) -> bool { + self.1.lock().unwrap().push(principal.into()); + self.0.load(Ordering::SeqCst) + } + } + + #[derive(Default)] + struct FakeHome { + failing: AtomicBool, + sent: Mutex>, + } + + impl Homeserver for FakeHome { + fn post(&self, txn_id: &str, text: &str) -> Result { + ensure!(!self.failing.load(Ordering::SeqCst), "the homeserver is down"); + let mut sent = self.sent.lock().unwrap(); + sent.push((txn_id.into(), text.into())); + Ok(format!("$event{}", sent.len())) + } + } + + struct Harness { + directory: TempDir, + ctx: Ctx, + systemd: Arc, + brake: Arc, + home: Arc, + queue: mpsc::Sender, + _orders: mpsc::Receiver, + } + + fn harness_for(peer_uid: u32) -> Harness { + let directory = TempDir::new().unwrap(); + let systemd = Arc::new(FakeSystemd { + units: Mutex::new(BTreeMap::new()), + runs: Mutex::new(Vec::new()), + shows: Mutex::new(Vec::new()), + report: Mutex::new(RunReport { + exit_code: Some(0), + ..RunReport::default() + }), + }); + let brake = Arc::new(FakeBrake(AtomicBool::new(true), Mutex::new(Vec::new()))); + let home = Arc::new(FakeHome::default()); + let (queue, orders) = mpsc::channel(); + let ctx = Ctx { + store: OrderStore::new(&directory.path().join("orders")), + state_store: directory.path().join("state").join("control.cc"), + steward_uid: STEWARD, + principal: "eureka".into(), + systemd: systemd.clone(), + peers: Arc::new(FakePeers(peer_uid)), + brake: brake.clone(), + homeserver: home.clone(), + }; + Harness { + directory, + ctx, + systemd, + brake, + home, + queue, + _orders: orders, + } + } + + fn harness() -> Harness { + harness_for(STEWARD) + } + + impl Harness { + fn exchange(&self, write: impl FnOnce(&mut UnixStream)) -> Response { + let (mut client, mut server) = UnixStream::pair().unwrap(); + client.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + write(&mut client); + handle(&self.ctx, &mut server, &self.queue).unwrap(); + let bytes = read_frame(&mut client, RESPONSE_FRAME_LIMIT).unwrap(); + rmp_serde::from_slice(&bytes).unwrap() + } + + fn ask(&self, request: &Request) -> Response { + self.exchange(|client| write_frame(client, request).unwrap()) + } + + fn issue(&self, operation: Operation) -> Response { + self.ask(&Request::Order(OrderRequest { + operation, + reason: "probe".into(), + })) + } + + fn issue_unit(&self, verb: Verb, unit: &str) -> Response { + self.issue(Operation::UnitAction { + verb, + unit: unit.into(), + }) + } + + fn accept(&self, response: Response) -> String { + match response { + Response::Accepted { order_id } => order_id, + other => panic!("expected an accepted order, got {other:?}"), + } + } + + fn outcome(&self, order_id: &str) -> Outcome { + self.ctx + .store + .ledger() + .unwrap() + .outcome_of(order_id) + .expect("the order has an outcome") + .outcome + .clone() + } + + fn runs(&self) -> Vec<(String, Step)> { + self.systemd.runs.lock().unwrap().clone() + } + + /// Tell the fake `systemctl show` what it prints for `name`. + fn describe_unit(&self, name: &str, text: String) { + self.systemd.units.lock().unwrap().insert(name.into(), text); + } + + fn describe(&self, name: &str, properties: &[(&str, &str)]) { + self.describe_unit(name, show_text(name, properties)); + } + } + + fn all_verbs() -> [Verb; 4] { + [Verb::Start, Verb::Stop, Verb::Restart, Verb::Reload] + } + + #[test] + fn foreign_uid_is_refused_at_the_door() { + let h = harness_for(STEWARD + 1); + let response = h.issue_unit(Verb::Restart, "nginx.service"); + assert_eq!( + response, + Response::Refused(Refusal::new("peer", Code::Unauthorized)) + ); + let response = h.ask(&Request::List); + assert_eq!( + response, + Response::Refused(Refusal::new("peer", Code::Unauthorized)) + ); + assert!(!h.ctx.store.path().exists(), "nothing is written for a foreign peer"); + assert!(h.runs().is_empty()); + } + + #[derive(Serialize)] + struct ForgedBody { + operation: Operation, + reason: String, + #[serde(skip_serializing_if = "Option::is_none")] + order_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + requested_by: Option, + #[serde(skip_serializing_if = "Option::is_none")] + command: Option, + } + + #[derive(Serialize)] + struct ForgedFrame { + order: ForgedBody, + } + + #[test] + fn order_id_and_requester_are_minted_not_declared() { + let h = harness(); + let forge = |order_id: Option<&str>, requested_by: Option<&str>, command: Option<&str>| { + let frame = ForgedFrame { + order: ForgedBody { + operation: Operation::UnitAction { + verb: Verb::Restart, + unit: "nginx.service".into(), + }, + reason: "probe".into(), + order_id: order_id.map(str::to_owned), + requested_by: requested_by.map(str::to_owned), + command: command.map(str::to_owned), + }, + }; + h.exchange(|client| write_frame(client, &frame).unwrap()) + }; + // The same frame without the extra field is a good order. + let honest = h.accept(forge(None, None, None)); + for forged in [ + forge(Some("ord-declared"), None, None), + forge(None, Some("steward:someone-else order x"), None), + forge(None, None, Some("reboot-now")), + ] { + assert_eq!( + forged, + Response::Refused(Refusal::new("frame", Code::Malformed)) + ); + } + let ledger = h.ctx.store.ledger().unwrap(); + assert_eq!(ledger.orders.len(), 1, "a forged frame leaves no record"); + let order = &ledger.orders[0]; + assert_eq!(order.order_id, honest); + let uuid = order.order_id.strip_prefix("ord-").expect("ord- prefix"); + assert_eq!(Uuid::parse_str(uuid).unwrap().get_version_num(), 4); + assert_eq!(order.issuer_unit, "mind-eureka-self.service"); + assert_eq!(order.issuer_invocation, "invocation-1"); + assert_eq!(order.principal, "eureka"); + assert_eq!(order.issuer_uid, STEWARD); + assert_eq!( + requester(&order.principal, &order.order_id), + format!("steward:eureka order {}", order.order_id) + ); + } + + #[test] + fn package_names_outside_grammar_are_refused() { + let h = harness(); + let bad = [ + "-y", + "--allow-unauthenticated", + "-", + "a", + "", + "Foo", + "a/b", + "/tmp/x", + "./x", + "x.deb", + "foo.deb", + "foo bar", + "foo;reboot", + "foo$(id)", + "foo`id`", + "foo|bar", + "foo&bar", + "foo\nbar", + "foo*", + "foo?", + "pkg:amd64", + "pkg/stable", + "libc6-", + "foo=", + "foo=1 2", + "foo=1;2", + "foo=1=2", + "fo\u{ff0f}o", + "f\u{43e}o", + ]; + for name in bad { + let response = h.issue(Operation::PackageInstall { + packages: vec!["git-lfs".into(), name.into()], + }); + assert!(matches!(response, Response::Refused(_)), "{name:?} was not refused"); + } + let many: Vec = (0..17).map(|n| format!("pkg-{n}x")).collect(); + assert_eq!( + h.issue(Operation::PackageInstall { packages: many }), + Response::Refused(Refusal::new("package", Code::TooMany)) + ); + assert_eq!( + h.issue(Operation::PackageInstall { packages: vec![] }), + Response::Refused(Refusal::new("package", Code::Empty)) + ); + assert!(!h.ctx.store.path().exists(), "no refusal left a record"); + + let good = [ + "git-lfs", + "matrix-synapse-py3", + "libnss-systemd", + "bubblewrap", + "socat", + "g++", + "libstdc++6", + "python3.11", + "nginx=1.24.0-2ubuntu7", + "foo=1:2.3-4+b1~rc1", + ]; + let id = h.accept(h.issue(Operation::PackageInstall { + packages: good.iter().map(|name| (*name).to_owned()).collect(), + })); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran); + let runs = h.runs(); + assert_eq!(runs.len(), 3); + assert_eq!(runs[0].1.argv, ["apt-get", "update", "-q"]); + // Each name selects exactly itself and apt may remove nothing. + let mut install = vec!["apt-get".to_owned(), "install".into(), "-y".into(), "-q".into()]; + install.extend(["--no-remove".to_owned(), "--".into()]); + install.extend(good.iter().map(|spec| match spec.split_once('=') { + Some((name, version)) => format!("?exact-name({name})={version}"), + None => format!("?exact-name({spec})"), + })); + assert_eq!(runs[1].1.argv, install); + assert!(install.contains(&"?exact-name(libstdc++6)".to_owned())); + assert!(install.contains(&"?exact-name(nginx)=1.24.0-2ubuntu7".to_owned())); + // The last step fails unless every named package is installed. + let mut present = vec!["dpkg".to_owned(), "-s".into(), "--".into()]; + present.extend(good.iter().map(|spec| spec.split('=').next().unwrap().to_owned())); + assert_eq!(runs[2].1.argv, present); + for (unit, step) in &runs { + assert!(unit.starts_with("idunn-order-")); + let apt = step.argv[0] == "apt-get"; + assert_eq!( + step.environment, + if apt { + vec![("DEBIAN_FRONTEND".to_owned(), "noninteractive".to_owned())] + } else { + Vec::new() + } + ); + } + assert_ne!(runs[0].0, runs[1].0, "each step is its own transient unit"); + assert_ne!(runs[1].0, runs[2].0, "each step is its own transient unit"); + + // A package whose name apt would read as a regex or a conflict is + // exactly itself and never a removal. + for spec in ["cowsa.", "iptables-persistent", "a.", "nginx+"] { + let argv = Operation::PackageInstall { packages: vec![spec.into()] }.steps()[1].argv.clone(); + assert_eq!(argv[4..], ["--no-remove", "--", &format!("?exact-name({spec})")]); + } + // A step that failed ends the order: the absent package is a failure. + *h.systemd.report.lock().unwrap() = RunReport { + exit_code: Some(1), + ..RunReport::default() + }; + let absent = h.accept(h.issue(Operation::PackageInstall { packages: vec!["cowsay".into()] })); + execute(&h.ctx, &absent).unwrap(); + assert!(matches!(h.outcome(&absent), Outcome::Failed { exit_code: Some(1), .. })); + assert_eq!(h.runs().len(), 4, "the update failed, so nothing else ran"); + } + + #[test] + fn any_unit_outside_the_refused_set_runs() { + let h = harness(); + let units = [ + "wg-quick@wg0.service", + "nginx.service", + "matrix-synapse.service", + "ssh.socket", + "huginn@eureka.service", + "claude-connector.service", + "persona-matrix-mouth.service", + "mind-heartbeat.service", + "apt-daily.timer", + // Near misses: the prefix is the rule, not a substring or a stem. + "midunn-x.service", + "not-idunn-x.service", + "idunn.service", + "mind-self.service", + "mind-selfx@a.service", + "mind-persona.service", + "mind-personal@a.service", + ]; + for verb in all_verbs() { + for unit in units { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran, "{verb:?} {unit}"); + let (transient, step) = h.runs().pop().unwrap(); + assert!(transient.starts_with("idunn-order-")); + assert_eq!( + step.argv, + ["systemctl", verb.as_str(), "--job-mode=ignore-requirements", "--", unit] + ); + } + } + assert_eq!(h.runs().len(), units.len() * 4); + } + + #[test] + fn refused_set_is_refused_by_name_and_alias() { + let h = harness(); + h.describe_unit( + "backup-name.service", + show_text( + "idunn-yggdrasil.service", + &[("Names", "idunn-yggdrasil.service backup-name.service")], + ), + ); + h.describe( + "quiet-alias.service", + &[("Names", "quiet-alias.service mind-persona@void.service")], + ); + let direct = [ + "idunn-yggdrasil.service", + "idunn-orders.service", + "idunn-vili-x.service", + "idunn-order-3f2c.service", + "idunn-yggdrasil.timer", + "idunn-x.socket", + "idunn-.service", + "idunn-yggdrasil.service.service", + "idunn-x@y.service", + "mind-self@eureka.service", + "mind-persona@void.service", + "mind-self@.service", + "mind-persona@a@b.service", + // Case folds: a lookalike is still refused, never run. + "IDUNN-yggdrasil.service", + "Idunn-Orders.service", + "MIND-SELF@eureka.service", + "Mind-Persona@Void.service", + ]; + for verb in all_verbs() { + for unit in direct { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + match h.outcome(&id) { + Outcome::Refused(Refused::AlwaysRefused { unit: hit, via }) => { + assert_eq!(hit, unit, "{verb:?} {unit}"); + assert_eq!(via, Via::Name); + } + other => panic!("{verb:?} {unit} was {other:?}"), + } + } + } + for verb in all_verbs() { + for (unit, hit) in [ + ("backup-name.service", "idunn-yggdrasil.service"), + ("quiet-alias.service", "mind-persona@void.service"), + ] { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + assert_eq!( + h.outcome(&id), + Outcome::Refused(Refused::AlwaysRefused { + unit: hit.into(), + via: Via::Alias + }), + "{verb:?} {unit}" + ); + } + } + assert!(h.runs().is_empty(), "no systemd verb ran for any refused order"); + // Every refusal is recorded and has a result post due. + let ledger = h.ctx.store.ledger().unwrap(); + let refusals = ledger + .outcomes + .iter() + .filter(|record| matches!(record.outcome, Outcome::Refused(_))) + .count(); + assert_eq!(refusals, direct.len() * 4 + 8); + let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); + let results = due + .iter() + .filter(|due| due.kind == PostKind::Result && due.text.contains("always-refused")) + .count(); + assert_eq!(results, refusals); + // A refused order never started. + assert!(ledger.starts.is_empty()); + } + + fn outcome_of_unit(h: &Harness, verb: Verb, unit: &str) -> Outcome { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + h.outcome(&id) + } + + fn hit(outcome: &Outcome, expected: Via) -> Option<&str> { + match outcome { + Outcome::Refused(Refused::AlwaysRefused { unit, via }) if *via == expected => { + Some(unit.as_str()) + } + _ => None, + } + } + + #[test] + fn a_quoted_name_in_a_dependency_list_is_the_name_it_quotes() { + let h = harness(); + // systemctl show quotes a name with a backslash and doubles the backslash. + h.describe( + "huginn.service", + &[("BoundBy", "\"mind-self@foo\\\\x2dbar.service\" other.service")], + ); + let outcome = outcome_of_unit(&h, Verb::Stop, "huginn.service"); + assert_eq!(hit(&outcome, Via::Bound), Some("mind-self@foo\\x2dbar.service")); + h.describe( + "quiet.service", + &[("Names", "quiet.service \"idunn-a\\\\x2db.service\"")], + ); + let outcome = outcome_of_unit(&h, Verb::Start, "quiet.service"); + assert_eq!(hit(&outcome, Via::Alias), Some("idunn-a\\x2db.service")); + assert!(h.runs().is_empty()); + } + + #[test] + fn unit_orders_run_as_one_job() { + let h = harness(); + for verb in all_verbs() { + let id = h.accept(h.issue_unit(verb, "nginx.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran, "{verb:?}"); + let (_, step) = h.runs().pop().unwrap(); + assert_eq!( + step.argv, + ["systemctl", verb.as_str(), "--job-mode=ignore-requirements", "--", "nginx.service"] + ); + } + } + + #[test] + fn the_rule_reads_only_the_named_unit() { + let h = harness(); + // Whatever the unit lists, no other unit is looked at. + h.describe( + "plain.service", + &[("Requires", "other.service"), ("Wants", "idunn-x.service"), ("Conflicts", "mind-self@a.service")], + ); + h.describe("other.service", &[("LoadState", "not-found")]); + for verb in all_verbs() { + assert_eq!(outcome_of_unit(&h, verb, "plain.service"), Outcome::Ran, "{verb:?}"); + } + assert_eq!(h.systemd.shows.lock().unwrap().clone(), ["plain.service"; 4]); + } + + #[test] + fn a_unit_that_acts_when_it_ends_is_refused() { + let h = harness(); + let properties = [ + ("SuccessAction", "reboot"), + ("FailureAction", "poweroff-force"), + ("StartLimitAction", "soft-reboot-immediate"), + ("JobTimeoutAction", "exit"), + ("FailureAction", "a-spelling-nobody-knows"), + ("OnFailure", "rescue.service"), + ("OnSuccess", "cleanup.service"), + ]; + for (property, value) in properties { + h.describe("ender.service", &[(property, value)]); + for verb in [Verb::Start, Verb::Stop, Verb::Restart] { + assert_eq!( + outcome_of_unit(&h, verb, "ender.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "ender.service".into(), + via: Via::OnEnd + }), + "{verb:?} {property}={value}" + ); + } + assert_eq!(outcome_of_unit(&h, Verb::Reload, "ender.service"), Outcome::Ran); + } + // `none` and an empty list say the unit does nothing at its end. + h.describe( + "calm.service", + &[("SuccessAction", "none"), ("FailureAction", "none"), ("OnFailure", ""), ("OnSuccess", "")], + ); + for verb in all_verbs() { + assert_eq!(outcome_of_unit(&h, verb, "calm.service"), Outcome::Ran, "{verb:?}"); + } + assert_eq!(h.runs().len(), 11); + } + + #[test] + fn host_names_are_refused() { + let h = harness(); + let names = [ + "systemd-suspend.service", + "systemd-hibernate.service", + "systemd-hybrid-sleep.service", + "systemd-suspend-then-hibernate.service", + "rescue.service", + "emergency.service", + ]; + h.describe("sleepy.service", &[("Names", "sleepy.service systemd-hibernate.service")]); + for verb in all_verbs() { + for name in names { + assert_eq!( + outcome_of_unit(&h, verb, name), + Outcome::Refused(Refused::AlwaysRefused { unit: name.into(), via: Via::Host }), + "{verb:?} {name}" + ); + } + assert_eq!( + outcome_of_unit(&h, verb, "sleepy.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "systemd-hibernate.service".into(), + via: Via::Host + }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + assert_eq!(outcome_of_unit(&h, Verb::Start, "systemd-suspendish.service"), Outcome::Ran); + } + + #[test] + fn a_unit_others_are_bound_to_only_starts() { + let h = harness(); + h.describe("base.service", &[("BoundBy", "first.service second.service")]); + for verb in [Verb::Stop, Verb::Restart] { + assert_eq!( + outcome_of_unit(&h, verb, "base.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "first.service".into(), + via: Via::Bound + }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + for verb in [Verb::Start, Verb::Reload] { + assert_eq!(outcome_of_unit(&h, verb, "base.service"), Outcome::Ran, "{verb:?}"); + } + } + + #[test] + fn a_unit_that_upholds_others_is_refused_where_it_would_start() { + let h = harness(); + h.describe("holder.service", &[("Upholds", "held.service")]); + for verb in [Verb::Start, Verb::Restart] { + assert_eq!( + outcome_of_unit(&h, verb, "holder.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "holder.service".into(), + via: Via::Upholds + }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + for verb in [Verb::Stop, Verb::Reload] { + assert_eq!(outcome_of_unit(&h, verb, "holder.service"), Outcome::Ran, "{verb:?}"); + } + } + + #[test] + fn a_wrapper_unit_is_refused_where_the_verb_would_touch_nothing() { + let h = harness(); + h.describe("wrapper.service", &[("ConsistsOf", "member@1.service member@2.service")]); + for verb in [Verb::Stop, Verb::Restart, Verb::Reload] { + assert_eq!( + outcome_of_unit(&h, verb, "wrapper.service"), + Outcome::Refused(Refused::Wrapper { member: "member@1.service".into() }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + assert_eq!(outcome_of_unit(&h, Verb::Start, "wrapper.service"), Outcome::Ran); + } + + #[test] + fn empty_dependency_lists_name_no_one() { + let h = harness(); + // systemctl prints every property it is asked for, empty or not. + h.describe( + "alone.service", + &[("BoundBy", ""), ("Upholds", ""), ("ConsistsOf", "")], + ); + for verb in all_verbs() { + assert_eq!(outcome_of_unit(&h, verb, "alone.service"), Outcome::Ran, "{verb:?}"); + } + } + + #[test] + fn a_refusal_reads_the_named_unit_once_and_a_refused_name_not_at_all() { + let h = harness(); + h.describe("base.service", &[("BoundBy", "first.service")]); + h.describe("idunn-ghost.service", &[("LoadState", "not-found")]); + assert!(hit(&outcome_of_unit(&h, Verb::Stop, "base.service"), Via::Bound).is_some()); + assert_eq!(h.systemd.shows.lock().unwrap().clone(), ["base.service"]); + h.systemd.shows.lock().unwrap().clear(); + assert_eq!( + hit(&outcome_of_unit(&h, Verb::Stop, "idunn-ghost.service"), Via::Name), + Some("idunn-ghost.service") + ); + assert!(h.systemd.shows.lock().unwrap().is_empty()); + } + + #[test] + fn each_refusal_is_posted_under_its_own_reason() { + for (via, word) in [ + (Via::Name, "name"), + (Via::Alias, "alias"), + (Via::Host, "host"), + (Via::OnEnd, "on-end"), + (Via::Bound, "bound"), + (Via::Upholds, "upholds"), + ] { + assert_eq!( + Outcome::Refused(Refused::AlwaysRefused { unit: "u.service".into(), via }) + .describe(), + format!("refused: always-refused unit u.service reached by {word}") + ); + } + assert_eq!( + Outcome::Refused(Refused::Wrapper { member: "m.service".into() }).describe(), + "refused: wrapper unit, it only wraps m.service" + ); + } + + #[test] + fn a_unit_systemd_cannot_show_is_a_failure_not_a_leaf() { + let h = harness(); + h.describe("ghost.service", &[("LoadState", "not-found")]); + for verb in all_verbs() { + let id = h.accept(h.issue_unit(verb, "ghost.service")); + execute(&h.ctx, &id).unwrap(); + assert!( + matches!(h.outcome(&id), Outcome::Failed { exit_code: None, signal: None }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + } + + #[test] + fn globs_slices_scopes_and_targets_are_not_units() { + let h = harness(); + let bad = [ + "idunn*", + "idunn-*.service", + "idunn-?ggdrasil.service", + "mind-?elf@eureka.service", + "mind-self@*.service", + "mind-[s]elf@eureka.service", + "system.slice", + "poweroff.target", + "multi-user.target", + "user.slice", + "session-1.scope", + "init.scope", + "idunn-yggdrasil", + "mind-self@eureka", + "nginx", + ".service", + "", + "-idunn-x.service", + "-x.service", + "/etc/systemd/system/idunn-yggdrasil.service", + "../idunn-yggdrasil.service", + "idunn-yggdrasil.service\n", + "idunn-yggdrasil.service ", + " idunn-yggdrasil.service", + "idunn-yggdrasil.service\0", + "idunn-yggdrasil.service;reboot", + "idunn\\x2dyggdrasil.service", + "idunn%2dyggdrasil.service", + // Unicode lookalikes of a refused name. + "\u{131}dunn-yggdrasil.service", + "idunn\u{2010}yggdrasil.service", + "idunn\u{ff0d}yggdrasil.service", + "idunn-yggdrasil.service\u{200b}", + "\u{200b}idunn-yggdrasil.service", + "idunn-yggdrasil\u{2024}service", + "mind-self\u{ff20}eureka.service", + ]; + for unit in bad { + for verb in all_verbs() { + assert!( + matches!(h.issue_unit(verb, unit), Response::Refused(_)), + "{unit:?} was not refused by the grammar" + ); + } + } + let long = format!("{}.service", "a".repeat(UNIT_LIMIT)); + assert_eq!( + h.issue_unit(Verb::Start, &long), + Response::Refused(Refusal::new("unit", Code::TooLong)) + ); + assert!(!h.ctx.store.path().exists(), "the grammar refuses before any record"); + } + + #[test] + fn binding_with_a_unit_list_is_refused() { + let valid = "principal = \"eureka\"\nuser = \"mind-eureka-self\"\n\ + mind_control = \"/usr/local/bin/mind-control\"\n\ + room = \"!abc:example.org\"\nhomeserver = \"http://127.0.0.1:8008\"\n\ + token_credential = \"idunn-matrix-token\"\n"; + let binding = StewardBinding::parse(valid).unwrap(); + assert_eq!(binding.principal, "eureka"); + assert_eq!(binding.address().unwrap().port(), 8008); + for extra in [ + "units = [\"nginx.service\"]", + "allow = [\"nginx.service\"]", + "deny = [\"idunn-*\"]", + "denylist = [\"idunn-*\"]", + "refused = [\"idunn-*\"]", + "unit = \"nginx.service\"", + "[units]\nnginx = true", + ] { + assert!( + StewardBinding::parse(&format!("{valid}{extra}\n")).is_err(), + "{extra} loaded" + ); + } + let swap = |from: &str, to: &str| valid.replace(from, to); + for broken in [ + swap("user = \"mind-eureka-self\"", "user = \"root\""), + swap("user = \"mind-eureka-self\"", "user = \"\""), + swap("/usr/local/bin/mind-control", "mind-control"), + swap("!abc:example.org", "#alias:example.org"), + swap("http://127.0.0.1:8008", "http://10.77.0.1:8008"), + swap("http://127.0.0.1:8008", "https://127.0.0.1:8008"), + swap("http://127.0.0.1:8008", "http://127.0.0.1:8008/path"), + swap("idunn-matrix-token", "../token"), + swap("principal = \"eureka\"\n", ""), + ] { + assert!(StewardBinding::parse(&broken).is_err()); + } + assert!(admitted(0, 0o100600)); + assert!(admitted(0, 0o100644)); + assert!(!admitted(0, 0o100664)); + assert!(!admitted(0, 0o100666)); + assert!(!admitted(0, 0o100602)); + assert!(!admitted(1000, 0o100600)); + } + + #[test] + fn held_brake_refuses_at_start() { + let h = harness(); + h.brake.0.store(false, Ordering::SeqCst); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); + assert!(h.ctx.store.ledger().unwrap().start_of(&id).is_none()); + assert!(h.runs().is_empty()); + // The deploy path is gated the same way and writes no command. + let deploy = h.accept(h.issue(Operation::Deploy { + selector: "vili".into(), + })); + execute(&h.ctx, &deploy).unwrap(); + assert_eq!(h.outcome(&deploy), Outcome::Refused(Refused::Braked)); + assert!(!h.ctx.state_store.exists()); + // A brake that cannot be asked is a held brake. + let directory = TempDir::new().unwrap(); + assert!(!MindControl { program: directory.path().join("absent") }.clear("eureka")); + } + + #[test] + fn brake_check_is_the_exit_status_of_mind_control_check() { + let directory = TempDir::new().unwrap(); + let program = |name: &str, body: &str| { + let path = directory.path().join(name); + std::fs::write(&path, body).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + MindControl { program: path } + }; + let recorder = directory.path().join("argv"); + let clear = program( + "clear", + &format!("#!/bin/sh\necho \"$@\" > {}\nexit 0\n", recorder.display()), + ); + assert!(clear.clear("eureka")); + assert_eq!( + std::fs::read_to_string(&recorder).unwrap().trim(), + "--instance eureka check" + ); + assert!(!program("held", "#!/bin/sh\nexit 1\n").clear("eureka")); + } + + #[test] + fn started_order_is_never_rerun() { + let h = harness(); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + let waiting = h.accept(h.issue_unit(Verb::Restart, "ssh.socket")); + // The first order began and its service died: a start with no outcome. + h.ctx + .store + .put( + &id, + START_SCHEMA, + &StartRecord { + schema_version: START_SCHEMA.into(), + order_id: id.clone(), + started_at_unix_millis: 1, + actuation: format!("idunn-order-{}", order_uuid(&id)), + }, + ) + .unwrap(); + let pending = recover(&h.ctx).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Lost); + assert_eq!(pending, std::slice::from_ref(&waiting), "an order that never started waits"); + execute(&h.ctx, &id).unwrap(); + assert!(h.runs().is_empty(), "a Lost order makes no systemd call"); + // Recovery is idempotent, and a finished order is not run again. + assert_eq!(recover(&h.ctx).unwrap(), std::slice::from_ref(&waiting)); + execute(&h.ctx, &waiting).unwrap(); + execute(&h.ctx, &waiting).unwrap(); + assert_eq!(h.runs().len(), 1); + assert!(recover(&h.ctx).unwrap().is_empty()); + } + + #[test] + fn deploy_order_uses_the_up_primitive() { + let h = harness(); + let id = h.accept(h.issue(Operation::Deploy { + selector: "vili".into(), + })); + execute(&h.ctx, &id).unwrap(); + let command_id = format!("up-{}", order_uuid(&id)); + assert_eq!( + h.outcome(&id), + Outcome::Submitted { + command_id: command_id.clone() + } + ); + let ledger = h.ctx.store.ledger().unwrap(); + assert_eq!(ledger.start_of(&id).unwrap().actuation, command_id); + assert!(h.runs().is_empty(), "a deploy runs no transient unit"); + let mut shown = Vec::new(); + crate::control_plane::status( + &mut shown, + &h.ctx.state_store, + &h.directory.path().join("incidents.cc"), + Some(&command_id), + ) + .unwrap(); + let shown = String::from_utf8(shown).unwrap(); + assert_eq!(shown.lines().count(), 1, "exactly one command: {shown}"); + assert!(shown.starts_with(&format!("{command_id} vili queued")), "{shown}"); + // Re-executing writes no second command. + execute(&h.ctx, &id).unwrap(); + let mut again = Vec::new(); + crate::control_plane::status( + &mut again, + &h.ctx.state_store, + &h.directory.path().join("incidents.cc"), + None, + ) + .unwrap(); + assert_eq!(String::from_utf8(again).unwrap().lines().count(), 1); + // The result post waits for the transactions: queued is not finished. + let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); + assert_eq!(due.len(), 1); + assert_eq!(due[0].kind, PostKind::Accepted); + assert!(!deploy_finished("queued") && !deploy_finished("running")); + assert!(deploy_finished("succeeded") && deploy_finished("failed")); + assert!(deploy_finished("complete")); + } + + #[test] + fn chat_outage_delays_posts_not_orders() { + let h = harness(); + h.home.failing.store(true, Ordering::SeqCst); + let id = h.accept(h.issue_unit(Verb::Restart, "mind-heartbeat.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran); + assert_eq!(h.runs().len(), 1); + assert!(post_pass(&h.ctx).is_err()); + assert!(h.ctx.store.ledger().unwrap().posts.is_empty()); + h.home.failing.store(false, Ordering::SeqCst); + assert_eq!(post_pass(&h.ctx).unwrap(), 2); + assert_eq!(post_pass(&h.ctx).unwrap(), 0, "each post goes out once"); + let sent = h.home.sent.lock().unwrap().clone(); + assert_eq!(sent.len(), 2); + assert_eq!(sent[0].0, format!("idunn-{id}-accepted")); + assert_eq!(sent[1].0, format!("idunn-{id}-result")); + assert!(sent[0].1.contains("unit restart mind-heartbeat.service")); + assert!(sent[0].1.contains("mind-eureka-self.service")); + assert!(sent[0].1.contains("reason: \"probe\"")); + assert!(sent[1].1.contains("ran (exit 0)")); + } + + #[test] + fn failed_runs_keep_bounded_tails_that_never_leave_the_store() { + let h = harness(); + *h.systemd.report.lock().unwrap() = RunReport { + exit_code: Some(100), + signal: None, + stdout_tail: vec![b'o'; 3 * TAIL_BYTES], + stderr_tail: b"E: secret-looking \x1b[31m\xe2\x80\xae line\n".to_vec(), + }; + let id = h.accept(h.issue(Operation::PackageInstall { + packages: vec!["git-lfs".into()], + })); + execute(&h.ctx, &id).unwrap(); + assert_eq!( + h.outcome(&id), + Outcome::Failed { + exit_code: Some(100), + signal: None + } + ); + assert_eq!(h.runs().len(), 1, "the install did not follow a failed update"); + let ledger = h.ctx.store.ledger().unwrap(); + let record = ledger.outcome_of(&id).unwrap(); + assert_eq!(record.stdout_tail.len(), TAIL_BYTES); + assert!(!format!("{record:?}").contains("secret-looking")); + // Neither the post nor the journal text carries a tail. + let results: Vec<_> = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)) + .into_iter() + .filter(|due| due.kind == PostKind::Result) + .collect(); + assert_eq!(results.len(), 1); + assert!(results[0].text.contains("failed (exit 100)")); + assert!(!results[0].text.contains("secret-looking")); + // The steward reads them through the escape sink and nothing raw. + let Response::Show(view) = h.ask(&Request::Show(ShowRequest { id: id.clone() })) else { + panic!("expected a show") + }; + let mut shown = Vec::new(); + emit(&mut shown, &render_view(&view)).unwrap(); + let shown = String::from_utf8(shown).unwrap(); + assert!(shown.contains("secret-looking")); + assert!(!shown.contains('\u{1b}') && !shown.contains('\u{202e}')); + assert!(shown.contains("\\u{1b}") && shown.contains("\\u{202e}")); + assert!(shown.contains("status failed")); + // A show of an unknown or malformed id names the field only. + assert_eq!( + h.ask(&Request::Show(ShowRequest { + id: format!("ord-{}", Uuid::new_v4()) + })), + Response::Refused(Refusal::new("id", Code::Unknown)) + ); + assert_eq!( + h.ask(&Request::Show(ShowRequest { id: "ord-../x".into() })), + Response::Refused(Refusal::new("id", Code::BadShape)) + ); + let Response::List(lines) = h.ask(&Request::List) else { + panic!("expected a list") + }; + assert_eq!(lines.len(), 1); + assert_eq!(lines[0].status, "failed"); + } + + #[test] + fn a_refusal_or_error_never_echoes_an_input_value() { + let h = harness(); + let mut seen: Vec> = Vec::new(); + let mut note = |response: Response| { + let bytes = rmp_serde::to_vec_named(&response).unwrap(); + if let Response::Refused(refusal) = &response { + seen.push(refusal.to_string().into_bytes()); + } + seen.push(bytes); + }; + for unit in [ + format!("{CANARY}*.service"), + format!("{CANARY}/../idunn-orders.service"), + format!("idunn-{CANARY};x.service"), + CANARY.to_string(), + ] { + note(h.issue_unit(Verb::Restart, &unit)); + } + note(h.issue(Operation::PackageInstall { + packages: vec![format!("{CANARY}/x")], + })); + note(h.issue(Operation::PackageInstall { + packages: vec![format!("pkg={CANARY} 1")], + })); + note(h.issue(Operation::Deploy { + selector: format!("{CANARY} bad"), + })); + note(h.ask(&Request::Order(OrderRequest { + operation: Operation::UnitAction { + verb: Verb::Start, + unit: "nginx.service".into(), + }, + reason: format!("{CANARY}\n"), + }))); + note(h.ask(&Request::Order(OrderRequest { + operation: Operation::UnitAction { + verb: Verb::Start, + unit: "nginx.service".into(), + }, + reason: CANARY.repeat(40), + }))); + note(h.ask(&Request::Show(ShowRequest { + id: format!("ord-{CANARY}"), + }))); + // A frame whose own keys and values are the canary, and plain garbage. + let named: std::collections::BTreeMap> = + [( + CANARY.to_owned(), + [(CANARY.to_owned(), CANARY.to_owned())].into_iter().collect(), + )] + .into_iter() + .collect(); + note(h.exchange(|client| write_frame(client, &named).unwrap())); + note(h.exchange(|client| { + let garbage = format!("{CANARY}{CANARY}"); + client + .write_all(&(garbage.len() as u32).to_be_bytes()) + .unwrap(); + client.write_all(garbage.as_bytes()).unwrap(); + })); + note(h.exchange(|client| { + client.write_all(&u32::MAX.to_be_bytes()).unwrap(); + })); + // The client grammar fails the same way. + let mut messages = Vec::new(); + let words = |line: &[&str]| line.iter().map(|word| (*word).to_owned()).collect::>(); + for line in [ + words(&["unit", "restart", &format!("{CANARY};x.service"), "--reason", "r"]), + words(&["unit", CANARY, "nginx.service", "--reason", "r"]), + words(&["unit", "restart", "nginx.service", &format!("--{CANARY}")]), + words(&["unit", "restart", "nginx.service", "--reason", &format!("{CANARY}\n")]), + words(&[CANARY]), + words(&["package-install", &format!("{CANARY}/x"), "--reason", "r"]), + words(&["deploy", &format!("{CANARY} x"), "--reason", "r"]), + words(&["show"]), + ] { + messages.push(format!("{:#}", parse_order(line.into_iter()).unwrap_err())); + } + for line in [ + words(&[CANARY]), + words(&["serve", &format!("--{CANARY}")]), + words(&["serve", "--store"]), + ] { + messages.push(format!("{:#}", parse_serve(line.into_iter()).unwrap_err())); + } + for bytes in &seen { + assert!( + !String::from_utf8_lossy(bytes).contains(CANARY), + "a response carried an input value" + ); + } + for message in &messages { + assert!(!message.contains(CANARY), "an error carried an input value: {message}"); + } + assert!(!h.ctx.store.path().exists()); + } + + #[test] + fn cgroup_names_the_issuer_unit() { + assert_eq!( + unit_from_cgroup("0::/system.slice/claude-connector.service\n"), + "claude-connector.service" + ); + // What a delegated subtree adds below the unit is the peer's to name. + assert_eq!( + unit_from_cgroup("0::/system.slice/huginn@eureka.service/idunn-yggdrasil.service/leaf\n"), + "huginn@eureka.service" + ); + assert_eq!( + unit_from_cgroup("0::/system.slice/system-getty.slice/getty@tty1.service\n"), + "getty@tty1.service" + ); + assert_eq!( + unit_from_cgroup("12:cpu:/x\n1:name=systemd:/system.slice/a.scope\n"), + "a.scope" + ); + // A user manager's units, and paths no manager made, name no one. + for path in [ + "0::/user.slice/user-985.slice/user@985.service/app.slice/idunn-yggdrasil.service\n", + "0::/user.slice/user-1000.slice/session-3.scope\n", + "0::/system.slice/not-a-unit/idunn-yggdrasil.service\n", + "0::/system.slice/\n", + "0::/\n", + "", + ] { + assert_eq!(unit_from_cgroup(path), "unknown", "{path}"); + } + } + + #[test] + fn systemctl_show_output_is_read_into_facts() { + let facts = parse_show( + "Id=a.service\nNames=a.service b.service\nLoadState=loaded\nActiveState=active\n\ + SuccessAction=none\nFailureAction=reboot-force\nStartLimitAction=\nJobTimeoutAction=none\n\ + OnFailure=f.service g.service\nOnSuccess=\nBoundBy=\"q\\\\x2dr.service\" s.service\n\ + Upholds=u.service\nConsistsOf=m.service n.service\nRequires=r.service\nConflicts=x.service\nOther=x\n", + ) + .unwrap(); + assert_eq!(facts.id, "a.service"); + assert_eq!(facts.names, ["a.service", "b.service"]); + assert!(!facts.not_found); + assert_eq!(facts.on_end, ["FailureAction", "OnFailure", "OnFailure"]); + // A quoted name holds one backslash, not two and not a quote mark. + assert_eq!(facts.bound_by, ["q\\x2dr.service", "s.service"]); + assert_eq!(facts.upholds, ["u.service"]); + assert_eq!(facts.consists_of, ["m.service", "n.service"]); + assert!(parse_show("Names=a.service\n").is_err()); + // The request asks for exactly the properties the parse reads. + assert_eq!( + show_properties(), + "Id,Names,LoadState,SuccessAction,FailureAction,StartLimitAction,JobTimeoutAction,\ + OnFailure,OnSuccess,BoundBy,Upholds,ConsistsOf" + ); + // A backslash outside quotes is a backslash; inside, it escapes one character. + assert_eq!(parse_names("a\\b c\\\\d"), ["a\\b", "c\\\\d"]); + assert_eq!(parse_names("\"a\\\"b\" \"\" c"), ["a\"b", "", "c"]); + assert!(parse_names(" ").is_empty()); + let gone = parse_show("Id=\\x22a.service\\x22.service\nLoadState=not-found\n").unwrap(); + assert!(gone.not_found); + } + + #[test] + fn the_store_refuses_a_second_record_and_a_foreign_document() { + let h = harness(); + let order = OrderRecord { + schema_version: ORDER_SCHEMA.into(), + order_id: "ord-x".into(), + principal: "eureka".into(), + issuer_uid: 1, + issuer_unit: "u.service".into(), + issuer_invocation: String::new(), + issued_at_unix_millis: 1, + operation: Operation::Deploy { + selector: "vili".into(), + }, + reason: "r".into(), + }; + assert!(h.ctx.store.put("ord-x", ORDER_SCHEMA, &order).unwrap()); + assert!(!h.ctx.store.put("ord-x", ORDER_SCHEMA, &order).unwrap()); + assert_eq!( + std::fs::metadata(h.ctx.store.path()).unwrap().permissions().mode() & 0o777, + 0o600 + ); + let foreign = typed_envelope("k", "idunn.other", "idunn.other.v1", &1u8, 1).unwrap(); + SingleFileMessagePackBackingStore::new(h.ctx.store.path()) + .compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: "idunn.other".into(), + key: "k".into(), + current: None, + }], + &[foreign], + ) + .unwrap(); + assert!(h.ctx.store.ledger().is_err()); + } + + #[test] + fn synapse_post_is_one_idempotent_put_on_loopback() { + let directory = TempDir::new().unwrap(); + let token_file = directory.path().join("token"); + std::fs::write(&token_file, "tok-123\n").unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let (sent, requests) = mpsc::channel(); + let server = thread::spawn(move || { + for reply in [ + "HTTP/1.0 200 OK\r\nContent-Type: application/json\r\n\r\n{\"event_id\":\"$ev1\"}", + "HTTP/1.0 403 Forbidden\r\n\r\n{\"errcode\":\"M_FORBIDDEN\"}", + ] { + let (mut socket, _) = listener.accept().unwrap(); + let mut received = Vec::new(); + let mut chunk = [0u8; 1024]; + while !received.windows(4).any(|window| window == b"\r\n\r\n") + || !String::from_utf8_lossy(&received).contains("\"body\"") + { + let count = socket.read(&mut chunk).unwrap(); + if count == 0 { + break; + } + received.extend_from_slice(&chunk[..count]); + } + sent.send(String::from_utf8_lossy(&received).into_owned()).unwrap(); + socket.write_all(reply.as_bytes()).unwrap(); + } + }); + let synapse = Synapse { + address, + room: "!room:example.org".into(), + token_file, + reply_within: Duration::from_secs(10), + }; + assert_eq!(synapse.post("txn-1", "hello \"quoted\"").unwrap(), "$ev1"); + let request = requests.recv().unwrap(); + assert!(request.starts_with( + "PUT /_matrix/client/v3/rooms/%21room%3Aexample.org/send/m.room.message/txn-1 HTTP/1.0\r\n" + )); + assert!(request.contains("Authorization: Bearer tok-123\r\n")); + assert!(request.contains("{\"body\":\"hello \\\"quoted\\\"\",\"msgtype\":\"m.text\"}")); + let error = synapse.post("txn-2", "x").unwrap_err(); + assert!(!format!("{error:#}").contains("tok-123")); + server.join().unwrap(); + assert!( + Synapse { + address, + room: "!r:e".into(), + token_file: directory.path().join("absent"), + reply_within: Duration::from_secs(10), + } + .post("t", "x") + .is_err() + ); + } + + #[test] + fn grammar_limits_are_exact_at_their_edges() { + let h = harness(); + let order = |operation: Operation, reason: &str| { + h.ask(&Request::Order(OrderRequest { + operation, + reason: reason.into(), + })) + }; + let packages = |names: Vec| Operation::PackageInstall { packages: names }; + let unit = |name: String| Operation::UnitAction { + verb: Verb::Start, + unit: name, + }; + // Package: two characters is the shortest name, 200 bytes the longest. + assert!(matches!(order(packages(vec!["ab".into()]), "r"), Response::Accepted { .. })); + assert!(matches!(order(packages(vec!["a".repeat(200)]), "r"), Response::Accepted { .. })); + assert_eq!( + order(packages(vec!["a".repeat(201)]), "r"), + Response::Refused(Refusal::new("package", Code::TooLong)) + ); + assert_eq!( + order(packages(vec!["a".into()]), "r"), + Response::Refused(Refusal::new("package", Code::BadShape)) + ); + let sixteen: Vec = (0..16).map(|n| format!("pkg-{n}x")).collect(); + assert!(matches!(order(packages(sixteen), "r"), Response::Accepted { .. })); + // Unit: 200 bytes with its suffix is the longest. + let longest = format!("{}.service", "a".repeat(UNIT_LIMIT - ".service".len())); + assert_eq!(longest.len(), UNIT_LIMIT); + assert!(matches!(order(unit(longest), "r"), Response::Accepted { .. })); + let too_long = format!("{}.service", "a".repeat(UNIT_LIMIT - ".service".len() + 1)); + assert_eq!( + order(unit(too_long), "r"), + Response::Refused(Refusal::new("unit", Code::TooLong)) + ); + // Reason: 300 bytes is the longest, and it must be there. + assert!(matches!( + order(unit("nginx.service".into()), &"r".repeat(REASON_LIMIT)), + Response::Accepted { .. } + )); + assert_eq!( + order(unit("nginx.service".into()), &"r".repeat(REASON_LIMIT + 1)), + Response::Refused(Refusal::new("reason", Code::TooLong)) + ); + assert_eq!( + order(unit("nginx.service".into()), ""), + Response::Refused(Refusal::new("reason", Code::Empty)) + ); + assert_eq!( + order(unit("nginx.service".into()), "caf\u{e9}"), + Response::Refused(Refusal::new("reason", Code::BadCharacter)) + ); + } + + #[test] + fn a_concluded_order_is_never_run_later() { + let h = harness(); + h.brake.0.store(false, Ordering::SeqCst); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); + // Someone else clears the brake; the refused order stays refused. + h.brake.0.store(true, Ordering::SeqCst); + execute(&h.ctx, &id).unwrap(); + assert!(h.runs().is_empty()); + let ledger = h.ctx.store.ledger().unwrap(); + assert!(ledger.start_of(&id).is_none()); + assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); + } + + #[test] + fn the_orders_cli_has_no_command_string() { + let parse = |line: &[&str]| parse_order(line.iter().map(|word| (*word).to_owned())); + let parsed = parse(&["unit", "restart", "nginx.service", "--reason", "why"]).unwrap(); + assert_eq!( + parsed.action, + OrderAction::Issue { + operation: Operation::UnitAction { + verb: Verb::Restart, + unit: "nginx.service".into() + }, + reason: "why".into() + } + ); + assert!(parse(&["package-install", "git-lfs", "socat", "--reason", "r"]).is_ok()); + assert!(parse(&["deploy", "profile:aetheria", "--reason", "r"]).is_ok()); + assert!(parse(&["show", "ord-x"]).is_ok()); + assert!(parse(&["list"]).is_ok()); + for refused in [ + vec!["unit", "restart", "nginx.service", "--reason", "r", "--command", "reboot-now"], + vec!["unit", "restart", "nginx.service", "--reason", "r", "--shell"], + vec!["package-install", "git-lfs", "--reason", "r", "--command", "x"], + vec!["run", "sh", "--reason", "r"], + vec!["exec", "--reason", "r"], + vec!["unit", "enable", "nginx.service", "--reason", "r"], + vec!["unit", "restart", "--reason", "r"], + vec!["unit", "restart", "a.service", "b.service", "--reason", "r"], + vec!["unit", "restart", "nginx.service"], + vec!["unit", "restart", "nginx*", "--reason", "r"], + vec!["package-install", "--reason", "r"], + vec!["package-install", "-y", "x", "--reason", "r"], + vec!["deploy", "--reason", "r"], + vec!["deploy", "a", "b", "--reason", "r"], + vec!["list", "extra"], + vec!["show"], + vec![], + ] { + assert!(parse(&refused).is_err(), "{refused:?} parsed"); + } + } + + #[test] + fn an_order_runs_only_under_the_steward_bound_now() { + let mut h = harness(); + let first = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + let second = h.accept(h.issue_unit(Verb::Restart, "ssh.socket")); + // The binding changes while both wait: another uid, then another + // principal with the old uid. Each is refused with a record, before + // the brake is asked and before anything runs. + h.ctx.steward_uid = STEWARD + 7; + execute(&h.ctx, &first).unwrap(); + h.ctx.steward_uid = STEWARD; + h.ctx.principal = "void".into(); + execute(&h.ctx, &second).unwrap(); + for id in [&first, &second] { + assert_eq!(h.outcome(id), Outcome::Refused(Refused::NotSteward)); + assert!(h.ctx.store.ledger().unwrap().start_of(id).is_none()); + } + assert!(h.runs().is_empty()); + assert!(h.brake.1.lock().unwrap().is_empty(), "a revoked steward's brake is never the gate"); + // The refusal is posted like any other. + let ledger = h.ctx.store.ledger().unwrap(); + let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); + assert!(due.iter().any(|due| due.kind == PostKind::Result + && due.text.contains("refused: the issuer is not the current steward"))); + // The other direction: an order the bound steward issued after the + // rebinding, and a recovered one under an unchanged binding, run and + // are gated by the bound principal's brake. + let third = h.accept(h.issue_unit(Verb::Restart, "cron.service")); + let resumed = harness(); + let waiting = resumed.accept(resumed.issue_unit(Verb::Restart, "nginx.service")); + assert_eq!(recover(&resumed.ctx).unwrap(), std::slice::from_ref(&waiting)); + execute(&resumed.ctx, &waiting).unwrap(); + execute(&h.ctx, &third).unwrap(); + assert_eq!(resumed.outcome(&waiting), Outcome::Ran); + assert_eq!(h.outcome(&third), Outcome::Ran); + assert_eq!(*h.brake.1.lock().unwrap(), ["void".to_owned()]); + assert_eq!(*resumed.brake.1.lock().unwrap(), ["eureka".to_owned()]); + } + + #[test] + fn root_reads_orders_and_never_issues_one() { + let mut h = harness(); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + h.ctx.peers = Arc::new(FakePeers(0)); + let refused = Response::Refused(Refusal::new("peer", Code::Unauthorized)); + assert_eq!(h.issue_unit(Verb::Restart, "cron.service"), refused); + let Response::Show(view) = h.ask(&Request::Show(ShowRequest { id: id.clone() })) else { + panic!("root could not read an order"); + }; + assert_eq!(view.order.order_id, id); + let Response::List(lines) = h.ask(&Request::List) else { + panic!("root could not list orders"); + }; + assert_eq!(lines.len(), 1, "root's refused order left no record"); + h.ctx.peers = Arc::new(FakePeers(STEWARD + 1)); + assert_eq!(h.ask(&Request::Show(ShowRequest { id })), refused); + } + + #[test] + fn every_order_posts_once_in_its_turn() { + let h = harness(); + let ids: Vec = ["nginx.service", "ssh.socket", "cron.service"] + .into_iter() + .map(|unit| h.accept(h.issue_unit(Verb::Restart, unit))) + .collect(); + for id in &ids { + execute(&h.ctx, id).unwrap(); + } + assert_eq!(post_pass(&h.ctx).unwrap(), 6, "an accepted and a result post for each"); + assert_eq!(post_pass(&h.ctx).unwrap(), 0); + let sent = h.home.sent.lock().unwrap().clone(); + let mut keys: Vec<_> = sent.iter().map(|(txn, _)| txn.clone()).collect(); + keys.sort(); + let mut expected: Vec = ids + .iter() + .flat_map(|id| [format!("idunn-{id}-accepted"), format!("idunn-{id}-result")]) + .collect(); + expected.sort(); + assert_eq!(keys, expected); + // An order accepted later is posted by the next pass alone. + let late = h.accept(h.issue_unit(Verb::Restart, "mind-heartbeat.service")); + assert_eq!(post_pass(&h.ctx).unwrap(), 1); + execute(&h.ctx, &late).unwrap(); + assert_eq!(post_pass(&h.ctx).unwrap(), 1); + } + + #[test] + fn a_deploys_result_is_posted_once_its_command_has_finished() { + let h = harness(); + let id = h.accept(h.issue(Operation::Deploy { + selector: "vili".into(), + })); + execute(&h.ctx, &id).unwrap(); + let ledger = h.ctx.store.ledger().unwrap(); + let results = |state: Option<(&'static str, &str)>| { + let owed = due_posts(&ledger, &|_| match state { + Some((state, detail)) => Ok((state, detail.to_owned())), + None => Err(anyhow!("the command cannot be read")), + }); + owed.into_iter().filter(|due| due.kind == PostKind::Result).collect::>() + }; + assert!(results(Some(("queued", ""))).is_empty()); + assert!(results(Some(("running", "vili:Sealing"))).is_empty()); + assert!(results(None).is_empty()); + for state in ["succeeded", "failed"] { + let owed = results(Some((state, "vili"))); + assert_eq!(owed.len(), 1, "{state}"); + assert!(owed[0].text.contains(&format!("{state} vili")), "{}", owed[0].text); + assert!(owed[0].text.contains("submitted deployment up-")); + } + } + + #[test] + fn intake_gives_up_on_a_peer_that_sends_no_frame() { + let h = harness(); + let (mut client, mut server) = UnixStream::pair().unwrap(); + client.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + // Half a length prefix, then nothing: the connection must end by itself. + client.write_all(&[0, 0]).unwrap(); + let (done, ended) = mpsc::channel(); + let worker = thread::spawn(move || { + let result = handle(&h.ctx, &mut server, &h.queue); + let _ = done.send(result.is_ok()); + h + }); + let answered = ended + .recv_timeout(Duration::from_secs(20)) + .expect("intake never gave up on a silent peer"); + assert!(answered, "a silent peer is refused as malformed, not left as an error"); + let h = worker.join().unwrap(); + assert_eq!( + rmp_serde::from_slice::(&read_frame(&mut client, RESPONSE_FRAME_LIMIT).unwrap()).unwrap(), + Response::Refused(Refusal::new("frame", Code::Malformed)) + ); + assert!(!h.ctx.store.path().exists()); + } + + #[test] + fn a_record_under_another_schema_is_not_read() { + let put = |h: &Harness, kind: &str, schema: &str, key: &str| match kind { + "order" => h.ctx.store.put( + key, + schema, + &OrderRecord { + schema_version: schema.into(), + order_id: key.into(), + principal: "eureka".into(), + issuer_uid: 1, + issuer_unit: "u.service".into(), + issuer_invocation: String::new(), + issued_at_unix_millis: 1, + operation: Operation::Deploy { + selector: "vili".into(), + }, + reason: "r".into(), + }, + ), + "start" => h.ctx.store.put( + key, + schema, + &StartRecord { + schema_version: schema.into(), + order_id: key.into(), + started_at_unix_millis: 1, + actuation: "idunn-order-x".into(), + }, + ), + "outcome" => h.ctx.store.put( + key, + schema, + &OutcomeRecord { + schema_version: schema.into(), + order_id: key.into(), + ended_at_unix_millis: 1, + outcome: Outcome::Lost, + stdout_tail: Vec::new(), + stderr_tail: Vec::new(), + }, + ), + _ => h.ctx.store.put( + key, + schema, + &PostRecord { + schema_version: schema.into(), + post_key: key.into(), + event_id: "$e".into(), + posted_at_unix_millis: 1, + }, + ), + }; + for (kind, current, key) in [ + ("order", ORDER_SCHEMA, "ord-a"), + ("start", START_SCHEMA, "ord-a"), + ("outcome", OUTCOME_SCHEMA, "ord-a"), + ("post", POST_SCHEMA, "ord-a:accepted"), + ] { + let h = harness(); + assert!(put(&h, kind, current, key).unwrap()); + assert!(h.ctx.store.ledger().is_ok(), "{current} is read"); + let h = harness(); + let other = current.replace(".v1", ".v0"); + assert!(put(&h, kind, &other, key).unwrap()); + assert!(h.ctx.store.ledger().is_err(), "{other} is not read"); + } + } + + #[test] + fn posts_are_retried_after_a_widening_pause() { + let pauses: Vec = (0..9).map(post_pause).collect(); + assert_eq!(pauses, [2, 4, 8, 16, 32, 60, 60, 60, 60]); + } + + #[test] + fn serve_options_are_read_from_every_flag() { + let parse = |line: &[&str]| parse_serve(line.iter().map(|word| (*word).to_owned())); + let defaults = parse(&["serve"]).unwrap(); + assert_eq!(defaults.store, PathBuf::from(DEFAULT_STORE)); + assert_eq!(defaults.steward_binding, PathBuf::from(DEFAULT_BINDING)); + assert_eq!(defaults.socket, PathBuf::from(DEFAULT_SOCKET)); + let given = parse(&[ + "serve", + "--state-store", + "/a/control.cc", + "--store", + "/b", + "--steward-binding", + "/c.toml", + "--socket", + "/d.sock", + ]) + .unwrap(); + assert_eq!( + given, + ServeOptions { + state_store: "/a/control.cc".into(), + store: "/b".into(), + steward_binding: "/c.toml".into(), + socket: "/d.sock".into(), + } + ); + assert!(parse(&["serve", "--store"]).is_err()); + assert!(parse(&["serve", "--other", "/x"]).is_err()); + assert!(parse(&["start"]).is_err()); + } + + #[test] + fn the_steward_user_is_a_real_user_and_never_root() { + assert!(user_uid("root").is_err()); + assert!(user_uid("no-such-user-here").is_err()); + assert!(user_uid("bad\0name").is_err()); + assert_eq!(user_uid("nobody").unwrap(), 65534); + } + + #[test] + fn the_orders_directory_is_made_private() { + let directory = TempDir::new().unwrap(); + let store = OrderStore::new(&directory.path().join("orders")); + store.prepare().unwrap(); + let mode = std::fs::metadata(directory.path().join("orders")).unwrap().mode() & 0o777; + assert_eq!(mode, 0o700); + } + + #[test] + fn the_socket_belongs_to_the_steward_alone() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("steward.sock"); + let me = std::fs::metadata(directory.path()).unwrap().uid(); + let _listener = bind_socket(&path, me).unwrap(); + let metadata = std::fs::metadata(&path).unwrap(); + assert_eq!((metadata.mode() & 0o777, metadata.uid()), (0o600, me)); + // A stale socket is replaced; anything else in the way is not. + let _again = bind_socket(&path, me).unwrap(); + let other = directory.path().join("other"); + std::fs::write(&other, "x").unwrap(); + assert!(bind_socket(&other, me).is_err()); + } + + #[test] + fn synapse_gives_up_on_a_reply_that_never_ends() { + let directory = TempDir::new().unwrap(); + let token_file = directory.path().join("token"); + std::fs::write(&token_file, "tok-123\n").unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + thread::spawn(move || { + // The first connection stays silent; the second drips a byte at a time. + for (index, stream) in listener.incoming().enumerate() { + let Ok(mut socket) = stream else { continue }; + thread::spawn(move || { + let mut chunk = [0u8; 4096]; + let _ = socket.read(&mut chunk); + for byte in b"HTTP/1.0 200 OK\r\n".iter().cycle().take(200) { + if index == 0 || socket.write_all(&[*byte]).is_err() { + break; + } + thread::sleep(Duration::from_millis(40)); + } + thread::sleep(Duration::from_secs(5)); + }); + } + }); + let synapse = Synapse { + address, + room: "!r:e".into(), + token_file, + reply_within: Duration::from_millis(600), + }; + let synapse = Arc::new(synapse); + for _ in 0..2 { + let started = std::time::Instant::now(); + let (done, ended) = mpsc::channel(); + let poster = Arc::clone(&synapse); + thread::spawn(move || { + let _ = done.send(poster.post("t", "x").map_err(|error| format!("{error:#}"))); + }); + let error = ended + .recv_timeout(Duration::from_secs(4)) + .expect("the post never gave up") + .unwrap_err(); + assert!(started.elapsed() < Duration::from_secs(3), "{error}"); + assert!(!error.contains("tok-123")); + } + } + + /// The whole service, as root, against fake `systemctl`, `systemd-run` + /// and a loopback homeserver, with the steward connecting from another + /// uid. It runs only where the test process is root (the verify container). + #[test] + fn serve_end_to_end_as_root() { + // SAFETY: geteuid reads the process credentials. + if unsafe { libc::geteuid() } != 0 { + eprintln!("serve_end_to_end_as_root: skipped, not root"); + return; + } + let directory = TempDir::new().unwrap(); + let root = directory.path().to_owned(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o755)).unwrap(); + let bin = root.join("bin"); + std::fs::create_dir(&bin).unwrap(); + let systemctl = format!( + "#!/bin/sh\necho \"$@\" >> {0}/systemctl.log\nfor a; do last=$a; done\n\ + case \"$*\" in *InvocationID*quiet*) ;; *InvocationID*broken*) echo junk; exit 1;; *InvocationID*) echo inv-fake;; *show*) printf 'Id=%s\\nNames=%s\\n' \"$last\" \"$last\";; esac\n", + root.display() + ); + let run = format!( + "#!/bin/sh\necho \"$@\" >> {0}/run.log\nif [ -e {0}/fail ]; then echo boom >&2; exit 3; fi\nexit 0\n", + root.display() + ); + for (name, text) in [("systemctl", systemctl), ("systemd-run", run)] { + let path = bin.join(name); + std::fs::write(&path, text).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + let credentials = root.join("cred"); + std::fs::create_dir(&credentials).unwrap(); + std::fs::write(credentials.join("idunn-matrix-token"), "tok-1\n").unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let (posted, posts) = mpsc::channel::(); + thread::spawn(move || { + for (index, stream) in listener.incoming().enumerate() { + let Ok(mut socket) = stream else { continue }; + let mut received = Vec::new(); + let mut chunk = [0u8; 4096]; + loop { + let text = String::from_utf8_lossy(&received).into_owned(); + if let Some(split) = text.find("\r\n\r\n") { + let length = text + .lines() + .find_map(|line| line.strip_prefix("Content-Length: ")) + .and_then(|value| value.trim().parse::().ok()) + .unwrap_or(0); + if received.len() >= split + 4 + length { + break; + } + } + let count = socket.read(&mut chunk).unwrap_or(0); + if count == 0 { + break; + } + received.extend_from_slice(&chunk[..count]); + } + let _ = posted.send(String::from_utf8_lossy(&received).into_owned()); + let _ = socket.write_all( + format!("HTTP/1.0 200 OK\r\n\r\n{{\"event_id\":\"$e{index}\"}}").as_bytes(), + ); + } + }); + let binding = root.join("steward.toml"); + std::fs::write( + &binding, + format!( + "principal = \"eureka\"\nuser = \"nobody\"\nmind_control = \"/bin/true\"\n\ + room = \"!abc:example.org\"\nhomeserver = \"http://{address}\"\n\ + token_credential = \"idunn-matrix-token\"\n" + ), + ) + .unwrap(); + std::fs::set_permissions(&binding, std::fs::Permissions::from_mode(0o644)).unwrap(); + let run_directory = root.join("run"); + std::fs::create_dir(&run_directory).unwrap(); + std::fs::set_permissions(&run_directory, std::fs::Permissions::from_mode(0o755)).unwrap(); + let socket = run_directory.join("steward.sock"); + let store = root.join("orders"); + let control = root.join("state").join("control.cc"); + // SAFETY: the only test that reads these variables is this one; PATH + // keeps its old entries. + unsafe { + std::env::set_var( + "PATH", + format!("{}:{}", bin.display(), std::env::var("PATH").unwrap_or_default()), + ); + std::env::set_var("CREDENTIALS_DIRECTORY", &credentials); + } + let options = ServeOptions { + state_store: control.clone(), + store: store.clone(), + steward_binding: binding, + socket: socket.clone(), + }; + // The unit's umask must not decide the modes of what the service writes. + // SAFETY: umask only sets the process file-mode mask. + let before = unsafe { libc::umask(0o117) }; + let (ended, serve_end) = mpsc::channel::(); + thread::spawn(move || { + let result = serve(options); + let _ = ended.send(format!("{result:?}")); + }); + for _ in 0..100 { + if socket.exists() || serve_end.try_recv().is_ok() { + break; + } + thread::sleep(Duration::from_millis(100)); + } + assert!(socket.exists(), "serve never bound its socket"); + let metadata = std::fs::metadata(&socket).unwrap(); + assert_eq!((metadata.mode() & 0o777, metadata.uid()), (0o600, 65534)); + // Root is not the steward: it may read, it may not order. + let unit = |name: &str| { + Request::Order(OrderRequest { + operation: Operation::UnitAction { + verb: Verb::Restart, + unit: name.into(), + }, + reason: "end to end".into(), + }) + }; + let as_root = |request: &Request| { + let mut stream = UnixStream::connect(&socket).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + write_frame(&mut stream, request).unwrap(); + rmp_serde::from_slice::(&read_frame(&mut stream, RESPONSE_FRAME_LIMIT).unwrap()).unwrap() + }; + assert_eq!( + as_root(&unit("nginx.service")), + Response::Refused(Refusal::new("peer", Code::Unauthorized)) + ); + assert_eq!(as_root(&Request::List), Response::List(Vec::new())); + let orders = OrderStore::new(&store); + let finished = |order_id: &str| { + for _ in 0..200 { + if let Some(record) = orders.ledger().unwrap().outcome_of(order_id) { + return record.clone(); + } + thread::sleep(Duration::from_millis(100)); + } + panic!("{order_id} never concluded"); + }; + // The steward is uid 65534 with gid 0: a service that read the gid for + // the uid would take it for root. + let first = steward_asks(&socket, &unit("nginx.service")); + let Response::Accepted { order_id: first } = first else { + panic!("the steward was not accepted: {first:?}"); + }; + assert_eq!(finished(&first).outcome, Outcome::Ran); + std::fs::write(root.join("fail"), "").unwrap(); + let Response::Accepted { order_id: second } = steward_asks(&socket, &unit("cron.service")) else { + panic!("the second order was not accepted"); + }; + let failed = finished(&second); + assert_eq!( + failed.outcome, + Outcome::Failed { + exit_code: Some(3), + signal: None + } + ); + assert_eq!(failed.stderr_tail, b"boom\n"); + let deploy = Request::Order(OrderRequest { + operation: Operation::Deploy { + selector: "vili".into(), + }, + reason: "end to end".into(), + }); + let Response::Accepted { order_id: third } = steward_asks(&socket, &deploy) else { + panic!("the deploy was not accepted"); + }; + assert!(matches!(finished(&third).outcome, Outcome::Submitted { .. })); + let ledger = orders.ledger().unwrap(); + let stored = ledger.orders.iter().find(|order| order.order_id == first).unwrap(); + assert_eq!((stored.issuer_uid, stored.issuer_invocation.as_str()), (65534, "inv-fake")); + // An invocation id is what a succeeding systemctl prints, if anything. + assert_eq!(HostSystemd.invocation("quiet.service"), None); + assert_eq!(HostSystemd.invocation("broken.service"), None); + assert_eq!(HostSystemd.invocation("some.service").as_deref(), Some("inv-fake")); + // What the service called, exactly. + let run_log = std::fs::read_to_string(root.join("run.log")).unwrap(); + assert!( + run_log.contains(&format!( + "--unit idunn-order-{} --wait --pipe --collect -p RuntimeMaxSec=15min -- systemctl restart --job-mode=ignore-requirements -- nginx.service", + order_uuid(&first) + )), + "{run_log}" + ); + let systemctl_log = std::fs::read_to_string(root.join("systemctl.log")).unwrap(); + assert!( + systemctl_log.contains(&format!("show --no-pager --property={} -- nginx.service", show_properties())), + "{systemctl_log}" + ); + // Every record is posted; the deploy's result waits for its command. + let mut seen = Vec::new(); + while seen.len() < 5 { + seen.push(posts.recv_timeout(Duration::from_secs(10)).expect("five posts within 10 s")); + } + for (id, kinds) in [ + (&first, ["accepted", "result"].as_slice()), + (&second, &["accepted", "result"]), + (&third, &["accepted"]), + ] { + for kind in kinds { + let wanted = format!("/send/m.room.message/idunn-{id}-{kind} HTTP/1.0"); + assert!(seen.iter().any(|post| post.contains(&wanted)), "no post {wanted}"); + } + } + // Modes: the store is private, and the deploy wrote control.cc under + // Idunn's umask, not the 0117 the process was started with. + let mode = |path: &Path| std::fs::metadata(path).unwrap().mode() & 0o777; + assert_eq!((mode(&store), mode(&store.join("orders.cc"))), (0o700, 0o600)); + let control_mode = mode(&control); + // SAFETY: umask only sets the process file-mode mask. + unsafe { libc::umask(before) }; + assert_eq!(control_mode, 0o640); + } + + /// One request as the steward: uid 65534 and gid 0, from a forked child + /// that only makes system calls. + fn steward_asks(socket: &Path, request: &Request) -> Response { + let body = rmp_serde::to_vec_named(request).unwrap(); + let mut frame = (body.len() as u32).to_be_bytes().to_vec(); + frame.extend_from_slice(&body); + // SAFETY: plain data. + let mut address: libc::sockaddr_un = unsafe { std::mem::zeroed() }; + address.sun_family = libc::AF_UNIX as libc::sa_family_t; + for (index, byte) in socket.as_os_str().as_encoded_bytes().iter().enumerate() { + address.sun_path[index] = *byte as libc::c_char; + } + let mut fds = [0i32; 2]; + // SAFETY: `fds` holds two descriptors. + assert_eq!(unsafe { libc::pipe(fds.as_mut_ptr()) }, 0); + // SAFETY: the child allocates nothing and ends with _exit. + let pid = unsafe { libc::fork() }; + if pid == 0 { + // SAFETY: only system calls on buffers made before the fork. + unsafe { + libc::close(fds[0]); + libc::alarm(10); + if libc::setgroups(0, std::ptr::null()) != 0 + || libc::setgid(0) != 0 + || libc::setuid(65534) != 0 + { + libc::_exit(2); + } + let fd = libc::socket(libc::AF_UNIX, libc::SOCK_STREAM, 0); + if libc::connect( + fd, + (&address as *const libc::sockaddr_un).cast(), + std::mem::size_of::() as libc::socklen_t, + ) != 0 + { + libc::_exit(3); + } + let mut offset = 0usize; + while offset < frame.len() { + let count = libc::write(fd, frame.as_ptr().add(offset).cast(), frame.len() - offset); + if count <= 0 { + libc::_exit(4); + } + offset += count as usize; + } + let mut buffer = [0u8; 4096]; + loop { + let count = libc::read(fd, buffer.as_mut_ptr().cast(), buffer.len()); + if count <= 0 { + break; + } + libc::write(fds[1], buffer.as_ptr().cast(), count as usize); + } + libc::_exit(0); + } + } + // SAFETY: the parent's copy of the write end. + unsafe { libc::close(fds[1]) }; + let mut reply = Vec::new(); + // SAFETY: `fds[0]` is an open descriptor this function now owns. + let mut pipe = unsafe { ::from_raw_fd(fds[0]) }; + pipe.read_to_end(&mut reply).unwrap(); + let mut status = 0; + // SAFETY: `pid` is our child. + unsafe { libc::waitpid(pid, &mut status, 0) }; + assert!(libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0, "child status {status}"); + rmp_serde::from_slice(&read_frame(&mut reply.as_slice(), RESPONSE_FRAME_LIMIT).unwrap()).unwrap() + } + + fn host_refusal(verb: Verb, unit: &str) -> Option { + refusal_for(&HostSystemd, unit, verb).unwrap_or_else(|error| panic!("{unit}: {error}")) + } + + #[test] + #[ignore = "reads the real unit set of a systemd host; run on Yggdrasil, read-only"] + fn real_host_unit_set() { + for name in [ + "bifrost", "forgejo", "forgejo-eureka", "nginx", "matrix-synapse", "redis-server", + "huginn", "huginn@void", "mind-heartbeat", "heimdall", "ssh", "docker", "containerd", + "postgresql@16-main", "claude-connector", "persona-matrix-mouth", "voidbot", + "wg-quick@wg-gamecult", "cron", "apt-daily", + ] { + let unit = format!("{name}.service"); + for verb in all_verbs() { + assert_eq!(host_refusal(verb, &unit), None, "{verb:?} {unit}"); + } + } + for name in [ + "initrd-cleanup", "initrd-parse-etc", "initrd-switch-root", "system-update-cleanup", + "systemd-battery-check", "systemd-exit", "systemd-fsck-root", "systemd-halt", + "systemd-kexec", "systemd-poweroff", "systemd-reboot", "systemd-soft-reboot", + "systemd-storagetm", + ] { + let unit = format!("{name}.service"); + for verb in [Verb::Start, Verb::Stop, Verb::Restart] { + assert_eq!( + host_refusal(verb, &unit), + Some(refused(&unit, Via::OnEnd)), + "{verb:?} {unit}" + ); + } + } + for unit in HOST_NAMES { + for verb in all_verbs() { + assert_eq!(host_refusal(verb, unit), Some(refused(unit, Via::Host)), "{verb:?} {unit}"); + } + } + for unit in [ + "systemd-networkd.service", + "user-runtime-dir@1000.service", + "user-runtime-dir@105.service", + "user-runtime-dir@978.service", + ] { + assert_eq!(host_refusal(Verb::Start, unit), None, "{unit}"); + assert!( + matches!( + host_refusal(Verb::Stop, unit), + Some(Refused::AlwaysRefused { via: Via::Bound, .. }) + ), + "{unit}" + ); + } + let listing = Command::new("systemctl") + .args(["list-units", "--all", "--plain", "--no-legend", "--no-pager", "--full"]) + .output() + .unwrap(); + let listing = String::from_utf8_lossy(&listing.stdout).into_owned(); + let units: Vec<&str> = listing + .lines() + .filter_map(|line| line.split_whitespace().next()) + .filter(|unit| validate_unit(unit).is_ok()) + .collect(); + for verb in all_verbs() { + let mut counts = BTreeMap::new(); + for unit in &units { + let key = match refusal_for(&HostSystemd, unit, verb) { + Ok(None) => "ok".to_owned(), + Ok(Some(Refused::AlwaysRefused { via, .. })) => format!("{via:?}"), + Ok(Some(other)) => format!("{other:?}"), + Err(_) => "unresolved".to_owned(), + }; + *counts.entry(key).or_insert(0usize) += 1; + } + println!("{verb:?} over {} units: {counts:?}", units.len()); + } + } + + #[test] + #[ignore = "starts throwaway units in the user manager; run on Yggdrasil"] + fn one_job_argv_against_a_real_manager() { + let directory = PathBuf::from(std::env::var("HOME").unwrap()).join(".config/systemd/user"); + std::fs::create_dir_all(&directory).unwrap(); + let (a, b) = ("onejob-probe-a.service", "onejob-probe-b.service"); + let body = "[Service]\nExecStart=/bin/sleep 600\n"; + std::fs::write(directory.join(a), body).unwrap(); + std::fs::write(directory.join(b), format!("[Unit]\nConflicts={a}\n{body}")).unwrap(); + let user = |args: &[&str]| { + let output = Command::new("systemctl").arg("--user").args(args).output().unwrap(); + (output.status.success(), String::from_utf8_lossy(&output.stdout).trim().to_owned()) + }; + let outcome = std::panic::catch_unwind(|| { + let one_job = |verb: Verb, unit: &str| { + let step = Operation::UnitAction { verb, unit: unit.into() }.steps().remove(0); + let mut argv = step.argv.clone(); + argv.insert(1, "--user".into()); + let status = Command::new(&argv[0]).args(&argv[1..]).status().unwrap(); + assert!(status.success(), "{verb:?} {unit}"); + }; + assert!(user(&["daemon-reload"]).0); + // A start of b leaves a running, though b conflicts with it. + assert!(user(&["start", a]).0); + one_job(Verb::Start, b); + assert_eq!(user(&["is-active", a]).1, "active", "start stopped a"); + assert_eq!(user(&["is-active", b]).1, "active", "start left b down"); + // A restart of b, with both running, leaves a running. + one_job(Verb::Restart, b); + assert_eq!(user(&["is-active", a]).1, "active", "restart stopped a"); + assert_eq!(user(&["is-active", b]).1, "active", "restart left b down"); + }); + user(&["stop", a, b]); + std::fs::remove_file(directory.join(a)).unwrap(); + std::fs::remove_file(directory.join(b)).unwrap(); + user(&["daemon-reload"]); + outcome.unwrap(); + } +} diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index 70060bf..b9f2637 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -10,3 +10,19 @@ It is checked in because the TypeScript runtime emits a camelCase MessagePack map while Rust `DatabaseEntry` derives tuple payloads. Idunn must prove it can decode the producer's real wire shape rather than round-tripping a fixture it serialized itself. + +# systemd 257 `systemctl show` captures + +`systemd-257/*.show` are the verbatim output of systemd 257.13 for the 27 +properties `show_unit` requested at `112a7a5`, captured read-only on Yggdrasil +on 2026-10-24: `vili-presence-identity-only.show` (a failed idunn-vili unit whose +only credential is the presence identity) and +`odin-presence-and-one-service-credential.show` (the live Odin unit). systemd 257 +prints `LoadCredential=[unprintable]` in both. Tests read these bytes; do not edit. + +# Pre-cut systemd observation records + +`pre-cut-systemd-observation.{compact,named}.hex` are the two MessagePack +encodings of a `SystemdWorkloadObservation` with `load_credential` set to +`[unprintable]`, produced by the code at `112a7a5` (the form live incumbents +persisted). They pin that later Idunns still decode such records. diff --git a/tests/fixtures/idunn.operator_incident.v1.cc b/tests/fixtures/idunn.operator_incident.v1.cc new file mode 100644 index 0000000..180c6f0 Binary files /dev/null and b/tests/fixtures/idunn.operator_incident.v1.cc differ diff --git a/tests/fixtures/pre-cut-systemd-observation.compact.hex b/tests/fixtures/pre-cut-systemd-observation.compact.hex new file mode 100644 index 0000000..e7d09fb --- /dev/null +++ b/tests/fixtures/pre-cut-systemd-observation.compact.hex @@ -0,0 +1 @@ +dc0033d94e6964756e6e2d616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e73657276696365aa4964756e6e2074657374a3696e7601a465786563a26e6fa56d69786564c3a6753631303030a6753631303030a0a0a0c3c3a9696e76697369626c65a3616c6cc3a430303037a0ad5b756e7072696e7461626c655dcd10920194cdee48cdee48cdee48cdee4894cdee48cdee48cdee48cdee4891cdee480000000000c391010708b52f6f70742f746573742f62696e2f736572766963650101d9477368613235362d31313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131d9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161a92f6f70742f74657374a92f72756e2f74657374d9477368613235362d3232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323290d9477368613235362d33333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333d9202f73797374656d2e736c6963652f6964756e6e2d746573742e73657276696365c090aa61637469766174696f6edc0020010101010101010101010101010101010101010101010101010101010101010190 \ No newline at end of file diff --git a/tests/fixtures/pre-cut-systemd-observation.named.hex b/tests/fixtures/pre-cut-systemd-observation.named.hex new file mode 100644 index 0000000..8898adf --- /dev/null +++ b/tests/fixtures/pre-cut-systemd-observation.named.hex @@ -0,0 +1 @@ +de0033a4756e6974d94e6964756e6e2d616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e73657276696365b0756e69745f6465736372697074696f6eaa4964756e6e2074657374ad696e766f636174696f6e5f6964a3696e76d923657865635f6d61696e5f73746172745f74696d657374616d705f6d6f6e6f746f6e696301ac736572766963655f74797065a465786563ae726573746172745f706f6c696379a26e6fa96b696c6c5f6d6f6465a56d69786564ac64796e616d69635f75736572c3ac73797374656d645f75736572a6753631303030ad73797374656d645f67726f7570a6753631303030b4737570706c656d656e746172795f67726f757073a0b76361706162696c6974795f626f756e64696e675f736574a0b4616d6269656e745f6361706162696c6974696573a0ae707269766174655f6d6f756e7473c3ac707269766174655f70696473c3ac70726f746563745f70726f63a9696e76697369626c65ab70726f635f737562736574a3616c6cb16e6f5f6e65775f70726976696c65676573c3a5756d61736ba430303037b2696e61636365737369626c655f7061746873a0af6c6f61645f63726564656e7469616cad5b756e7072696e7461626c655da86d61696e5f706964cd1092b270726f636573735f73746172745f74696d6501ac70726f636573735f7569647394cdee48cdee48cdee48cdee48ac70726f636573735f6769647394cdee48cdee48cdee48cdee48ae70726f636573735f67726f75707391cdee48b770726f636573735f6361705f696e686572697461626c6500b570726f636573735f6361705f7065726d697474656400b570726f636573735f6361705f65666665637469766500b470726f636573735f6361705f626f756e64696e6700b370726f636573735f6361705f616d6269656e7400b970726f636573735f6e6f5f6e65775f70726976696c65676573c3b670726f636573735f6e616d6573706163655f706964739101b26d6f756e745f6e616d6573706163655f696407b07069645f6e616d6573706163655f696408aa65786563757461626c65b52f6f70742f746573742f62696e2f73657276696365b165786563757461626c655f64657669636501b065786563757461626c655f696e6f646501b165786563757461626c655f736861323536d9477368613235362d31313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131b372756e74696d655f696e7374616e63655f6964d9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161b1776f726b696e675f6469726563746f7279a92f6f70742f74657374ae72756e74696d655f62756e646c65a92f72756e2f74657374b3636f6d6d616e645f6c696e655f736861323536d9477368613235362d32323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232b1656e7669726f6e6d656e745f6e616d657390bb656e7669726f6e6d656e745f636f6e74726163745f736861323536d9477368613235362d33333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333ad636f6e74726f6c5f67726f7570d9202f73797374656d2e736c6963652f6964756e6e2d746573742e73657276696365b563726564656e7469616c735f6469726563746f7279c0bc706172656e745f6f6e6c795f66696c655f64657363726970746f727390bd61637469766174696f6e5f7369676e65725f6964656e746974795f6964aa61637469766174696f6ebc61637469766174696f6e5f7369676e65725f7075626c69635f6b6579dc00200101010101010101010101010101010101010101010101010101010101010101b3736572766963655f63726564656e7469616c7390 \ No newline at end of file diff --git a/tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show b/tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show new file mode 100644 index 0000000..21e4573 --- /dev/null +++ b/tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show @@ -0,0 +1,28 @@ +Type=exec +Restart=no +MainPID=2571 +OpenFile=/run/idunn/activation-credentials/sha256-7cfe5cd1e87839e029c7425eb1b586b6b0d94dd79d7376eed3b156386f2c7af8.credential:gamecult-idunn-runtime-activation-key:read-only +OpenFile=/etc/gamecult/odin/runtime-presence-identity.cc:gamecult-runtime-presence-identity:read-only +ExecMainStartTimestampMonotonic=20630000 +ControlGroup=/system.slice/idunn-odin-7cfe5cd1e87839e029c7425eb1b586b6b0d94dd79d7376eed3b156386f2c7af8.service +UMask=0007 +WorkingDirectory=/srv/odin/releases/sha256-e5c7d0b2b9c0a8ffba368aeb08abdbe37ae0d17484a785e60a66ec4a62b2f18c +CapabilityBoundingSet= +AmbientCapabilities= +User=_duc23369f1b8b167db +Group=odin +DynamicUser=yes +LoadCredential=[unprintable] +SupplementaryGroups= +InaccessiblePaths=/run/idunn/activation-credentials +PrivateMounts=yes +PrivatePIDs=yes +NoNewPrivileges=yes +ProtectProc=invisible +ProcSubset=all +KillMode=mixed +Description=Idunn odin sha256-7cfe5cd1e87839e029c7425eb1b586b6b0d94dd79d7376eed3b156386f2c7af8 +LoadState=loaded +ActiveState=active +SubState=running +InvocationID=ada48921043449f39dacc87b14a1bf70 diff --git a/tests/fixtures/systemd-257/vili-presence-identity-only.show b/tests/fixtures/systemd-257/vili-presence-identity-only.show new file mode 100644 index 0000000..fc78fd1 --- /dev/null +++ b/tests/fixtures/systemd-257/vili-presence-identity-only.show @@ -0,0 +1,28 @@ +Type=exec +Restart=no +MainPID=0 +OpenFile=/run/idunn/activation-credentials/sha256-b507e1705a70f8eee72f355e7db637acab8eb37880b5d25245fda68d80f63041.credential:gamecult-idunn-runtime-activation-key:read-only +OpenFile=/etc/gamecult/vili/runtime-presence-identity.cc:gamecult-runtime-presence-identity:read-only +ExecMainStartTimestampMonotonic=520083420000 +ControlGroup= +UMask=0007 +WorkingDirectory=/srv/vili-idunn/releases/sha256-aeaee4e1669742572c45a3f26bc86893c44b2a4149520975eb7d8a28aa931906 +CapabilityBoundingSet= +AmbientCapabilities= +User=_du8c06c563aac6d205 +Group=vili-state +DynamicUser=yes +LoadCredential=[unprintable] +SupplementaryGroups= +InaccessiblePaths=/run/idunn/activation-credentials +PrivateMounts=yes +PrivatePIDs=yes +NoNewPrivileges=yes +ProtectProc=invisible +ProcSubset=all +KillMode=mixed +Description=Idunn vili sha256-b507e1705a70f8eee72f355e7db637acab8eb37880b5d25245fda68d80f63041 +LoadState=loaded +ActiveState=failed +SubState=failed +InvocationID=8f7a349f2f8f49fe87ac5d89239260b4