diff --git a/Cargo.lock b/Cargo.lock index dfc3087..44c9457 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -482,6 +482,7 @@ dependencies = [ "libc", "rmp-serde", "serde", + "serde_json", "sha2", "tempfile", "toml", diff --git a/Cargo.toml b/Cargo.toml index 713ad33..ee797ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,10 @@ path = "src/bin/idunn-provision.rs" name = "idunn-host" path = "src/bin/idunn-host.rs" +[[bin]] +name = "idunn-launch" +path = "src/bin/idunn-launch.rs" + [lib] name = "idunn_daemon" path = "src/lib.rs" @@ -34,6 +38,7 @@ serde = { version = "1", features = ["derive"] } ed25519-dalek = "2.2" toml = "0.8" uuid = { version = "1", features = ["v4"] } +serde_json = "1" [dev-dependencies] tempfile = "3" diff --git a/README.md b/README.md index 297a9f1..8c6935c 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,64 @@ idunn status --command the bindings, so `idunn up profile:full-gamecult` brings up everything bound into that profile. +### Steward orders + +A second root unit from the same binary, `idunn-orders` (`idunn orders serve`), +takes typed orders from one steward Mind over `/run/idunn-orders/steward.sock`. +It has its own store, `orders.cc`, and its own binding, `steward.toml`; the +survival daemon takes no order. There is still no command string: + +```bash +idunn order package-install [=version]... --reason TEXT # apt-get update, install named packages, check them +idunn order unit --reason TEXT +idunn order deploy --reason TEXT # the same commit as `idunn up` +idunn order show # status and output tails +idunn order list +``` + +A unit order submits exactly one job, for the unit it names: every verb runs +`systemctl --job-mode=ignore-requirements -- `, so systemd builds a +transaction of one job. It does not start what the unit requires or restart what +is part of it: order the dependency first, and order `postgresql@16-main.service` +rather than the `postgresql.service` wrapper. Targets are not orderable. The job is one; what the unit does afterwards is +not: systemd still runs full transactions of its own for `Restart=` after a +failure, for a `StopWhenUnneeded=` dependency left without a user, and for a +`Conflicts=` partner of a unit that fails, so an ordered stop of a unit +that something restarts does not last, and a start of a `Restart=always` +unit can report failure and be active seconds later. A unit +order may act on any unit except Idunn's own (`idunn-*`) and the minds' own +(`mind-self@*`, `mind-persona@*`), and the units that end or take the host +(`systemd-suspend`, `systemd-hibernate`, `systemd-hybrid-sleep`, +`systemd-suspend-then-hibernate`, `rescue`, `emergency`). That set is compiled +into the binary and read from no configuration. An order is refused, recorded +and posted when the ordered unit, or an alias it resolves from, is one of them; +when it has an emergency action (`SuccessAction=`, `FailureAction=`, +`StartLimitAction=`, `JobTimeoutAction=`) other than `none` or an `OnFailure=` +or `OnSuccess=` unit (not for a reload); or, for a stop or restart, when another +unit is `BoundBy=` it, because systemd stops that unit with it whatever the job +mode; for a start or restart, when it has an `Upholds=` unit, which systemd +starts in a full transaction whatever the job mode; or, for a stop, restart or +reload, when it has a `ConsistsOf=` member, a wrapper such as +`postgresql.service` whose verbs would exit 0 and touch nothing. A unit +`systemctl show` cannot resolve fails the order. Slices, scopes, +targets and globs are not units an order can name. + +A package order installs the packages it names and nothing else: apt runs with +`--no-remove`, each name is passed as an exact-name pattern so `.` and `+` +match themselves, and a last step fails the order unless every named package +is installed. + +An order runs once, after the steward's uid is checked and the instance's +brake check (`mind-control --instance check`) exits 0. The check +that the order's principal and uid are the bound steward's runs again at every +execution, so an order accepted before `steward.toml` changed is refused and +recorded, not run. The socket belongs to the steward user alone (mode 0600); +root may `show` and `list` orders but never issue one. An order that had +started when the service died is recorded as lost and never run again. No order releases a brake. Acceptance and result are posted to +`#operator` as a retried projection of `orders.cc`: a chat outage delays the +posts and not the orders. A refusal names the field and a code, never the value +it refused. + ## Authority **Owner.** Deployment admission, and the continuity of installed bodies. @@ -150,6 +208,79 @@ omits one. Unknown fields are rejected on both sides. A repository that gets compromised can therefore change what is built, but not what it is allowed to touch on the host. +An upstream daemon that does not speak Idunn's runtime contract is deployed +with `idunn-launch` as its executable artifact and the upstream tree as a +required adjacent artifact; the launcher finds the child inside the release that +holds it. The launcher is the unit's process: it binds the +candidate endpoint and answers Idunn's challenge there, renders the child's +config, starts a stateful child only once it holds the write lease, and signs +presence. Members never pass through it: the binding's `[route.member]` makes +Idunn route a second listener straight to the child, which binds the address +Idunn gives it in `GAMECULT_IDUNN_MEMBER_BIND`. + +```toml +# recipe fragment — the upstream tree is built or fetched as an artifact +[service] +executable_artifact = "idunn-launch" +arguments = [ + { kind = "literal", value = "--child" }, + { kind = "literal", value = "upstream/bin/daemon" }, + { kind = "literal", value = "--template" }, + { kind = "literal", value = "config.tmpl" }, + { kind = "literal", value = "--health-path" }, + { kind = "literal", value = "/health" }, + { kind = "literal", value = "--health-contract" }, + { kind = "literal", value = "example.presence-launcher.v1" }, +] +required_environment = [ + "GAMECULT_IDUNN_RUNTIME_BUNDLE", + "GAMECULT_IDUNN_CANDIDATE_BIND", + "GAMECULT_IDUNN_MEMBER_BIND", +] +``` + +A directory artifact is sealed read-only: its files are 0444 and its +directories 0555, whatever mode the build or git gave them. A file the tree +must run, such as the launcher's `upstream/bin/daemon` child, is declared in +`executable_members` and sealed 0555. The members are part of the sealed release +address, so changing the list makes a new release. Idunn refuses the release if a +member is absolute, climbs with `..`, is missing, is a symlink or sits behind +one, or is not a regular file; `executable = true` on a directory output and +`executable_members` on a file output are refused too. + +A tree may hold relative symlinks whose `..` steps all come first and climb no +higher than the artifact's top (`share/terminfo/4/4025ex -> ../t/tek4025ex` is +kept). An absolute link, a `..` after a name, or a climb out of the artifact +refuses the release when it is sealed. Host targets (Windows) record +`executable` and `executable_members` in the receipt and do not apply them. + +```toml +# recipe fragment — the upstream artifact declares the child it launches +[[artifacts]] +id = "upstream" +source_kind = "runner-output" +runner = "build" +source = "dist/upstream" +destination = "upstream" +executable_members = ["bin/daemon"] +``` + +```toml +# binding fragment — members connect to the member endpoint, which Idunn +# forwards to the private port derived from the candidate: member +# private_port_start + (candidate port - route private_port_start) +[route] +stable_endpoint = "tcp://10.77.0.1:8008" +private_port_start = 18831 +private_port_end = 18839 +# ... + +[route.member] +stable_endpoint = "tcp://10.77.0.1:8448" +private_port_start = 18841 +# max_connections = 128 # nginx limit_conn on the member listener (1-65535) +``` + ## Contracts Idunn's typed contracts are published by @@ -204,7 +335,9 @@ purpose; Idunn discovers no authority implicitly. Idunn targets Linux and does not compile on Windows (`drivers.rs` uses `std::os::unix` unconditionally). A local `cargo check` on a Windows -workstation proves nothing. Test and build where it runs, in the same image +workstation proves nothing. `scripts/verify.sh` is the verification command: +the library tests plus a Windows-target check of `idunn-host`. Run it on Linux +with the `x86_64-pc-windows-gnu` std installed. Test and build where it runs, in the same image Odin uses, with the working tree shipped to the host's build root: ```bash diff --git a/deploy/idunn-orders.service b/deploy/idunn-orders.service new file mode 100644 index 0000000..f2681c2 --- /dev/null +++ b/deploy/idunn-orders.service @@ -0,0 +1,47 @@ +[Unit] +Description=Idunn steward orders: typed package, unit and deploy orders from the steward +# Deliberately no After= or Requires= on any managed target, Synapse or +# idunn-yggdrasil: orders are how an operator repairs those, so none of them +# may be a condition of this unit starting. Posts are a retried projection and +# wait for Synapse on their own. + +[Service] +Type=simple + +# Root, from the same binary as idunn-yggdrasil, with its own store and socket. +# idunn-yggdrasil keeps its no-imperative-input invariant and its write set: +# no order code runs in `idunn serve`. +# +# The unit name matches idunn-*, which is in the always-refused set, so no +# order can stop or restart this service. +User=root +# No Group= and no UMask=: the service gives its socket to the steward user +# (mode 0600) itself and sets Idunn's own umask for the files it writes, so +# nothing here reaches control.cc. The steward user is the only peer the +# service answers: it checks SO_PEERCRED on every connection. +ExecStart=/usr/local/bin/idunn orders serve \ + --store /var/lib/gamecult/idunn-orders \ + --state-store /var/lib/gamecult/idunn/control.cc \ + --steward-binding /etc/gamecult/idunn/steward.toml \ + --socket /run/idunn-orders/steward.sock +Restart=always +RestartSec=5 +TimeoutStopSec=20 + +RuntimeDirectory=idunn-orders +RuntimeDirectoryMode=0755 + +# The Synapse access token of @idunn, copied from a root-only file, and read +# through $CREDENTIALS_DIRECTORY; the service names it in steward.toml. +LoadCredential=idunn-matrix-token:/etc/gamecult/idunn/matrix-token + +PrivateTmp=yes +ProtectHome=yes +ProtectSystem=strict +# Its own order store, and the control store a deploy order commits a command +# to. Transient units started with systemd-run run in the manager's namespace, +# so packages and units are not limited by this list. +ReadWritePaths=/var/lib/gamecult/idunn-orders /var/lib/gamecult/idunn + +[Install] +WantedBy=multi-user.target diff --git a/deploy/idunn-yggdrasil.service b/deploy/idunn-yggdrasil.service index 5474c67..1c45594 100644 --- a/deploy/idunn-yggdrasil.service +++ b/deploy/idunn-yggdrasil.service @@ -41,6 +41,7 @@ ExecStart=/usr/local/bin/idunn serve \ --source-root /var/lib/gamecult/idunn/sources \ --staging-root /var/lib/gamecult/idunn/staging \ --topology-store /var/lib/gamecult/idunn-projection/topology.cc \ + --incident-store /var/lib/gamecult/idunn-projection/incidents.cc \ --idunn-identity-store /var/lib/gamecult/idunn/idunn-service-identity.cc \ --odin-correlation-store /var/lib/gamecult/odin/topology.cc \ --odin-trust-anchor /etc/gamecult/idunn/odin-topology-anchor.cc \ @@ -76,7 +77,10 @@ ReadWritePaths=/etc/ufw # idunn-projection is deliberately outside Idunn's own 0750 directory: the # topology store is Idunn's *published* surface, and every managed target must # read it to verify its own Expected incarnation against the Idunn anchor. It -# was inside the private root, where no workload could traverse to it. +# was inside the private root, where no workload could traverse to it. The +# published topology store is read lock-free (Odin) and incidents.cc is read by +# Bifrost's reader; managed targets get Expected from their runtime bundle. Only +# the stores are readable: each .lock beside them is Idunn's alone, 0600. ReadWritePaths=/var/lib/gamecult/idunn /var/lib/gamecult/idunn-authority /var/lib/gamecult/idunn-projection ReadWritePaths=/srv/build # Each managed target gets only its bound roots. A target Idunn cannot write is @@ -100,6 +104,11 @@ ReadWritePaths=/etc/gamecult/ghostlight/runtime /var/lib/gamecult/ghostlight-wor ReadWritePaths=/etc/gamecult/streampixels-service/runtime /var/lib/gamecult/streampixels-service /srv/streampixels-idunn/releases/service # StreamPixels web is stateless; it needs only its runtime bundle and releases. ReadWritePaths=/etc/gamecult/streampixels-web/runtime /srv/streampixels-idunn/releases/web +# Vili (motion worker): runtime bundle, state, and releases. Prefixed "-" so a +# target whose roots are not created yet cannot stop Idunn itself from starting +# (Idunn startup never depends on a managed target). A root created after Idunn +# starts is writable only from Idunn's next restart. +ReadWritePaths=-/etc/gamecult/vili/runtime -/var/lib/gamecult/vili -/srv/vili-idunn/releases [Install] WantedBy=multi-user.target diff --git a/deploy/legacy/README.md b/deploy/legacy/README.md index dd91a25..54bb919 100644 --- a/deploy/legacy/README.md +++ b/deploy/legacy/README.md @@ -4,11 +4,16 @@ kept because they are what is *running*, and a migration needs both sides. `idunn-yggdrasil` is a root shell actuator reached through the narrow sudo grant -in `idunn-yggdrasil.sudoers`. It hardcodes eleven targets, their repositories +in `idunn-yggdrasil.sudoers`. It hardcodes ten targets, their repositories and their admitted refs, then delegates to executable manifests under -`/srv/odin/deploy-manifests/`. It calls `idunn validate-release-authority` -and `idunn validate-minimum-source-revision` — subcommands the current binary -does not have. +`/srv/odin/deploy-manifests/`. It calls `idunn validate-minimum-source-revision`, +a subcommand the current binary does not have. + +Checked on yggdrasil 2026-09-30: `/usr/local/libexec/idunn-yggdrasil` is installed +and matches this file byte for byte (before the retirement below), but +`idunn-yggdrasil.sudoers` is not installed. `sudo -l -U idunn` says the `idunn` +user may not run sudo, so the daemon cannot reach this actuator; only an +operator running it as root can. ## What replaced it @@ -16,10 +21,10 @@ Everything this script does by convention, the current Idunn does by contract: | Legacy mechanism | Current owner | |---|---| -| `deployment_target_policy()` — a `case` over eleven target names | one operator binding per target, in `--bindings-dir` | +| `deployment_target_policy()` — a `case` over ten target names | one operator binding per target, in `--bindings-dir` | | `upstream_ref=refs/heads/...` in shell | `[repository] admitted_ref` | | `minimum_source_revision_required` | `[repository] minimum_revision` | -| `requires_bifrost_authority` + `--release-authority-store` | `[repository] selection = "signed-release"`, per target | +| `requires_bifrost_authority` + `--release-authority-store` (retired with its only target) | `[repository] selection = "signed-release"`, per target | | `/srv/odin/deploy-manifests/` shell scripts | recipe `[[steps]]` in the target's own repository | | `run_manifest_while_holding_authority_locks` | the deployment transaction and process write-lease | | `flock` on a brake store | `[brakes] deployment_store` and `lifecycle_store` | @@ -45,15 +50,26 @@ This is the real migration scope, read out of `deployment_target_policy()`: | `heimdall` | GameCult/Heimdall | `main` | restart is `docker compose restart` | | `epiphany` | GameCult/Epiphany | `codex/epiphany-shakedown-live` | | | `epiphany-capstone-17` | GameCult/Epiphany | `codex/epiphany-shakedown-live` | second target, same repo | -| `bifrost-persona-feedback` | GameCult/Bifrost | `main` | **only** target requiring Bifrost release authority | +| ~~`bifrost-persona-feedback`~~ | GameCult/Bifrost | `main` | **retired 2026-09-30**, see below | | `repixelizer` | GameCult/repixelizer | `main` | | | `streampixels` | GameCult/StreamPixels | `main` | two units, one target | | `gjallar` | GameCult/Gjallar | `codex/yggdrasil-aggregate-daemon` | | -Seven of eleven are pinned to a non-`main` branch. That is its own problem — the +Seven of the ten live targets are pinned to a non-`main` branch. That is its own problem — the swarm goal is everyone on `main` and current with each other — and migrating a target is the moment to fix its ref, not to copy it forward. +## Retired targets + +- `bifrost-persona-feedback`, 2026-09-30. Its manifest ran the Bifrost compose lane + (`docker compose up persona-feedback`), while the live provider is the native + `bifrost-persona-feedback.service`, so firing it would have started a container + beside the native unit. The operator ruled "delete the Bifrost lane". Its grant, + dispatch case and the Bifrost release-authority check it alone used are gone; a + deploy request that claims a Bifrost authority is refused. It was the only + caller of `idunn validate-release-authority`. Rebinding persona-feedback to a + current Idunn binding is a separate follow-up. + ## When these files die When every target above has a binding and a recipe, and `idunn serve` owns diff --git a/deploy/legacy/idunn-yggdrasil b/deploy/legacy/idunn-yggdrasil index 1d83b52..26970c0 100644 --- a/deploy/legacy/idunn-yggdrasil +++ b/deploy/legacy/idunn-yggdrasil @@ -96,14 +96,12 @@ deployment_target_policy() { daemon_id=yggdrasil-odin repository_full_name=GameCult/Odin upstream_ref=refs/heads/codex/ygg-idunn-independent-bootstrap - requires_bifrost_authority=false manifest_takes_action=true ;; codex-connector) daemon_id=yggdrasil-codex-connector repository_full_name=GameCult/CodexConnector upstream_ref=refs/heads/codex/ghostlight-release-binding - requires_bifrost_authority=false minimum_source_revision_required=true manifest_takes_action=true ;; @@ -111,7 +109,6 @@ deployment_target_policy() { daemon_id=yggdrasil-ghostlight repository_full_name=GameCult/Ghostlight upstream_ref=refs/heads/codex/ghostlight-dungeon-mvp - requires_bifrost_authority=false minimum_source_revision_required=true manifest_takes_action=true ;; @@ -119,52 +116,39 @@ deployment_target_policy() { daemon_id=yggdrasil-voidbot repository_full_name=GameCult/VoidBot upstream_ref=refs/heads/main - requires_bifrost_authority=false manifest_takes_action=true ;; heimdall) daemon_id=yggdrasil-heimdall repository_full_name=GameCult/Heimdall upstream_ref=refs/heads/main - requires_bifrost_authority=false ;; epiphany-capstone-17) daemon_id=yggdrasil-epiphany-capstone-17 repository_full_name=GameCult/Epiphany upstream_ref=refs/heads/codex/epiphany-shakedown-live - requires_bifrost_authority=false manifest_takes_action=true ;; epiphany) daemon_id=yggdrasil-epiphany repository_full_name=GameCult/Epiphany upstream_ref=refs/heads/codex/epiphany-shakedown-live - requires_bifrost_authority=false manifest_takes_action=true ;; - bifrost-persona-feedback) - daemon_id=yggdrasil-bifrost-persona-feedback - repository_full_name=GameCult/Bifrost - upstream_ref=refs/heads/main - requires_bifrost_authority=true - ;; repixelizer) daemon_id=yggdrasil-repixelizer repository_full_name=GameCult/repixelizer upstream_ref=refs/heads/main - requires_bifrost_authority=false ;; streampixels) daemon_id=yggdrasil-streampixels repository_full_name=GameCult/StreamPixels upstream_ref=refs/heads/main - requires_bifrost_authority=false ;; gjallar) daemon_id=yggdrasil-gjallar repository_full_name=GameCult/Gjallar upstream_ref=refs/heads/codex/yggdrasil-aggregate-daemon - requires_bifrost_authority=false ;; *) echo "deployment target has no root actuator policy: $target" >&2; exit 1 ;; esac @@ -190,19 +174,10 @@ require_signed_deployment_grant() { *) echo "signed deployment request is not for root target $target" >&2; exit 1 ;; esac - case "$requires_bifrost_authority" in - true) - [ -n "$release_authority_id" ] || { echo "missing Bifrost release authority id" >&2; exit 1; } - case "$release_authority_sha256" in - *[!0-9a-f]*|'') echo "invalid or missing Bifrost release authority digest" >&2; exit 1 ;; - esac - [ "${#release_authority_sha256}" -eq 64 ] || { echo "Bifrost release authority digest must be 64 lowercase hex characters" >&2; exit 1; } - ;; - false) - [ -z "$release_authority_id" ] || { echo "source-observed release cannot claim Bifrost authority id" >&2; exit 1; } - [ -z "$release_authority_sha256" ] || { echo "source-observed release cannot claim Bifrost authority digest" >&2; exit 1; } - ;; - esac + # No target carries Bifrost release authority: its only one, bifrost-persona-feedback, + # was retired on 2026-09-30. A request that claims one is refused. + [ -z "$release_authority_id" ] || { echo "source-observed release cannot claim Bifrost authority id" >&2; exit 1; } + [ -z "$release_authority_sha256" ] || { echo "source-observed release cannot claim Bifrost authority digest" >&2; exit 1; } require_root_owned_resolved_executable "$idunn_provision" require_root_owned_regular_path "$brake_store" @@ -234,18 +209,6 @@ require_signed_deployment_grant() { --daemon "$daemon_id" \ --source-revision "$source_commit" fi - - if [ "$requires_bifrost_authority" = true ]; then - require_root_owned_resolved_executable /usr/local/bin/idunn - /usr/local/bin/idunn validate-release-authority \ - --store /var/lib/gamecult/idunn/yggdrasil.keepalive.cc \ - --release-authority-store /srv/bifrost/state/repository-release-authority.cc \ - --repository-full-name "$repository_full_name" \ - --upstream-ref "$upstream_ref" \ - --source-revision "$source_commit" \ - --release-authority-id "$release_authority_id" \ - --release-authority-sha256 "$release_authority_sha256" - fi } manifest_group_exists() { @@ -355,7 +318,7 @@ case "$action:$target" in run_manifest_while_holding_authority_locks "$manifest" restart exit $? ;; - deploy:voidbot|deploy:epiphany|deploy:epiphany-capstone-17|deploy:odin|deploy:codex-connector|deploy:ghostlight|deploy:heimdall|deploy:repixelizer|deploy:streampixels|deploy:bifrost-persona-feedback|deploy:gjallar) + deploy:voidbot|deploy:epiphany|deploy:epiphany-capstone-17|deploy:odin|deploy:codex-connector|deploy:ghostlight|deploy:heimdall|deploy:repixelizer|deploy:streampixels|deploy:gjallar) manifest="/srv/odin/deploy-manifests/$target" require_root_owned_regular_path "$manifest" [ -x "$manifest" ] || { echo "root actuator manifest is not executable: $manifest" >&2; exit 1; } @@ -368,7 +331,8 @@ case "$action:$target" in export BIFROST_RELEASE_AUTHORITY_ID="$release_authority_id" export BIFROST_RELEASE_AUTHORITY_SHA256="$release_authority_sha256" export IDUNN_DEPLOYMENT_REQUEST_ID="$deployment_request_id" - export IDUNN_REQUIRES_BIFROST_AUTHORITY="$requires_bifrost_authority" + # Installed manifests (Epiphany's) still require this to read false. + export IDUNN_REQUIRES_BIFROST_AUTHORITY=false if [ "$manifest_takes_action" = true ]; then run_manifest_while_holding_authority_locks "$manifest" "$action" exit $? diff --git a/deploy/legacy/idunn-yggdrasil.sudoers b/deploy/legacy/idunn-yggdrasil.sudoers index 8872c25..a855527 100644 --- a/deploy/legacy/idunn-yggdrasil.sudoers +++ b/deploy/legacy/idunn-yggdrasil.sudoers @@ -1,7 +1,7 @@ # Narrow root actuator grant for the Idunn Yggdrasil deployment/continuity path. -# One line per live target. codex-connector and gjallar were retired 2026-09-06; -# their grants are removed here first, because the privileged half of a retired -# target must not outlive its consumer. +# One line per live target. codex-connector and gjallar were retired 2026-09-06, +# and bifrost-persona-feedback on 2026-09-30; their grants are removed here first, +# because the privileged half of a retired target must not outlive its consumer. Cmnd_Alias IDUNN_YGGDRASIL_DEPLOY = \ /usr/local/libexec/idunn-yggdrasil deploy odin *, \ /usr/local/libexec/idunn-yggdrasil deploy ghostlight *, \ @@ -9,7 +9,6 @@ Cmnd_Alias IDUNN_YGGDRASIL_DEPLOY = \ /usr/local/libexec/idunn-yggdrasil deploy heimdall *, \ /usr/local/libexec/idunn-yggdrasil deploy epiphany-capstone-17 *, \ /usr/local/libexec/idunn-yggdrasil deploy epiphany *, \ - /usr/local/libexec/idunn-yggdrasil deploy bifrost-persona-feedback *, \ /usr/local/libexec/idunn-yggdrasil deploy repixelizer *, \ /usr/local/libexec/idunn-yggdrasil deploy streampixels * Cmnd_Alias IDUNN_YGGDRASIL_RESTART = \ diff --git a/docs/authority-map.md b/docs/authority-map.md index dc918fd..c8a8546 100644 --- a/docs/authority-map.md +++ b/docs/authority-map.md @@ -55,7 +55,10 @@ A deployable repository publishes one strict declaration, normally - immutable artifact outputs, optionally pinned to an expected SHA-256; - a constrained launch contract; - one health-contract identity; -- an optional typed state contract with state schema generation and slots; +- an optional typed state contract with state schema generation and slots + (kinds `cultcache-file`, `external-file` and `external-directory`; the + external kinds are process-owned state an external runtime imposes, which + Idunn preserves and fences but never interprets); - provided capabilities, dependency kinds, startup requirements, capacity, and conflicts. @@ -67,6 +70,52 @@ recipe fields. Stateless services declare no state block. Their operator binding therefore has no state root, state-transition policy, or process write-lease record. +## Foreign workloads: idunn-launch + +An upstream daemon that cannot speak Idunn's runtime contract (Synapse, MAS) +runs under `idunn-launch`, the unit's own process. Idunn's two-witness rule is +unchanged: Idunn observes the unit, and the launcher, inside the unit and +supervising the child, signs presence with the provider identity and the +activation key. + +- **Owner:** the launcher owns presence, config rendering and the child's + lifetime inside the unit, and the route answers it signs for that child. + Idunn owns admission, lease, route and continuity, as before. +- **Inputs:** the runtime bundle, the activation credential and provider + identity descriptors, the process write lease, the systemd credentials the + binding names, and the child's health over TCP at the member bind. +- **Outputs:** signed `gamecult.runtime_presence_health.v2` (Warming, Active + and failed to Odin; route observations on the candidate endpoint), one child + process, and a rendered config in the unit's private tmp that is never read + back. +- **Route:** Idunn owns both listeners of the route: the Expected route's + stable endpoint (presence) and, when the binding has `[route.member]`, the + member endpoint, rendered as a second server block in the one nginx fragment + with a `limit_conn` on one per-route zone (`max_connections`, default 128), so + members cannot exhaust the connections the presence block shares with them. + The launcher binds the candidate endpoint and answers Idunn's challenge there, + one connection at a time within a one-second deadline; any other first byte + closes the connection. The answer never waits on the child: it reports the + lifecycle's last probe, and a failed, missing or stale probe reads `degraded`. + The child binds the member bind Idunn sets in + `GAMECULT_IDUNN_MEMBER_BIND`, derived from the candidate port. Member bytes + never touch the launcher. +- **Lease:** with `write_lease_required` the launcher starts the child only + after it holds the lease (bound to this Expected, this activation and one of + its own Warming proofs) and keeps the lease store share-locked while the + child lives. Losing the lease stops the child. +- **Config:** one template from the release is rendered with the closed + placeholder set `{{child.listen}}`, `{{child.listen_host}}`, + `{{child.listen_port}}` (all three the member bind), `{{credential:NAME}}`, + `{{credential_path:NAME}}` and `{{state_root}}`. Any other placeholder + refuses to start. +- **Forbidden:** member traffic through the launcher; the child binding the + candidate endpoint; a child of a lease-requiring target running without a held lease; any presence not signed + by both keys. +- **State kind `external-directory`:** a process-owned directory imposed by an + external runtime contract. Same writer, recovery and startup rules as + `external-file`: it is lease-gated, preserved and fenced, never interpreted. + ## Operator binding The root-admitted operator binding supplies the physical policy a repository @@ -173,6 +222,41 @@ health plus Ready correlation. It changes and observes route membership only afterward. Only after the proxy has adopted the candidate may Idunn clear the promotion fence and drain the incumbent. +A target has two slots: the incumbent (the admitted generation) and a +transaction's candidate. Continuity owns the incumbent slot and keeps the +admitted generation alive; it never stops. A live `DeploymentTransaction` owns +its private candidate before Fencing and takes the target itself at Fencing (a +stop-then-start rollout, at Starting) through one `take_target` CAS, so of two +transactions entering from one view exactly one succeeds. A continuity +restart is itself a transaction and takes the target the same way. + +Phases from Warming through Committing carry deadlines, as do aborts and +drains; an expired deadline never waits silently. A stateless deployment +aborts to its incumbent. A deployment whose candidate was issued a write lease +is left to the operator (`operator-required` incident, the second incident +condition beside `continuity-exhausted`; its subject is the transaction id and +it closes when the record ends or its step goes on). A continuity after +Fencing retries its phase forward on a capped backoff and never stops its live +candidate, since that is the only running copy of the admitted body; the +deadline opens the same incident. The one exception is yielding: a continuity +whose route proof has not succeeded gives way, through its own post-fencing +abort, to a deployment the deployment brake has released and that awaits the +handover; a continuity that has proven its route never yields. The +`ContinuityExhausted` incident (six charged attempts in an hour, +`restarts_page`) pages and gates nothing. When continuity next acts is derived +only from its own record in `TargetSupervision` (schema v2): the attempt +streak, the backoff anchor (written by the dead-and-free pass, the only +writer), and the once-per-admission re-materialization bound. The workload +driver alone answers whether a lost unit was a host reboot +(`WorkloadPort::lost_to_host_reboot`). Operator-facing incidents and +`idunn status` are projections that decide nothing. A boot lift returns a +continuity abort left by an older Idunn, one that had not stopped its running +candidate, to forward retry; if it cannot decide, every continuity abort is +held until a lift succeeds. It is a migration with a deletion line (follow-up +`c2c-delete-boot-lift`). Cut line: cuts 3 and 4 of `reincarnation-map.md` +still owe dependency failures not growing the streak and the dead-generation +route refresh. + The deployment brake gates changes to source, release, configuration, schema, unit, or authority binding. The lifecycle brake separately gates restart of an already-admitted incarnation. Neither target brake may gate Idunn itself or an diff --git a/docs/deploy-provenance-map.md b/docs/deploy-provenance-map.md new file mode 100644 index 0000000..95c0eee --- /dev/null +++ b/docs/deploy-provenance-map.md @@ -0,0 +1,394 @@ +# Deployment provenance: map + +Status: map, Imagination, 2026-10-23 (campaign `idunn-deploy-provenance`, +session `self-2026-10-23-eureka-b`). The ends are in +[`deploy-provenance-target.md`](deploy-provenance-target.md). Questions, +rulings, cut specs and follow-ups live in Eureka's mind; this page holds what +has no kind: body facts, rationale and sequencing. Line numbers in T1-T12 are +at `d74ac8d` (origin/main on 2026-10-23); T4, T9 and T13-T16 are at `fd040f0`, +the head of Hands attempt 1, which revision 2 of the spec builds on. + +## Cut `tree-executable-members` + +Spec `idunn-deploy-provenance:cut_spec:cut-tree-executable-members.r2` +(revision 2, after Soul pass 1 found the real interpreter tree refused). It +serves `motion-pipeline:follow_up:idunn-tree-executable-members` (Vili's +admission waits on it) and will serve gamecult-chat C3 too: the presence +launcher runs a child such as `upstream/bin/daemon` from inside a tree artifact +(README, launcher fragment), and Synapse needs a Python interpreter there. + +### Body facts + +- T1. `ArtifactOutput` (`src/deployment.rs:106-117`) carries one `executable` + bool. `ArtifactReceipt` (`src/deployment_plan.rs:731-737`) copies it and + `validate` (`:740-760`) requires it to equal the declaration. +- T2. `harden_installed_release` (`src/drivers.rs:6848-6855`) builds its + executable set from `root.join(receipt.destination)` only. `harden_root_tree` + (`:6858-6892`) seals every directory 0555 and every file 0444 unless that + exact path is in the set. A directory artifact with `executable = true` is + therefore silently meaningless today: its files are all 0444. Nothing refuses + it. No live recipe declares one (CodexConnector, Ghostlight, Heimdall, Odin, + StreamPixels service and web surveyed): `executable = true` is on files only. +- T3. A tree output is copied by `copy_artifact`/`copy_tree` + (`src/drivers.rs:8053-8126`), which recreate symlinks verbatim. The digest + (`digest_tree`, `:8142-8175`) covers names, file bytes and link targets, not + mode bits. So runner or git exec bits never reach the release: harden is the + only mode authority. +- T4. `validate_release_symlink` (`src/drivers.rs:6897-6916`) keeps a symlink + only if its target is relative with no `..`. It runs only inside + `harden_root_tree`, at install, so a tree with a `..` or absolute link copies, + digests and seals, then fails at `install_release` (Soul s1: copying and + digesting the real tree twice took 160 s before harden refused it). The rule + is sound but too narrow: it refuses the real interpreter tree (T13). The + frozen-source path has a physical resolver (`resolve_frozen_source_symlink`, + `:6617`) that walks the filesystem with a 40-link budget and a memo. Its + first version was exponential (Soul S4-1, recorded in `d562a12`), and it needs + the tree on disk to answer. Revision 2 does not reuse it (T14). +- T5. The sealed release id hashes `(schema, plan_id, artifacts, + external_inputs)` (`src/deployment_plan.rs:1000-1011`), and `plan_id` hashes + the whole plan including `recipe_blob`. A changed member list is therefore + already a new plan and a new release. The receipt must still carry the list, + because `harden_installed_release` reads receipts only; its two callers + (`install_release` `:2788`, `validate_installed_release` `:2822`) re-harden + every installed release on each observation. +- T6. Encoding is `rmp_serde::to_vec`, positional. A trailing field with + `#[serde(default, skip_serializing_if = "BTreeSet::is_empty")]` keeps every + existing receipt's bytes and release id: the plan's `deadlines` field + (`:574-577`) is the precedent. +- T7. `digest_artifact` and `digest_tree` (`:8128-8175`) `fs::read` each file + whole. A torch CPU tree's largest file (`libtorch_cpu.so`, hundreds of MB) + is held in Idunn's memory three times per install (seal, install, every + validate). `hash_frozen_source_tree` (`:6788-6840`) already streams in 64 KiB + chunks (S9). The cut collapses both onto one streaming helper. +- T8. `require_leaf_path` (`src/deployment.rs:1951-1958`) only keeps the + artifact destination a single filename. Members are paths inside that + destination, so it does not change. `require_relative_path` (`:1936-1949`) + with `allow_current = false` already refuses absolute, `..`, root and prefix + components. +- T9. The host actuator re-spells the seal (`src/host.rs:1050-1086`): copy, + mode check, digest, pinned-digest check and receipt, the same steps as + `collect_artifact` (`src/drivers.rs:2305-2326`). Soul s1: deleting its mode + check survives every test. Revision 2 collapses both onto one `seal_artifact` + in `drivers.rs`. Each caller keeps only how it finds the source path. +- T10. The transient unit runs with `ProtectSystem=strict` and the release in + `ReadOnlyPaths` (`src/drivers.rs:3675-3698`), `/tmp` mounted `noexec`, and no + `MemoryDenyWriteExecute`. Executing a 0555 file from the release works. + Shared libraries need no exec bit. +- T11. Live body on Yggdrasil: `/usr/local/bin/idunn` sha256 `44cf8a14...`, + installed 2026-10-20 02:34 CEST, before `6e2a31c` (02:40 CEST). `src/` is + identical between `8760a4f` and `6e2a31c`, so the live code is the + reincarnation c2 merge. `origin/main` is 26 commits ahead (+4194/-112 in + `src/`: the presence launcher and member route). Installing this cut ships all + of them. Idunn's own body changes only through gamecult-ops + `scripts/install-idunn-yggdrasil-release.sh` (staged binaries, unit, + bindings). The stage at `/home/gamecultadmin/idunn-stage` is stale (2026-09-11). +- T13. Link census of the real interpreter tree (Yggdrasil probe of + `cpython-3.12.15+20261021-x86_64-unknown-linux-gnu-install_only.tar.gz`, + sha256 `5ff9f060...77191124`; torch adds no links, as Soul s1 counted 1,049 + after pip): 1,049 links. 0 are absolute, 749 have no `..`, and 300 have all + their `..` components before the first name (every one is + `share/terminfo// -> ..//`, one `..` each). 0 have a `..` + after a name, 0 climb above the tree, 0 dangle and 0 resolve outside it. + Blender 5.2.2 has 79 links, none absolute or with `..` (Soul s1). +- T14. Lexically normalising `/` does not check + containment. The kernel resolves a target one component at a time, so a `..` + after a symlinked name climbs from where that link points, not from where its + name sits. Probe on Yggdrasil: `root/a/s -> ..` and `root/a/L -> s/../x` + normalise to `root` and `root/a/x`, both inside, yet `L` reads the file `x` + beside `root`. `root/a/d -> .` with `root/a/M -> d/../../x` escapes the same + way. Idunn has met this before: verify-transaction-cut F5 (`D -> .`, + `L -> D/D/../../../etc/passwd` passed a lexical check). + The sound lexical rule constrains the target's shape: all its `..` components + come before its first name, and they number no more than the link's depth + below the artifact's top. The climb then starts from the link's own + directory, which the walk reached through real directories only (copy and + digest never descend through a link), so it lands on a real ancestor inside + the artifact. After that the target only descends, and any link it passes + through obeys the same rule. By induction on the kernel's hops (capped at 40, + then ELOOP), every hop stays inside. The check is one pass over the target's + components with no filesystem reads, so its cost is linear in the tree and no + resolver is needed. The shapes it refuses (`s/../x`) occur in neither real + tree (T13). + Prior art, from documentation as known to Imagination (high confidence for Go + and securejoin, medium for the tar details): + - Go `path/filepath.IsLocal` (1.20) is lexical and says so: it does not account + for symlinks. It would admit `s/../x`. Used alone, it is the T14 bug. + - Go `os.Root` (1.24) and `cyphar/filepath-securejoin` (used by runc, + containerd and umoci) resolve each component inside the root when a path is + opened, refusing or clamping an escape. They guard one later access, need + the tree on disk, and check again on every open. Idunn instead certifies a + sealed tree once, so that the kernel's own resolution stays inside on every + later access. + - Zip-slip defences and archive extractors check entry names lexically and + protect against link targets by refusing to write through links + (libarchive's `ARCHIVE_EXTRACT_SECURE_SYMLINKS`, and `SECURE_NODOTDOT` for + names; GNU tar creates such links last). They protect the extraction, not + what a kept link points at afterwards. + - OCI layer extraction keeps link targets verbatim, absolute ones included, + because the runtime resolves them inside the container's root. Idunn runs a + release in the host's mount namespace, so absolute targets stay refused. + - Nix keeps link targets verbatim and tracks dependencies by scanning for + references. The whole store is its root, so it has no per-artifact + containment to compare with. + None of these is a static per-link rule for a tree that runs unconfined. The + T14 shape rule fits that case. +- T15. Host targets: `idunn-host` observes, starts and stops processes on + Windows only (the non-Windows `platform` module in `src/host.rs` bails), and + `host.rs` sets no file modes. On Windows a file runs by its type, not by a + mode bit, so `executable` and `executable_members` have nothing to set there. + The file `executable` flag has always been carried and never applied on host + targets. A recipe does not know which workload kind its binding will choose. +- T16. The release root holds only artifact destinations. Runner workspaces, + where external inputs land, are removed before `SealedRelease::new` + (`src/drivers.rs:2421-2423`). `install_release` (`:2781-2788`) and + `validate_installed_release` (`:2815-2822`) digest every receipt's + destination before calling harden. So a rule in the digest walk runs at seal, + at install and at every validate, over every link harden would see. +- T12. Disk: `/` on Yggdrasil has 1.3 TB free. A tree release costs about three + copies (runner workspace, staging, installed). Nothing collects old releases or + staging (`ghostlight-verse:follow_up:idunn-generation-retention`), so every + Vili deploy leaves about 7 GiB behind. + +### Contract, corrected against the body + +- Any directory artifact may declare `executable_members` (a set of relative + paths). The follow-up limited it to runner-output; that limit protects no + invariant, and the launcher's worktree or fetched trees need the same feature. +- `executable = true` names a file artifact. On a directory output the release + is refused, and `executable_members` on a file output is refused (T2). +- A member is refused at recipe parse if it is empty, absolute, or holds `.`, + `..`, a root or a prefix. At seal it is refused if it is missing, is a + symlink, passes through a symlink or non-directory, or is not a regular file. + Refusal happens before `SealedRelease::new`, so no sealed release can name a + member it cannot seal. +- Harden seals each member 0555 and fails if any declared executable path was + not visited as a regular file. "Every declared executable is sealed 0555, or + the install fails" holds structurally, not by a lookup that can miss. +- Symlinks: a link in a tree artifact is kept when its target is relative, its + `..` components all come before its first name, and they number no more than + the link's depth below the artifact's top (T14). One function, + `check_release_link`, decides this. Only the digest walk calls it, so a bad + link refuses the release at seal, before `SealedRelease::new`, and again at + install and validate (T16). Harden no longer judges links. The climb is + bounded by the artifact's top, not the release root: no live release has a + `..` link (the old rule refused them all), so nothing installed is newly + refused, and each artifact's digest describes a closed tree. +- Host targets carry `executable` and `executable_members` in the receipt, + shape-checked at seal by the shared `seal_artifact`, and do not apply them + (T15). Refusing members there would make a recipe's validity depend on its + binding. +- Vili's trees must + not come from `python -m venv`: its `bin/python` is an absolute link and its + `pyvenv.cfg` names the build workspace. Install packages into the relocatable + interpreter tree instead (`motion-pipeline:follow_up:vili-relocatable-python-tree`). + +### Authority + +Owner: the seal (`seal_artifact`, called by `collect_artifact` for systemd +targets and by `materialize` in `host.rs` for host targets) decides +membership from the recipe and writes it into the receipt. Harden enforces the +receipt and decides nothing. Runner exec bits and git `100755` modes are dead +inputs for release modes. Link containment belongs to `check_release_link`, +which only `digest_tree` calls. `validate_release_symlink` is deleted, and +harden's symlink arm leaves links as they are. + +### Acceptance sizing + +The big-tree job (spec `verification.operator`) needs about 12 GiB of scratch +on Yggdrasil: a 3.5 GiB tree, its staging copy, downloads and pip cache. It +removes it at the end. Revision 2 runs it on the trees as shipped, with the 300 +terminfo links (T13). If a tree holds a link the T14 rule refuses, the job +prints it and stops, and the link is reported to Self, not deleted. + +## Cut `isolation-credential-evidence` + +Spec `idunn-deploy-provenance:cut_spec:cut-cut-isolation-credential-evidence.r1`, +at `112a7a5` (origin/main, live on Yggdrasil since 2026-10-24 11:29 UTC). It +serves Vili's admission (`motion-pipeline:cut_report:cut-vili-admission.h4`, +first deviation): every `idunn up vili` is refused before fencing. + +### Body facts (probed read-only on Yggdrasil, 2026-10-24) + +- I1. `idunn up vili` runs 6 and 7 failed with + `systemd workload isolation properties differ from the Idunn contract`, the + message of `observe_unit` (`src/drivers.rs:2975-3005`). Of its conjuncts, + only one differs on the failed Vili unit: `LoadCredential`. Idunn expects + `""` because Vili has no service credential besides the presence identity + (`service_credential_names` empty); systemd shows `[unprintable]`. Every + other conjunct matches, InaccessiblePaths included (one value across all + five idunn-vili/odin units). +- I2. systemd 257.13 `systemctl show -p LoadCredential` prints + `LoadCredential=[unprintable]` for every unit, whatever the credentials: + `ssh.service` and `systemd-journald.service` (not DynamicUser, zero + LoadCredential), the four failed `idunn-vili-*` units (zero), and the Odin + unit (one). The display form carries no information, so the conjunct was + vacuous whenever credentials existed and always false when none did. Odin + and StreamPixels passed only because each carries a regular credential. +- I3. The exact value is readable: D-Bus + `org.freedesktop.systemd1.Service.LoadCredential` (`busctl get-property`) + answers `a(ss) 0` for the Vili units and ssh, `a(ss) 1 + "ODIN_TOPOLOGY_IDENTITY" ""` for Odin. +- I4. What the workload can actually read is the delivered directory. On the + host, `/run/credentials/` holds a directory only for units with credentials + (getty, journald, Odin, StreamPixels service and web). Inside Odin's MainPID + root, `/run/credentials//` holds exactly `ODIN_TOPOLOGY_IDENTITY` + (`-r--r-----+ root root`). The presence identity is a parent-only `OpenFile` + descriptor, never a credential. +- I5. The rule arrived whole in `1fc0dad` (2026-09-03, "Rebuild Idunn + deployment transaction authority"), with its twin clause in + `validate_launch_observation` (`:3468-3472`, `:3513-3516`); no later commit + touched it. No ruling or finding in any campaign concerns workload isolation + verification; gamecult-chat's foreign-workloads cut touched + `CREDENTIALS_DIRECTORY` reservation, not this check. +- I6. `SystemdWorkloadObservation` is persisted by previous Idunns in both + MessagePack encodings, compact (positional) included + (`persisted_systemd_observations_decode_as_the_systemd_variant`, + `:9915-9930`), so deleting the `load_credential` field shifts every later + slot of a live incumbent's record. The field is also in the drift list + (`:2462`) that `stop` (`:4064-4070`) uses to refuse stopping a changed + workload: changing what the field holds without leaving that list would make + the next Idunn refuse to stop the live Odin and StreamPixels incumbents. +- I7. Fixtures: the verbatim 257.13 `systemctl show` output of base's 27 + properties for a Vili unit (presence identity only) and the Odin unit + (presence identity plus one service credential), captured 2026-10-24 into + this Imagination's scratchpad `fixtures/`. +- I8. The new rule holds for every live systemd incumbent: inside each + MainPID root the delivered entries equal its binding's `secret_files` keys + minus `GAMECULT_RUNTIME_PRESENCE_IDENTITY` (Odin 1, StreamPixels service 11, + StreamPixels web 1; key names compared, no values read). An Idunn restart onto + this cut re-observes them (`:3955-3970`) and passes. + +### Decision + +A property systemd cannot display does not belong in the comparison. The +invariant it reached for, that the workload holds exactly the credentials Idunn +delivered, is observed where it is decided: the entries of +`/run/credentials/` in the MainPID's root equal Idunn's service +credential names, absent directory equal to none. That covers +`LoadCredential`, `SetCredential`, `ImportCredential` and drop-ins alike, and +needs no new program. The D-Bus read (I3) is exact but sees only one of those +properties and adds a second systemd client and parser; rejected. + +### Authority + +Owner: `observe_unit` decides the credential rule, from the delivered +directory. `LoadCredential`'s display form is no longer read or compared; the +`load_credential` field is a retired wire slot, written empty, out of the drift +list, kept only so persisted records decode. Forbidden writers: the +`observe_unit` conjunct and the `validate_launch_observation` clause. Dropping +the slot waits on the next schema revision of the observation. + +### Deploy + +Idunn's body changes only through gamecult-ops +`scripts/install-idunn-yggdrasil-release.sh` (stage `idunn` only; unit and +bindings unchanged). After it, Vili's Hands reruns from step 4 of +`motion-pipeline:cut_report:cut-vili-admission.h4`'s undone list. + +## Cut `runner-failure-evidence` + +Spec `idunn-deploy-provenance:cut_spec:cut-runner-failure-evidence.r1`, at +`804ed75` (origin/main, live on Yggdrasil). It serves the StreamPixels-area +Self: `idunn up vili` (command `up-4866467c-ee98-4a2b-b4a9-a008d7c44acd`, +transaction `tx-a04b68c7-c742-4d7d-8220-8aa08b9febae`, 2026-10-25 00:50-00:55 +UTC) failed in Sealing with `Docker runner exited with exit status: 1: ` and +nobody could tell which recipe step failed. + +### Body facts (source read at `804ed75`; host read-only, 2026-10-25) + +- R1. The runner is `docker run --rm` (`src/drivers.rs:2088-2090`), launched by + `DockerRunnerDriver::docker` (`:2173-2195`) with stdin null and + `Command::output()`: stdout and stderr are each buffered whole in the + daemon's memory, unbounded. On a non-zero exit the error is + `"Docker runner exited with {status}: {stderr}"`; stdout is dropped. The + container is gone the moment it exits, so `docker ps -a` and `docker logs` + hold nothing. +- R2. The empty message: the failing process wrote nothing to stderr, and + stdout, the only stream that carried its report, is discarded at + `:2182-2193`. The same interpolation echoes whatever a step does write to + stderr into `last_error`, `pre_fencing_abort.error` and + `completion.error` (`bounded_detail`, 2048 bytes), and from there into + `idunn up`'s `deployment failed:` line and `idunn status`. +- R3. No step identity reaches the error. `materialize` (`:2368-2391`) loops + over `declaration.steps` and calls `run_in_workspace` with argv only; + `step.id` is used only in the environment precondition. External inputs + (`curl` in the runner, `:2247-2258`) fail through the same helper with the + same anonymous message. +- R4. Which step failed on 2026-10-25, inferred from the failed transaction's + staging root, which survives (926 MiB, + `/var/lib/gamecult/idunn/staging/tx-a04b68c7-.../`): in `.runner-node-build`, + `node_modules/.package-lock.json` was written at 00:51:24 (so + `install-vili-dependencies`, `npm ci`, finished), nothing outside + `node_modules` is newer, and `release/vili` (the next step's output) is + absent; the python workspace holds only fetched inputs. The failing step was + `test-daemon` (`node --test ... scripts/test-vili-daemon.mjs`), whose test + reporter writes to stdout: exactly the stream R1 discards. Inference from + residue, not from output. +- R5. The failure path: `resume_one_transaction` (`src/control_plane.rs:6291-6320`) + sends a pre-fence error to `begin_pre_fencing_abort` (`:9916-9935`), which + persists `bounded_detail(format!("{error:#}"))`. Nothing reaches the journal + on this path: the idunn-yggdrasil journal for 00:45-01:10 holds only + provider-exclusion lines. +- R6. Stores. `control.cc` (root 0640) holds commands, transactions + (`idunn.deployment_transaction.v4`, keys 0-36), generations, supervision and + expiry requests; `ControlSnapshot::read` refuses any other type + (`:3420`). Records must re-encode byte-exactly (`decode_record`, + `:3717-3727`), so a new transaction field is a schema bump and a v4 lift + (`:4100-4110`). Terminal transactions move verbatim to `history.cc` + (`:4366-4396`; 39.8 MB on the host), which status reads for `--command`. + `incidents.cc` lives in the published projection directory and by its own + doc (`src/incident.rs:1-11`) carries no free text and is never `control.cc`, + because a type added there stops continuity for every target on a rollback. +- R7. The daemon runs as root (`User=root`). Runner secrets reach the + container as read-only file mounts named by env (`:2041-2047`), so any text + a step derives from them can land in its output. +- R8. Staging residue: 475 transaction roots, 55 GiB under + `/var/lib/gamecult/idunn/staging`; a failed Sealing leaves its workspaces + because `materialize` returns before `remove_tree_inside`. + +### Decision + +The failure becomes a typed value at the runner, `RunnerStepFailure`: the +subject (recipe step or external input, by id), the runner id, the exit code or +signal, and the last 8 KiB of stdout and of stderr with each stream's total +byte count. Its `Display` carries the subject and status, never output. +Capture streams both pipes through bounded tails, so a verbose build no longer +buffers whole in the daemon. + +The tail lives in a new private store, `runner-evidence.cc`, beside +`control.cc` (mode 0600), one record per transaction +(`idunn.runner_failure.v1`), newest 32 kept by the writer. Not in the +transaction record: that is a v5 bump whose rollback makes every older Idunn +refuse its control store (R6), the scar `incident.rs` already names. Not in +`incidents.cc`: published, and free text is barred there. Not in the journal: +journal readers are a wider set than root, so the journal line names the +transaction, the subject, the status and the store, and no output. + +No redaction. Matching secret bytes would make the daemon read every secret, +and still misses any encoding, prefix or transform of one; a claimed redaction +that leaks is worse than a stated rule. The rule instead is the reader set: the +store's only readers (root) can already read every secret the runner was +given. + +Keeping the failed container (dropping `--rm`) is worse: its output then lives +in Docker's json-file log, unbounded, untyped, outside Idunn's state and +readable by the docker group; containers accumulate with no owner to reap them; +`docker system prune` erases the evidence; and nothing ties a container to the +transaction in `idunn status`. + +### Authority + +Owner: the runner decides what failed and keeps the bounded tail; the engine's +pre-fence failure path is the one writer of `runner-evidence.cc`. +`last_error`, `pre_fencing_abort.error` and `completion.error` keep only the +failure's `Display`. Forbidden writers: the `stderr` interpolation in +`DockerRunnerDriver::docker`. Retention is by count inside the writer, so the +store needs no external owner; staging roots (R8) stay with +`ghostlight-verse:follow_up:idunn-generation-retention`. + +### Deploy + +Idunn's body changes only through gamecult-ops +`scripts/install-idunn-yggdrasil-release.sh` (stage `idunn` only; unit and +bindings unchanged). The next failed Sealing then shows its step and tail on +`sudo idunn status --command `. diff --git a/docs/deploy-provenance-target.md b/docs/deploy-provenance-target.md new file mode 100644 index 0000000..74015c7 --- /dev/null +++ b/docs/deploy-provenance-target.md @@ -0,0 +1,61 @@ +# Deployment provenance instead of an approval gate: target + +Status: target, Self, 2026-10-19 (campaign `idunn-deploy-provenance` in +Eureka's mind). It states the ends. The map will live in +`deploy-provenance-map.md`. + +## Why + +Operator, 2026-10-19: + +> The rule about the deployment brake was instituted by a cautious agent, but +> it doesn't actually suit how GameCult operates. The whole point of Eureka is +> to give the Mind, Persona and Self agency over its own Body. It's requiring +> my approval for every deployment that violates our principles. Yes, we +> certainly want to know which agent approved the deployment and why (this is +> the authority provenance which we might want to add to the transaction), but +> that's not the same as gating every deploy on my approval. + +On the same day a StreamPixels deploy (tx-b40c7240) waited on the brake +because only the root-held operator identity could release it. The agent +running the deploy could neither proceed nor record why it should. + +`F:\Projects\CLAUDE.md` says a deployment brake *may* gate artifact, revision, +configuration, schema, unit or authority changes. It does not say that +operator approval is the gate. + +## Invariants + +1. **No deployment waits on a per-transaction operator approval.** An agent + acting for the Mind, Persona or Self may admit a deployment of a target it + is authorized for. +2. **Every transaction carries typed authority provenance.** That is: who + approved it (agent or instance, and faculty), under which session or + ruling, and why. It is written before Fencing, it is immutable, and + `idunn status` and history show it. It is part of the transaction + document, not a log line. +3. **A brake that survives is a stop, not a gate.** The operator or an + authorized agent can halt deployment of a target, and the halt is itself + provenance-bearing. The deployment brake and the continuity lifecycle brake + stay separate (`F:\Projects\CLAUDE.md` Infrastructure). Neither gates Idunn + itself, and the deployment brake never gates continuity. +4. **Existing invariants hold.** These are `idunn-reincarnation` target r2, + `route-continuity-target.md` 1-7 and `authority-map.md`. + +## Scope + +In: + +- The deployment-brake release path in Idunn and `idunn-provision`. +- The provenance record on the transaction. +- `idunn status` and history projection of the provenance. +- The gamecult-ops runbooks that describe a per-transaction operator release + (`heimdall-discord-launch.md`, `idunn-host-raven.md`, the StreamPixels + runbook). + +Out: + +- `idunn-reincarnation`. This campaign is mapped after that campaign's cut 2 + merges, and coordinates with it rather than bundling. +- Harness permission rules on the workstation, which are the operator's. +- Heimdall's onboarding failures. diff --git a/docs/deployment-authority.md b/docs/deployment-authority.md index da7f028..bdc7efc 100644 --- a/docs/deployment-authority.md +++ b/docs/deployment-authority.md @@ -305,6 +305,45 @@ continuity restart does not fetch a branch, rebuild, rerun a migration, alter the launch contract, or consume the deployment brake. An explicit lifecycle brake may suspend this actuation without changing deployment authority. +Continuity never stops. Six charged attempts in an hour only open the +ContinuityExhausted incident (a page; the wire name stays, nothing is +exhausted) and gate no mint, wait or yield. When the next attempt is due is a +function of continuity's own record: a stored attempt streak, a backoff anchor +written when the pass first finds the admitted generation dead and the target +free, and a capped doubling wait from that anchor (5 s base, 300 s cap). A run +of 600 s or more by the admitted body resets the streak. A generation lost to a +host reboot (the workload driver's evidence, `lost_to_host_reboot`) is +re-materialized once per admission without charge or wait. + +Each target has two slots, the incumbent and a transaction's candidate. The +incumbent is continuity's until a transaction takes the target, at Fencing (at +Starting for a stop-then-start rollout), through one `take_target` CAS; of two +transactions entering from one view exactly one succeeds. Before that, a +deployment owns only its private candidate and the target is free for it +between continuity attempts. + +Phases Warming through Committing, post-fencing aborts and drains all carry +deadlines (the plan's frozen `PhaseDeadlines`; `WARMING_DEADLINE_SECONDS`, and +`PRE_FENCING_ABORT_DEADLINE_SECONDS` for a pre-fencing abort). What an expired +post-fencing deadline does depends on the owner. A deployment whose candidate +was never issued a write lease aborts to the incumbent. One whose candidate was +issued a lease may have written under a new state contract and is left to the +operator. A continuity retries its phase forward on a capped backoff and never +stops its live candidate, because that candidate is the only running copy of +the admitted body. Both operator cases open an `operator-required` incident +(subject: the transaction id) that closes when the record ends or its step goes +on. + +The one yield: a continuity whose route proof has not succeeded gives way to a +deployment the deployment brake has released that awaits the handover of the +target. It begins its own post-fencing abort and the deployment fences what is +admitted. A continuity that has proven its route is serving never yields. + +At boot, a lift returns to forward retry any continuity abort an older Idunn +left over a running candidate. If the lift errs or cannot observe the +candidate, the abort is held, not run. The lift is a migration with a deletion +line (follow-up `c2c-delete-boot-lift`). + An admitted generation keeps its Ready and latest Odin receipts as durable proof of the exact activation at their recorded admission times. Dependency selection reauthenticates those receipts at those times; continuity and route diff --git a/docs/guide.md b/docs/guide.md index 59ec1d7..e48bc85 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -44,7 +44,8 @@ Idunn centralizes that lifecycle work: - record every deployment/restart request and result as typed state; - record release targets, deployment artifacts, state migrations, and rollout results as typed state; -- escalate to an operator through Bifrost when human action is needed. +- record operator incidents in its published `incidents.cc`, which Bifrost reads + and delivers. Idunn never waits on delivery. The desired shape is simple: @@ -305,11 +306,11 @@ layer must call `.ps1` launchers directly instead of `cmdPath` trampolines. Those three live Raven tasks have been repaired and verified. The repair actuator now uploads its Raven PowerShell body with `sftp` and runs a tiny cleanup wrapper, so the hidden-task repair does not hit Windows command-line -limits before it can run. The long-running Muninn serve bodies on Raven, -Nightwing, and Starfire now also carry their own `--idunn-rudp-health`, -`--idunn-daemon`, and `--idunn-health-contract` arguments, and live Idunn -accepts those daemon-owned health records directly instead of relying on -one-shot health commands for the truth. +limits before it can run. The Muninn serve bodies no longer publish a +daemon-health document: `--idunn-rudp-health`, `--idunn-daemon` and +`--idunn-health-contract` are retired flags that the binary rejects. Raven's +liveness is Odin-correlated runtime presence; Starfire and Nightwing are +checked by their `scripts/health-*-muninn.ps1` witness scripts. From here on out, the work is not "teach another daemon to speak." It is "delete the witness masks and keep ownership where it belongs." diff --git a/docs/host-actuator.md b/docs/host-actuator.md index 499ab23..c595c5d 100644 --- a/docs/host-actuator.md +++ b/docs/host-actuator.md @@ -82,8 +82,9 @@ running a script, and a dead `muninn serve` is seen the tick it dies. 2. No path on yggdrasil can start, stop or observe a host workload except through the actuator session bound to that host's anchor. 3. A host with no attached actuator is a failed observation, which - continuity counts and gives up on after three attempts, exactly like a - unit that will not start. + continuity counts as a charged attempt and retries on its capped backoff + (it never gives up; six in an hour only pages), exactly like a unit that + will not start. 4. The actuator cannot be told to run a program the recipe did not declare; `allowed_programs` on the host-native runner is enforced on the host. @@ -96,7 +97,8 @@ Each of these was found by running the pipe against Raven, not by reading. bind time. `rollout.strategy = "stop-then-start"` exists for exactly a route-less host-actuator workload: the admitted incumbent is stopped at Starting, before the candidate starts, and supervision leaves the target - alone from Starting on. A systemd-transient workload may not declare it. + alone from Starting on: the transaction takes the target there (one + `take_target` CAS) rather than at Fencing. A systemd-transient workload may not declare it. - **An unobservable host is silence.** The hub returns a typed `HostUnobservable` for not-attached, detached, reconnected and unanswered. Supervision skips that tick and logs it. Before this, an Idunn restart diff --git a/docs/migration.md b/docs/migration.md index 0c810fc..a78b4a0 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -520,7 +520,7 @@ started, ran, and then died on a file it could not open: | `runtime_root` and every directory above it | traversable by the workload: `o+x`, or group-owned by the state group with `g+x` | the workload is a `DynamicUser` and owns nothing, and `ReadOnlyPaths=` binds the bundle's leaf into the namespace without granting traversal above it. Idunn now checks the whole chain and names the closed directory | | `runtime_root` | setgid, group-owned by the state group (`2750 root:`) | `harden_root_authority_file` requires the write-lease record's group to equal its parent's, and only setgid makes a newly created record inherit it. Without it the record lands `root:root` and the workload cannot read its own lease | | the write-lease record | `root:`, mode `0640` | follows from the setgid parent; the workload reads it through the state group. Idunn writes it, so fixing an existing wrong-group record means deleting it, and that is safe only when no transaction is live | -| `/var/lib/gamecult/idunn-projection` | default ACL `u::rw-,g::r--,o::r--` | Idunn runs with `UMask=027`, so each publish creates the store and its `.lock` at `0640`. The code re-opens them after every write, but that leaves a window; a default ACL sets the mode at creation and closes it | +| `/var/lib/gamecult/idunn-projection` | default ACL `u::rw-,g::r--,o::r--` | Idunn runs with `UMask=027`, so a store is created `0640` and each publish re-opens it `0644`; the default ACL sets the mode at creation and closes that window for stores. The `.lock` is not published and no reader opens it: Idunn creates it `0600` (replacing one found wider, so a descriptor another uid already holds is orphaned), and the ACL never decides its mode | | `/etc/nginx/idunn-stream-routes` | `root:root`, not group-writable | "route authority parent is not canonical root-owned and nonwritable". It was `root:idunn 0775` from the generation where Idunn ran unprivileged and needed to write it as a group member | Two shapes are worth stating plainly because they are easy to get backwards. diff --git a/docs/reincarnation-map.md b/docs/reincarnation-map.md new file mode 100644 index 0000000..4f99aed --- /dev/null +++ b/docs/reincarnation-map.md @@ -0,0 +1,600 @@ +# Reincarnation after reboot: map + +Status: map, Imagination, 2026-10-18 (campaign `idunn-reincarnation`, session +`idunn-reincarnation-2026-10-18`). The ends are in +[`reincarnation-target.md`](reincarnation-target.md). Questions, rulings, the +cut spec and follow-ups live in Eureka's mind; this page holds what has +no kind: body facts, the model page, the authority map's rationale and the +cut sequence. Line numbers are at `74a7008`; `src/` is byte-identical to +`c2477d4`, so the earlier anchors still hold. + +Cut 2 pass, Imagination, 2026-10-19 (same campaign and session): body facts +B13-B18, the cut-2 rows of the model page and the cut-2 parts of sections 3-5 +are anchored at `9a0a6a2`, whose `src/` is byte-identical to `ffb4293` (cut 1's +head). The B1-B12 anchors above stay at `74a7008`. + +Read the in-force set with the rulings-in-force and open-follow-ups recipes +for root `idunn-reincarnation`; the cut in flight is the specs-with-no-report +recipe. The first spec (`cut-c1-durable-stop.r1`) was withdrawn when the +operator ruled that continuity never stops; its replacement is cut +`c1-never-stop`. Cited from `idunn-watchdog`: `exhaustion-close-rule-op` and +`paging-conditions`. + +## 1. Body facts + +Evidence: the `idunn-yggdrasil` journal from 11:40 UTC (94,670 lines), the +per-unit journals of the streampixels-service candidates and of Odin +(`idunn-odin-7cfe5cd1…`), `idunn status` and `idunn status --command ` +on Yggdrasil, `journalctl -u nginx`, and the source at `c2477d4`. All times +UTC, 2026-10-18. + +**B1. The reboot did not stop continuity from re-materializing anything.** +Every streampixels-service continuity attempt minted a fresh transient unit +from the installed release `sha256-df663f0a…`: 54 continuity commands from +`continuity-85887ef2` (11:46) to `continuity-52bbbd66` (19:57), most listed by `idunn status` as +`continuity-… failed systemd unit idunn-streampixels-service-.service +is not running`. A continuity transaction is a full `DeploymentTransaction` +built by `from_continuity` (`control_plane.rs:1678`) from the admitted plan, +sealed release, installed release and Expected. It runs every phase: +Starting, Warming, Fencing, Leasing, AwaitingReady, Routing, Committing. The +absent admitted unit `461c6443…` was never the problem. Each new candidate +was. + +**B2. Every candidate died of a dependency, not of itself.** The +streampixels-service process exits about 30 s after start when Odin does not +admit its initial presence. Unit `2bc02905…` at 11:47:21 logged "StreamPixels +service failed to enter its Idunn incarnation: Error: Odin did not admit the +initial Idunn runtime presence ... Timed out waiting for CultNet RUDP response +after 5000 ms." The same is logged by `42ecf363…` at 13:47:13, `03acf44d…` +at 19:47:14, `45276008…` at 19:49:54 and `599b705b…` at 19:51:18. Odin's +RUDP endpoint is `rudp://10.77.0.1:17871`, which nginx proxies. nginx failed +at 11:46:26 ("bind() to 10.77.0.1:17871 failed (99: Cannot assign requested +address)") and started at 19:46:24. Between 11:46 and 19:44 Odin's journal +shows no presence message from streampixels-service at all: none arrived. +After nginx returned, the first one arrived at 19:49:25 and was rejected: +"runtime presence has no exact current activation and provider anchor". The +attempts minted at about 19:53 and 19:55 got past Fencing and then failed +"the Leasing phase passed its deadline" (FailedAfterFencing, RestoreIncumbent). +Other early failures: "observed write lease is not the current Idunn grant" +(11:46:47, the first attempt after boot), and three "reading process stat +/proc/N/stat: No such file or directory" races. + +**B3. Every attempt was charged at mint.** `record_restart(now)` runs inside +the mint CAS (`control_plane.rs:6156-6157`). Nothing reads a continuity's +outcome back into the log. The only exemption anywhere is `HostUnobservable` +(`:5944`), and it applies to an observation, not to a candidate. So the 6-per- +hour window (`CONTINUITY_RESTART_ATTEMPTS` = 6, window 3,600,000 ms, `:71-72`) +filled at 11:50:52 and refilled every hour as entries slid out. The bursts are +in the journal at :47, :48, :49, :51, :52 and :53 of every hour, 39 times +"Idunn stopped restarting admitted streampixels-service". This answers Q2: +the nginx outage counted against the body's budget, and the place that decides +it is the unconditional charge at `:6156`. + +**B4. The deployment was killed by a window slide, not by a live +continuity.** `up-3ad6b05b` (release `sha256-67e227b2…`, a new release) was +recorded at 20:24:51. It was still before Fencing at 20:46:41, when it was +completed `FailedBeforeFencing` with "admitted incumbent failed before +candidate fencing; deployment yielded to continuity: systemd unit +idunn-streampixels-service-461c6443….service is absent". Two seconds later +continuity minted `tx-0804185a` with the OLD release `df663f0a…`, and it was +admitted. The restart that left the window was minted at about 19:46:40 (its +unit `03acf44d…` started at 19:46:42), exactly one window before the abort. +The mechanism, from the source: + +- `:5805-5821`: a live transaction for the target is the `blocker`. From + Fencing on (or Starting for stop-then-start) supervision skips the target + entirely. +- `:6033-6061`: with a blocker present, continuity never restarts and never + reports. It may abort the blocker, and the predicate for that is + `Deploy && !supervision.restarts_exhausted(now) && phase < Fencing && no + abort && not held` (`:6045-6050`). +- `:6062-6084`: only with no blocker does an exhausted continuity open the + incident and say "The target is free for a deployment to replace it". + +So from 20:24:51 the deployment owned the target (`owns_target_authority`, +phase != Complete, `:1772`). Continuity deferred to it through the blocker +`continue` at `:6060`. The deployment deferred to continuity whenever the +window had a free slot, through `:6046`. "Stopped" existed only as a durable +incident (incidents.cc, a projection) and a report. The ownership decision +read the window. When the window gained a slot, continuity reclaimed the +target with no record that it had resumed. This answers Q3: the two owners are +the live `DeploymentTransaction` (it owns while not Complete) and +continuity-as-window (it owns whenever the window is not full). Continuity +defers to the deployment while any transaction is live. The deployment defers +to continuity while the window has room. The "free for a deployment" message +names the first ownership, and the code acts on the second. + +**B5. Fencing an absent incumbent already works.** `advance_fencing` +(`:6869-6952`) stops the exact incumbent and revokes its lease. The systemd +driver's `stop` returns `Ok(())` for an absent unit (`drivers.rs:4003-4007`). +A deployment that is not aborted therefore replaces an absent incumbent with +nothing to fence, which is target invariant 3's last clause, already true. +`exact_incumbent` (`:7556-7569`) fails "target incumbent changed after +transaction creation" if a continuity commit replaced the generation during +the deployment. That is why a phase-split handover needs a rebase +(`handover-pre-fencing`). + +**B6. The two rejection streams are consequences of a dead admitted +generation, not separate faults.** "preserved admitted streampixels-service +after rejecting topology observation: topology correlation does not bind the +current activation" was logged 46,271 times, from 11:46:49 to exactly +20:46:48, about 5,300 an hour. Ghostlight's 42,482 identical lines ran to +19:56:29. Since 20:47, raven-muninn, newly exhausted, shows the same line +621 times. `refresh_admitted_topology` (`:6395-6457`) authenticates Odin's +correlation against the admitted generation's activation. When that +incarnation is dead and Odin's latest record names another one (a +continuity candidate's, or a stale one), authentication fails with an `Err` +rather than `Ok(false)`, and `:5834-5840` prints it on every pass. The pass +order is what does it: route supervision (`:5822`) and topology refresh +(`:5834`) run before the workload is observed (`:5891`). So a dead +generation is "supervised" every tick. "rejected admitted +streampixels-service route continuity: admitted process write lease is no +longer exact" (3,615, from 20:02:23 to 20:46:48) followed the two +post-fencing aborts at 19:54 and 19:57. Fencing revoked the dead incumbent's +lease, "restore incumbent" cannot re-grant a lease to a process that does not +exist, and `supervise_admitted_route` (`:6250-6255`) then refuses the +admitted generation on every challenge. Both streams stopped the moment +`tx-0804185a` was admitted. This answers Q4: both streams follow from one +model gap, that a dead admitted generation is treated as a live incumbent. +That gap is cut 4. They are not the deadlock. + +**B7. Ghostlight is a separate fault.** Its continuity `tx-039b91e9` is +wedged in a post-fencing abort. It was "lease-not-adopted" at its AwaitingReady +deadline, and the abort withdraws the candidate's lease (`:8895-8928`) before +it stops the candidate (`:8929-8946`). The candidate unit `6c51e35e…` has +been running since 19:55:15 and holds the lease's shared lock, so every +attempt fails "process write lease is held by a shared-lock consumer" +(`drivers.rs:4179/4224`). The transaction owns the target, so continuity is +locked out and the route degraded (53 consecutive failures). This is follow-up +`ghostlight-abort-lease-before-stop` (route-continuity invariant 5), not the +stopped-owner deadlock. + +**B8. Unexplained, recorded.** Idunn's record of Odin's stable route reads +"degraded-since 11:46:31, consecutive-failures 1, next-challenge-at 11:47:01" +about nine hours later. It has not been re-challenged since. This is follow-up +`odin-route-record-stale`. It blocks cut 3. Under `budget-never-stop`, a +failure attributed to a declared dependency must not grow the backoff, and the +only dependency-health record Idunn holds for Odin's stable route is this +one. + +**B9. Luna's weekend commits on this path (Q5).** None of them caused the +deadlock. The yield predicate dates from the route-continuity and watchdog +campaigns, not from the weekend. Where they touch: + +| Commit | On the path? | Status | +|---|---|---| +| `9f00e7a` replay provider receipts at admission time | Yes: the deployment's Sealing compile (`advance_sealing` → `current_ready_provider_tokens`) logged "excluded non-current provider ghostlight / raven-muninn: admitted provider's latest receipt is not Ready at its admission time" at 20:24:53 (`:8588`, `:8609`) | Still live: route-continuity B4 has not landed. It did not block this deployment | +| `29946bb` wait for the stable route listener | Yes: the route proofs during the outage were "Connection refused" on 127.0.0.1:8831-8833 | Kept by route-continuity (honest scope) | +| `8ae00a1` cancel live stateless candidates | No: no cancel ran | Still present (`:2888`); route-continuity S2 owns its deletion | +| `ae8ac45`, `8651906`, `e976d32`, `5e94eb8` identity and launch evidence | Only in `observe` and `stop`; nothing here drifted | Kept | +| `a148802` | No | Reverted (`87fa57d`) | + +The target forbids building on them. Cut 1 touches none of their lines. + +**B10. The stop was the only door a deployment had into a crash loop.** +Source read at `74a7008`. A queued `idunn up` becomes a transaction only in +`freeze_command`, which refuses while any transaction for any of its targets +`blocks_new_target_mutation` (`:5741-5747`); the command just stays queued. +The scheduler tick runs `retire_one_terminal_transaction` (returns early when +it retires), then `resume_one_transaction`, then +`supervise_one_admitted_generation`, then `freeze_one_queued_command` +(`:5414-5423`). Today's doubling wait is anchored at the previous attempt's +**mint** (`next_restart_at`, `:1163-1170`). So once an attempt lasts longer than +its wait, the tick after its transaction is retired mints the next attempt +before the freeze runs, and the queued deployment never finds the target +free. At `c2477d4` that ended when the window filled and continuity stopped. +Delete the stop without opening another door and a crash-looping release can +never be replaced. The door the cut uses is kubelet's own anchor +(`docs/research/restart-budget-prior-art.md` §1.1, the exited container's +`FinishedAt`): the wait runs from the first supervision pass that finds the +target dead **and free**. That leaves at least one wait (5 s or more) with no +live transaction, and the freeze runs every tick. This is reasoning over the +source, not a probe. The cut pins it with a tick-level test. + +**B11. The yield cannot outlive the stop.** With the window term gone from +`:6046`, the yield aborts every pre-fencing Deploy whenever the incumbent is +dead (`:6045-6059`). For a crash-looping release the incumbent is always +dead, so the deployment that would fix it is aborted at the first +supervision pass after it is frozen. That is the 2026-10-18 failure: on that +day the operator's `idunn up` existed to replace a release that would not +start. Ruling `handover-phase-split` already says nothing yields before +Fencing. So cut 1 deletes the whole yield. Continuity keeps deferring to any +live transaction for its target (`:6033`, the `continue`). Cut 2 keeps +the incumbent alive alongside the build. Between them, a dead incumbent stays +down while a pre-fencing deployment over it builds. That is bounded by the +deployment's own deadlines, and the deployment then fences it as nothing +(B5). + +**B12. The doubling function already exists once.** `backoff_wait` +(`:4000-4007`) doubles from a base up to `RESUME_BACKOFF_CEILING_MILLIS`, and +has one caller (`back_off`, `:5621`). Continuity's wait is a second spelling +of the same rule (`5_000 << (live.len() - 1)`, `:1169`). The cut gives +`backoff_wait` a ceiling parameter and deletes the second spelling. +`#[cultcache(key = N, default)]` is how an absent key decodes as its default +(precedent `DeploymentTransaction` key 34, `:1552-1557`). New +`TargetSupervision` keys use it, because every record on Yggdrasil predates +them. + +**B13. Two live transactions for one target make the control store +unreadable.** Source read at `9a0a6a2`. `ControlSnapshot::validate` refuses +"multiple transactions claim current-incarnation authority for one target" +over `owns_target_authority` (`:3245-3254`), and that predicate is just +`phase != Complete` (`:1896-1898`). Every reader of the store goes through it: +the scheduler tick, `idunn status`, cancel, expire. So if supervision minted a +continuity beside a pre-fencing deployment today, the next read would fail +the whole engine. Coexistence needs the identity rule changed first, and that +is the cut's first commit. + +**B14. One predicate answers four questions, and the phase rule lives in +one caller.** `blocks_new_target_mutation` (`:1903-1909`, live or post-commit +cleanup pending) is read by the freeze busy check (`:5934-5942`), the +supervision blocker (`:5998-6001`, a single `.find`), candidate-port +reservation (`:7997`) and status (`:4613`). `owns_target_authority` is read by +validation (`:3249`) and by sibling ordering inside one command (`:3384`). +Neither looks at the phase. The only phase-aware ownership rule is inline in +supervision (`:6002-6012`, owns from Fencing, or from Starting for +stop-then-start). The freeze refusal is why a `profile:` command needs every +target free in one tick (follow-up `profile-deploy-joint-gap`). + +**B15. What a pre-fencing deployment reads about its incumbent.** Warming's +semantic warming check uses the incumbent's lease digest +(`:6904-6905`, `is_semantic_warming` `:9530-9560`: Odin reports the candidate's +one disagreement as the lease the incumbent holds). Route preflight uses the +incumbent's route observation (`:7026-7047`). Isolation is proven against the +incumbent's workload (`:7050-7076`). Each goes through `exact_incumbent` +(`:7768-7781`), which bails "target incumbent changed after transaction +creation" once a continuity commit replaced the generation. Sealing, and +Starting for candidate-then-promote, read no incumbent. Stop-then-start's +Starting stops it (`:6857-6863`), which is already the owner's act. The +Sealing-to-Starting write is a `replace_transaction` at `:6806-6808`, not +`transition` (`:8914-8927`), so the two edges at which a transaction can take +a target are written by two different functions today. + +**B16. The commit already guards the incumbent; nothing guards the +handover.** `advance_committing` expects the admitted envelope it read +(`:7701-7702`, CAS `:7743-7764`), so a deployment can never commit over a +generation that moved under it. Entering Fencing is an unguarded +`replace_transaction` of the transaction alone. Continuity's mint CAS +(`:6371-6400`) guards its command, transaction and supervision record, and no +other transaction. The store accepts read-only expectations: an expected +entry with no replacement only guards (CultLib +`packages/cultcache-rs/src/lib.rs:1327-1368`). One engine holds the store +(`ProcessLock::acquire`, `:5514`), but the CLI writes transactions too: +`cancel` sets a pre-fencing abort on a live deployment (`:3014-3083`). So a +handover decided by one CAS over the target's records is exact, and a tick +order is not. + +**B17. What coexisting candidates share, and what they do not.** Continuity +reuses the admitted plan, so its candidate port is the admitted generation's, +already reserved (`:7988-7992`); a deployment picks another. Topology is keyed +per incarnation (`:8495-8498`): a continuity publishes under the admitted key, +a deployment under its own. No candidate holds a write lease before its own +Leasing, which follows Fencing, so two pre-fencing candidates cannot both +write. Route actuation by a continuity is always `Survival`, never refused +(`:4870-4876`); a deployment's `Forward` cannot starve it. The Odin replay +floor `max_odin_sequence` (`:3399-3435`) spans every transaction of the target +and the admitted cursor; incumbent refresh and candidate warming already share +it during every deployment today, so the replay check (`:9498-9501`) gains no +new exposure. Stop-then-start is declared only by route-less host-actuator +bindings (`deployment.rs:756-764`, `:1245-1249`), and no control-plane test +builds one (grep `StopThenStart` in `src/control_plane.rs`: the definition +site only). It is live on Yggdrasil: raven-muninn declares it +(`git grep stop-then-start` in `gamecult-ops`: +`idunn/yggdrasil/bindings/raven-muninn.toml.in`). So the Starting edge of the +handover carries production traffic, and its engine-level proof is owed +before the closing install (follow-up `stop-first-handover-fixture`). + +**B18. A pre-fencing error is an abort.** `resume_candidate` turns any `Err` +from a pre-fencing step into `begin_pre_fencing_abort` unless it is +`LockContended` (`:5768-5783`). So anything the cut makes a deployment wait on +must return `Ok` (a gate wait, `record_gate_wait` `:8170-8182`), or the wait +becomes the yield that ruling `handover-phase-split` forbids. The same path is +how a superseded continuity ends itself, which is wanted. + +## 2. Model page + +One row per persistent kind the campaign touches. + +| Kind | What names it | Lifecycle | Who decides | +|---|---|---|---| +| Paging log: `TargetSupervision.continuity_restarts` (control.cc, `idunn.target_supervision.v1`, key 3) | key = target | The newest 6 charged attempt times, ascending, read through a sliding 1 h window. Outlives every generation. After cut 1 it is the paging threshold only (`restarts_page`): it opens and holds the ContinuityExhausted incident and is shown in status. It gates no mint, wait or yield | Continuity supervision, pushed in the mint CAS | +| Attempt streak (new, cut 1: key 7, `default`) | target | Charged attempts since the admitted body last ran a sustained run. +1 in each charged mint CAS. Reset to 0 when the dead generation was admitted after the last charged attempt and ran ≥ 600 s before it was found dead. Saturating, and the wait is capped, so it can be unbounded | Continuity supervision only | +| Backoff anchor (new, cut 1: key 8, `default`) | target | Written on the first pass that finds the admitted generation dead and the target free after an attempt. The next attempt is due at anchor + `backoff_wait(5 s, streak - 1, 300 s)`. Cleared by every mint. Stale, and ignored, once older than the admitted generation's `admitted_at` | Continuity supervision only | +| Re-materialization bound (new, cut 1: key 9, `default`) | target | Set by the one attempt that is not a backoff step, when the driver reports a reboot loss. Spent while it is not older than the admitted generation's `admitted_at` | Continuity supervision, from the workload driver's evidence | +| Projection deferral (keys 4-5) | target | Unchanged: a failed demotion defers the next mint 30 s. It spaces a re-materialization too | Continuity supervision | +| Admitted generation (`idunn.admitted_generation`) | key = target, `generation_id` = `generation-` | Replaced atomically by every committed deployment or continuity. A dead incarnation keeps its record until replaced | The committing transaction | +| Incident `ContinuityExhausted` (incidents.cc) | condition + subject (target) | Opened at 6 charged attempts in the hour while the target is dead and free. Closed when the target runs and the window no longer holds 6 (`exhaustion-close-rule-op`). Retired after retention. The name is the wire name and stays, though nothing is exhausted any more | Idunn's incident writer, a notification projection: never read by ownership | +| Transaction and pre-fencing abort (`idunn.deployment_transaction.v4`, history.cc once terminal) | `tx-` and its `command_id` (`up-…` or `continuity-…`). Keyed by transaction, so a continuity and a deployment for one target are two records with no shared key | Sealing … Complete. A pre-fencing abort completes `FailedBeforeFencing`, a post-fencing abort `FailedAfterFencing` with a named recovery. Retired to history. After cut 2, per target at any instant: at most one live Deploy, at most one live Continuity, at most one owner (validated by `ControlSnapshot::validate`, B13) | The scheduler advances it. After cut 1, continuity supervision never aborts a transaction; the transaction's own steps do. After cut 2, a continuity whose target was taken, or whose source generation was replaced, aborts itself (superseded) | +| Target ownership (cut 2: derived, no record) | target | A transaction owns its target from `takes_target_at()` (Fencing, or Starting for stop-then-start) until Complete; a Complete transaction with pending post-commit cleanup drains it. Entered only through the handover CAS (`Engine::take_target`) | Derived from the phase. The handover CAS is the one writer of a phase entry into `takes_target_at()` | +| `incumbent_generation_id` on a transaction (key 9, unchanged format) | the generation's id | Deploy: set at freeze; before the handover it names the generation its incumbent-relative evidence was proven against and is rebased when the admitted generation moves (clearing that evidence); pinned by the handover CAS, exact from then on. Continuity: its source generation, never rebased | Deploy: its own pre-handover rebase step and the handover CAS. Continuity: the mint | +| Unit materialization (systemd transient unit `idunn--`) | the activation's hash | Created by Starting. Lost on a host reboot (`/run/systemd/transient`). A failed unit stays loaded; a cleanly stopped one is collected | The workload driver actuates. Whether a loss was a reboot is the driver's evidence (`WorkloadPort::lost_to_host_reboot`, cut 1) | + +No cell is open. Both forks are ruled (`budget-never-stop`, +`handover-phase-split`). The numbers that `budget-never-stop` left open are +the defaulted ruling `continuity-backoff-defaults`. Cut 2 opens one question, +`fencing-over-owning-continuity` (does a deployment ready to fence wait for a +continuity that already owns the target, or preempt it); the spec is written +for the recommended answer, wait. + +## 3. Authority map: continuity and deployment + +The authority map for cut 1 is in its spec. This section is the +campaign-level map, and cut 1 is one step of it. + +- **Owner.** Exactly one authority actuates a target at each instant. A live + `DeploymentTransaction` owns the target from Fencing on, and a stop-first + rollout owns it from Starting. Before Fencing (`handover-phase-split`), a + deployment owns only its private candidate, and continuity owns the + incumbent slot. Continuity owns the admitted generation's survival and + never stops. When it next acts is a function of its own record: the + streak, the backoff anchor and the re-materialization bound. +- **Inputs.** For continuity: the admitted generation (id, `admitted_at`, + workload, plan), the workload observation and the driver's reboot + evidence, its own record, and whether a live transaction blocks the + target. After cut 3 it also reads Idunn's dependency-health observation. + For a deployment: its own plan, its brake, and at Fencing whatever + generation is admitted at that moment (cut 2). +- **Outputs.** Continuity: mints, either charged or the one re-materialization; + backoff anchors; deferrals; and, as notifications only, the + ContinuityExhausted incident and its once-report. Deployment: a new admitted + generation, or an abort that names its recovery. +- **Derived state.** `restarts_page(now)` (6 charged attempts in the hour) is + derived from the paging log and is notification-only: incident open and + close, and status. The next attempt's due time is derived from the + anchor and the streak. The incident and `idunn status` are projections. +- **Forbidden writers.** The paging window gates no mint, no wait and no + yield. Continuity supervision never aborts a deployment: + `begin_pre_fencing_abort` has no caller in it. No deployment path writes + continuity's fields. The incident store decides nothing. No stopped state + exists: no field, and no branch that ends a pass because a count is + reached. +- **Shared paths.** A crash restart, a restart after a sustained run, and a + reboot re-materialization all mint through the one CAS (`:6158-6183`). The + dead-and-free pass is the only writer of the backoff anchor. Every + deployment, operator or scheduled, enters through `freeze_command` and + advances through `advance_transaction`. +- **Deletion line.** Cut 1 deletes the yield (`:6034-6059`), the + exhaustion `continue` and its "free for a deployment" text (`:6062-6085`), + and the mint-anchored wait (`:1160-1184`). Cut 2 (anchors at `9a0a6a2`) + deletes `owns_target_authority` and `blocks_new_target_mutation` + (`:1894-1909`), the inline phase rule and the single-blocker `.find` in + supervision (`:5998-6012`), the blocker `continue` (`:6229-6233`), the + freeze busy check (`:5934-5942`), the one-live-transaction-per-target + validation (`:3245-3254`), and the "target incumbent changed after + transaction creation" text (`:7779`). + +### Cut 2: the two slots and the handover + +Each target has one **incumbent slot**: its `AdmittedGeneration`, the body +running it, its write lease and its stable-route membership. Each live +transaction has its own **candidate slot**: its incarnation key, activation, +candidate port and unit. + +- **Owner.** Before any transaction takes the target, continuity supervision + owns the incumbent slot: it alone decides to mint a restart. From the + handover CAS until Complete, the one transaction that took the target owns + it, fencing, leasing, routing and committing. Its commit hands the slot + back to supervision as a new generation. A transaction owns its own + candidate slot from mint or freeze until it is terminal. A pre-handover + deployment owns nothing else. +- **The handover.** `Engine::take_target` is the only writer of a phase entry + into `takes_target_at()` (Fencing, or Starting for stop-then-start), for + both kinds. It is one CAS. It expects the transaction's own envelope, the + target's admitted envelope (the one `incumbent_generation_id` names, or + absent), and every other non-terminal transaction of the target, each + unchanged, and it writes only the transaction. Of two transactions + entering from one view, exactly one CAS succeeds. The loser's next step + re-reads: a deployment gate-waits for the owner, and a continuity aborts + itself as superseded. +- **Inputs.** Continuity: as cut 1, plus whether any transaction owns or + drains the target, or a continuity is live for it (the mint gate). A + pre-handover deployment: its plan, its brake, and the admitted generation as + it is now, which it rebases onto. The handover: the target's records above. +- **Outputs.** The handover writes the new phase. A rebase rewrites + `incumbent_generation_id` and clears the evidence proven against the old + incumbent: `warming`, `route_preflight`, `isolation`. A superseded continuity + writes its own pre-fencing abort. +- **Derived state.** `owns_target()` is derived from the phase and + `takes_target_at()`. `drains_incumbent()` is derived from the post-commit + cleanup. The freeze refusal is derived from "a live Deploy for this + target". `incumbent_generation_id` on a pre-handover deployment is the + generation its evidence was proven against, not a pin. +- **Forbidden writers.** No path enters `takes_target_at()` except + `take_target`. Continuity supervision aborts nothing, and mints nothing + while another transaction owns or drains the target or a continuity is + live. A deployment never aborts because another transaction owns the + target: it gate-waits (B18). A continuity is never rebased. Freeze reads no + continuity state. No new persistent record decides ownership: it is the + phase. +- **Shared paths.** Candidate-then-promote's Warming-to-Fencing and + stop-then-start's Sealing-to-Starting go through one phase-entry seam, + which calls `take_target` exactly when the entered phase is + `takes_target_at()`. Deployments and continuities use the same CAS. A + single-target and a `profile:` command use the same freeze rule. + +Demotions, named: `incumbent_generation_id` is no longer a pin before the +handover; it is derived from the admitted generation by the rebase. +`blocks_new_target_mutation` is no longer an owner; it splits into +`is_live`, `owns_target` and `drains_incumbent`, each read by the consumer +that asks that question. The freeze busy check is no longer a +target-ownership gate; it guards only one live deployment per target. + +Demotions, named: `restarts_exhausted` is no longer an owner of anything. It +is renamed `restarts_page`, and it opens and holds the incident. The +ContinuityExhausted incident was never read as "stopped", and there is now no +stopped state for it to stand for. The "free for a deployment" report is +deleted: every target is free for a deployment between attempts (B10). + +## 4. Rationale + +The fault is a split brain, not a missing rule. Continuity's "I have stopped" +was written in two places, a durable incident and a log message, and neither +was what the ownership decision read: the yield re-derived "stopped" from a +clock. The withdrawn first spec moved "stopped" into control state. The +operator's ruling is better, and it is also smaller: there is no stopped +state. The window it was derived from becomes a pager. Every case the stop +existed for is then handled by an owner that already exists. A broken release +is replaced by a deployment, which gets in through the gap after each +attempt (B10) and is never yielded (B11). A host-dependency outage costs one +attempt every five minutes or so, and pages. + +Why the wait is anchored at "found dead and free" rather than at the mint: +the anchor is the door. A mint-anchored wait can be swallowed by a long +attempt, and then no tick ever sees the target free (B10). Kubelet anchors +CrashLoopBackOff on `FinishedAt` for the same reason, so this is the +established spelling, not a new one. A streak is stored, not recomputed from +the paging log. The log holds only six entries and slides by the hour, while +the streak must outlive both. Reading the backoff from the window count was +today's spelling, and it is exactly the mutant the cut's tests must kill. + +The reset, "admitted after the last charged attempt and ran at least 600 s +before it was found dead", needs no new writer. Supervision observes the +admitted generation on every pass, so the first pass that finds it dead is +within one poll of the death. A death hidden behind a live transaction or an +unobservable host overstates the run, and so resets early. That is the safe +side, because it shortens one wait. + +The reboot invariant needs evidence of what happened, not a guess. "Unit +absent" alone is not a reboot, because a cleanly stopped transient unit is +collected too. "Host booted after admission" alone is not a loss, because +the unit might have been started this boot. The systemd driver owns both +facts (a `systemctl show` that returns nothing, and the `/proc/stat` btime). +So it answers one question, `lost_to_host_reboot`, and supervision keeps the +budget. The once-per-admission bound is still needed under never-stop. A +failing re-materialization leaves the same admitted generation, unit absent, +booted after admission. Without the bound it would repeat with no step, no +wait and no page, every tick. The clock comparison is safe: admission +happens at least one full transaction, tens of seconds, after boot. + +What cut 1 deliberately does not fix. A dependency failure still grows the +streak (cut 3). A dead incumbent stays down while a pre-fencing deployment +builds over it (cut 2). A dead generation is still route-supervised and +topology-refreshed every tick (cut 4). Paging residual: the cap gives 3600 / +(300 + D) attempts an hour, where D is the attempt's duration, so a crash +loop whose attempts last longer than 300 s settles under 6 an hour. Its +incident opens in the first hour's burst and stays open while the release +never runs, but a release that runs for one pass after each restart can see +it close and not reopen. That is the operator's threshold, applied as ruled. +It is recorded here, not changed. + +Replayed against 2026-10-18 with only cut 1. The reboot loss at 11:46 is +re-materialized at once, with no step. It fails on the nginx outage. The +next attempt resets the streak, because the generation had run for hours, +and then waits 5, 10, 20, 40, 80, 160 s, and 300 s after that. The incident +opens within the first hour and stays open (the release never runs), so the +operator is paged during the outage, not at 20:24. The `idunn up` at 20:24 +freezes in the next gap. Continuity defers to it, nothing aborts it, and it +fences the absent incumbent as nothing and admits the new release. + +Why cut 2 is shaped this way. The handover is the phase, guarded by one CAS +over the target's records, rather than a new per-target "owner" record. A +record would be a second spelling of what the phase already says, and it +would need its own lifecycle, crash repair and migration. A CAS that reads +every other live transaction of the target is enough because a newly minted +or frozen transaction starts at Sealing and cannot own anything (B16). +The rebase clears exactly the evidence that names an incumbent (B15): warming +names its lease, preflight its route and isolation its workload. Keeping any +of them would let the deployment fence one generation with proofs about +another. A deployment waits in Warming while another transaction owns the +target, because Warming's steps read the incumbent that owner is fencing, and +a failed pre-fencing step is an abort (B18). Sealing and Starting keep +running, so the long part, the build, still overlaps. A continuity that loses +the target is aborted, not parked: it restarts a generation that is about to +be replaced, and while it lives it blocks continuity of the generation that +replaces it. + +Prior art on question `fencing-over-owning-continuity` (wait for a +continuity that already owns the target, or preempt it), from +`docs/research/restart-budget-prior-art.md` at `74a7008`. It leans the other +way from the recommendation, and the recommendation says why. Kubernetes' +rolling update scales old ReplicaSets down unhealthy pods first, so that +"unhealthy replicas will block deployment" does not happen (§1.4, +`rolling.go`). That is preemption of a failing old replica. Nomad separates +the two by layer: the client restarts old allocations under their own policy, +and the server reconciler places the new ones (§2.3). Neither waits on an +old replica's restart. But deleting a pod is the kubelet's ordinary teardown. +In Idunn, preempting a continuity past its own Fencing means a deployment +driving another transaction's post-fencing abort. That abort revokes a lease +and restores a projection, and it is the path that wedged Ghostlight (B7). +Waiting is bounded by the owner's own phase deadlines (`PhaseDeadline`, +`:938-960`) and adds no new abort path. Recommendation: wait, at medium +confidence. Preempt is the better answer once post-fencing aborts are known +not to wedge, which is follow-up `ghostlight-abort-lease-before-stop`. + +Residuals, recorded and not fixed. A wedged post-fencing abort of a +continuity (B7, follow-up `ghostlight-abort-lease-before-stop`) holds a +waiting deployment as long as it holds the target. A continuity superseded +after its mint stays charged; that lengthens one wait for a generation that +is being replaced. Stop-then-start has no control-plane fixture (B17), so in +cut 2 its edge is pinned structurally (one seam, one call) and by a unit test +of the phase mapping, not by an engine run. raven-muninn uses it, so the +engine test is owed before cut 5. + +## 5. Cut sequence + +1. **c1-never-stop**: spec in the mind (specs-with-no-report recipe). Depends + on nothing. +2. **c2-phase-split** (`handover-phase-split`): spec + `cut-c2-phase-split.r1` in the mind, branch `idunn/reincarnation-c2` from + `9a0a6a2`. Continuity restarts the incumbent while a pre-fencing + deployment builds. The ownership predicates split (B13, B14). The handover + is one CAS at `takes_target_at()`. A pre-handover deployment rebases onto + whatever is admitted, and a superseded continuity aborts itself. Freeze + refuses only a second live deployment for a target, which dissolves + follow-up `profile-deploy-joint-gap`. It folds in the test-only follow-up + `c1-sustained-reset-test-reach`. Open question: + `fencing-over-owning-continuity`. Depends on cut 1 (merged at `1da150a`). +3. **Dependency failures do not grow the backoff** + (`budget-never-stop`): an attempt that failed because of a declared + dependency (Odin's stable route, the target's own stable route + listener), as Idunn itself observed it, leaves the streak unchanged. + Waits on follow-up `odin-route-record-stale`, because the record it + would read is not trustworthy, and on cut 1. +4. **A dead generation is not supervised as alive** (target invariant 5): + observe the workload before route supervision and topology refresh, so + that a dead admitted generation gets no topology authentication and no + lease-exactness challenge. Its non-binding correlation is reported once, + not every tick. Waits on cut 1, because it rewrites the same function. +5. **Closing proof on Yggdrasil**: an operator-approved install of the merged + cuts. streampixels-service survives the install, and `idunn status` is + checked before and after. Waits on 1-4 and on follow-up + `stop-first-handover-fixture` (raven-muninn is stop-then-start, B17). + +Ops, out of scope: nginx boot order (`gamecult-ops`), and retention of the +roughly 130 dead transient units. + +## Installed + +Cut 1 (`1da150a`, merge of `idunn/reincarnation` at `ffb4293`) was installed on +Yggdrasil on 2026-10-19 at 08:34:45 UTC with the operator's OK. The binary is +`/usr/local/bin/idunn` sha256 `f0e58308…`, built in `/srv/build/idunn-1da150a` +(`cargo test --lib`: 414 passed). The pre-install store backup is +`/root/idunn-store-backup-*-pre-reincarnation-c1`. Restoring it means restoring +its stores and `idunn.running` together, because the cb60965 binary cannot read +`idunn.target_supervision.v2`. At first open Idunn logged +"migrated 5 control record(s) … idunn.target_supervision.v2". streampixels-service +kept running through the restart (unit 73ca40ba, PID 968752, NRestarts 0). The +catalog returned 200, and 4443/udp was listening. + +## Installed: cut 2 + +Cuts c2-phase-split, c2b-owner-deadlines, c2c-post-fence-forward and +c2d-continuity-yields (branch tip `8760a4f`) were installed on Yggdrasil on +2026-10-20 at 00:34:06 UTC, with the operator's OK. + +- **The hotfix before it.** `ec45906` (`bounded_detail`) was installed on its + own on 2026-10-19 at 17:32 UTC. It unwedged tx-f978ab86. +- **Build.** `/srv/build/idunn-8760a4f`; `cargo test --lib` passed 471. +- **Backup.** `/root/idunn-store-backup-*-pre-reincarnation-c2` holds the + stores, `incidents.cc` and the binaries. A rollback must restore them + together: older binaries refuse two live transactions on one target, and + they refuse the `operator-required` incident condition. +- **First boot.** The boot lift returned exactly tx-039b91e9 (ghostlight) and + tx-4eaed669 (odin) to forward retry, as the Soul install gate predicted. + Odin's continuity, wedged in Routing since the 2026-10-18 reboot, committed + 4 s later. Generation tx-4eaed669 is admitted, its route is healthy, and the + same PID 2571 still runs. Ghostlight's continuity is retrying forward in + AwaitingReady. Its daemon's readiness is a ghostlight-side fault + (`Permission denied` reading its session-wrapping key). +- **Live processes.** Every live MainPID was the same before and after the + install. The streampixels catalog returned 200, and 4443/udp was listening. diff --git a/docs/reincarnation-target.md b/docs/reincarnation-target.md new file mode 100644 index 0000000..72e50f7 --- /dev/null +++ b/docs/reincarnation-target.md @@ -0,0 +1,95 @@ +# Reincarnation after reboot: target + +Status: target, Self, 2026-10-18 (campaign `idunn-reincarnation` in Eureka's +mind). It states the ends. The body facts, model page and cut sequence will +live in `reincarnation-map.md`. + +## Why + +Yggdrasil rebooted at 11:46 UTC on 2026-10-18. Admitted bodies run as +transient systemd units under `/run/systemd/transient`, and the reboot wiped +them all. nginx failed at boot, because it bound `10.77.0.1:17871` before +`wg-gamecult` carried that address. Every stable route, Idunn's own +`127.0.0.1:8831-8833` included, stayed down until a manual +`systemctl restart nginx` at about 19:46. + +What Idunn did: + +- From 11:50 onward continuity spent streampixels-service's whole restart + budget (6 per hour) and logged "The target is free for a deployment to + replace it: systemd unit … is absent". The log repeated in hourly bursts as + the window slid. +- When nginx returned, ghostlight and streampixels-web were re-incarnated, and + streampixels-service was not. +- An operator-approved `idunn up streampixels-service` (tx-e69f5402) sat in + Sealing for about 15 minutes. It then failed with "admitted incumbent failed + before candidate fencing; deployment yielded to continuity". Continuity had + freed the target for a deployment. The deployment handed it back to + continuity. Neither actuated. +- At 20:46:43 continuity re-incarnated the service under generation + tx-0804185a, once the window had slid to 4/6. So the outage was not ended by + the deployment the operator approved. The restart window bounded it, a full + hour after its cause had cleared. +- Throughout, several times a second: "preserved admitted … after rejecting + topology observation: topology correlation does not bind the current + activation", and "rejected admitted streampixels-service route continuity: + admitted process write lease is no longer exact". + +## Invariants + +1. **A reboot is not a fault of the body.** If an admitted body's unit is + absent because the host lost its transient units, continuity + re-materializes it from the installed body. It needs no deployment, no + operator, and no part of the restart budget for that absence. +2. **The restart budget measures the body.** A failure caused by a shared host + dependency, such as nginx, the stable route's listener or Odin, never + charges a target's restart budget. A dependency outage can never exhaust + the budget of every target behind it. +3. **Exactly one authority owns each slot at every instant, and nothing + waits on an owner that has stopped.** Continuity never stops (operator, + 2026-10-18: capped backoff, and the 6 per hour count only pages). It never + aborts a deployment. Before Fencing a deployment owns only its candidate, + and continuity owns the incumbent slot. At Fencing the deployment takes the + target and fences whatever is admitted then; an absent incumbent fences as + nothing. Every continuity attempt leaves a free gap in which a queued + deployment can be admitted. There is no reachable state in which + continuity waits for a deployment while that deployment waits for + continuity. A negative test proves it. +4. **Continuity is daemon survival; deployment is change** (`F:\Projects\CLAUDE.md`). + Restarting the admitted installed body never needs a deployment and is + never gated by a deployment brake. A deployment brake gates only artifact, + revision, configuration, schema, unit or authority changes. +5. **Rejections of stale observations are not a log stream.** An observation + that cannot bind the current activation is either resolved by the owner + that can refresh it, or reported once as an incident. It is never + re-rejected several times a second. +6. **Existing invariants hold.** `route-continuity-target.md` 1 to 7 (in + particular no actuation from observation, bounded backoff, every + post-fencing phase ends) and `authority-map.md` (Idunn's own recovery + depends on no managed target). + +## Scope + +In: + +- Idunn's continuity of admitted bodies across a reboot: absent units, the + restart budget and what charges it. +- The handover between continuity and deployment, in both directions. +- Classifying the topology-correlation and write-lease rejections: the same + fault, or a separate one with its own cut. +- Re-admitting streampixels-service on Yggdrasil. It is running again under + tx-0804185a, and the closing proof checks that it survives the fix's + install. + +Out, and not consumers of this change: + +- **nginx's boot order on Yggdrasil.** It is host configuration, owned by + `gamecult-ops`. It gets its own operator-approved change and record, not an + Idunn cut. +- **Ghostlight's degraded route** (53 consecutive failures, incident open) is + in scope only if the map shows the same fault. Otherwise it is a follow-up. +- **Heimdall's onboarding failures and its environment.** +- **Retention of the ~130 dead transient units** on Yggdrasil. It is an ops + follow-up already recorded under route continuity. +- **Luna's 2026-09-27 weekend patches** are not foundation to build on. Where + this map meets them, it cuts them. diff --git a/docs/research/restart-budget-prior-art.md b/docs/research/restart-budget-prior-art.md new file mode 100644 index 0000000..a320c42 --- /dev/null +++ b/docs/research/restart-budget-prior-art.md @@ -0,0 +1,124 @@ +# Restart-budget and handover prior art (Eyes facts pass) + +Serves two operator questions in Eureka campaign `idunn-reincarnation` (session idunn-reincarnation-2026-10-18): + +- `idunn-reincarnation:question:budget-charge` (what charges a target's continuity restart budget) +- `idunn-reincarnation:question:handover-pre-fencing` (who acts when the incumbent dies while a deployment is before Fencing) + +Facts and evidence pointers only. No recommendation. Gathered 2026-10-18 from official docs and upstream source (main branches at fetch time). Quotes are under 15 words. "Not found" means the primary source read does not state it, not that it is false. + +## 1. Kubernetes + +### 1.1 CrashLoopBackOff schedule, cap, reset (kubelet) + +- Docs, pod lifecycle, "Container restarts": backoff is "exponential backoff delay (10s, 20s, 40s, ...)", "capped at 300 seconds (5 minutes)". Reset: after the container has run "for 10 minutes without any problems". https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ +- Source pkg/kubelet/kubelet.go (https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/kubelet.go): `initialCrashLoopBackOff = time.Second * 10`; max is `MaxCrashLoopBackOff = v1beta1.MaxContainerBackOff` (value lives in the kubelet config API package, not re-read; docs say 300 s). Under alpha gate `ReduceDefaultCrashLoopBackOffDecay`: initial 1 s, max 60 s. Gate `KubeletCrashLoopBackOffMax` lets the config set the max (`CrashLoopBackOff.MaxContainerRestartPeriod`). +- Reset rule in code: `klet.crashLoopBackOff.HasExpiredFunc` returns `eventTime.Sub(lastUpdate) > 600*time.Second` (kubelet.go ~line 1102). I.e. the entry resets when more than 600 s have passed since the entry was last updated (last restart event). The client-go default is 2*maxDuration; the kubelet overrides it. +- Growth: client-go `Backoff.Next` doubles (`delay := entry.backoff * 2`) and caps with `min(delay, p.maxDuration)` (https://github.com/kubernetes/client-go/blob/master/util/flowcontrol/backoff.go). +- Window anchor: the exited container's `FinishedAt`; keyed per pod+container; event text "Back-off restarting failed container" (pkg/kubelet/kuberuntime/kuberuntime_manager.go, `doBackOff`). +- The backoff table is an in-memory object created in `NewMainKubelet` (`flowcontrol.NewBackOff(base, boMax)`); no persistence found in the code read. Not found: any doc statement that a node reboot does or does not reset or count toward it. Docs only say restart policy "only applies to restarts that make replacement containers inside the same Pod and running on the same node." Docs on kubelet restarts: pods "continue to run even during the restart" (kubelet restart, not node reboot). + +### 1.2 Probes + +Docs (pod lifecycle "Container probes"; https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/): + +- Liveness: past the failure threshold "the kubelet restarts that container" (per pod restartPolicy). +- Readiness: the EndpointSlice controller "removes the Pod's IP address from the EndpointSlices"; the container is not restarted. +- Startup: until it succeeds "Kubernetes does not execute liveness or readiness probes"; on failure "the kubelet kills the container", subject to restart policy. +- Backoff keys on a container having exited (`doBackOff` reads the last exited container), so crash, liveness kill and startup-probe kill all reach it; readiness failure leaves the container running and does not. The docs do not describe backoff per probe type. + +### 1.3 Dependency down + +Docs, same page, "Dependencies and init containers": init containers "run only once (if successful), during Pod startup"; failed init containers are restarted by the kubelet when pod restartPolicy is OnFailure or Always. Readiness gates: pod is ready only when all `readinessGates` conditions are True. No general documented rule for a main container whose external dependency is down: an app that exits is restarted under backoff; an app that stays up and fails readiness is only removed from endpoints. + +### 1.4 RollingUpdate and the old ReplicaSet + +- Deployment docs (https://kubernetes.io/docs/concepts/workloads/controllers/deployment/): "Once new Pods are ready, old ReplicaSet can be scaled down further"; at the end "all old ReplicaSets is scaled to 0". maxUnavailable and maxSurge default 25%. progressDeadlineSeconds default 600; when exceeded, "Kubernetes takes no action on a stalled Deployment" beyond the Progressing=False condition. +- ReplicaSet docs: it fulfills its purpose by "creating and deleting Pods as needed to reach the desired number." https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ +- Deployment controller source, pkg/controller/deployment/rolling.go (https://github.com/kubernetes/kubernetes/blob/master/pkg/controller/deployment/rolling.go): `reconcileOldReplicaSets` computes `minAvailable = replicas - maxUnavailable`, runs `cleanupUnhealthyReplicas` first (comment: "Clean up unhealthy replicas first, otherwise unhealthy replicas will block deployment"), then `scaleDownOldReplicaSetsForRollingUpdate`. The Deployment controller decides old-RS replica counts during rollout, scaling unavailable ones down first. +- Layering: restarts of a dying container are done by the kubelet (restartPolicy with backoff), not the ReplicaSet. The ReplicaSet recreates a pod only when the pod object is gone or terminal. Not found: a statement in those words that the old ReplicaSet keeps recreating its dying pods until scaled down. + +## 2. Nomad + +### 2.1 restart block (https://developer.hashicorp.com/nomad/docs/job-specification/restart) + +- `attempts` restarts allowed within `interval`; interval "starts when the first task starts". Defaults: service/system attempts 2, interval 30m, delay 15s, mode fail; batch attempts 3, interval 24h, delay 15s, mode fail. Delay has up to 25% jitter. +- `mode = "fail"` (default): after attempts are used the allocation is marked failed and goes to `reschedule`. `mode = "delay"`: "The task waits another full interval before restarting"; the allocation is never failed by restarts. +- Source client/allocrunner/taskrunner/restarts/restarts.go (https://github.com/hashicorp/nomad/blob/main/client/allocrunner/taskrunner/restarts/restarts.go): in-memory `RestartTracker` with `count` and `startTime` set at construction (`time.Now()`); a new interval resets count; `count > Attempts` returns TaskNotRestarting in fail mode, or TaskRestarting with delay to interval end in delay mode. Constructed in `NewTaskRunner` (task_runner.go ~line 468) from the alloc's job type and the task group's policy. `SetKilled` returns TaskKilled without counting. A restart triggered with failure=false (operator/API) returns TaskRestarting without consulting the policy. Unrecoverable start errors return TaskNotRestarting. +- Client restart: restarts.go keeps no persisted counter. `TaskRunner.Restore` (task_runner.go ~line 1244) restores local state and the driver handle (`RecoverTask`); if the handle cannot be restored for a non-system job the task "will not run until server is contacted" (`waitOnServers`). The tracker is rebuilt at task-runner construction. Persisted `TaskState` carries restart counts separately; `GetState` does not read them. + +### 2.2 reschedule block (https://developer.hashicorp.com/nomad/docs/job-specification/reschedule) + +- Applies when an allocation fails to set up or its tasks fail more than `restart` allows. Drains trigger migrations and lost nodes trigger replacements, so reschedule is ignored in both cases. Not applied to system/sysbatch. +- Service defaults: `delay_function = exponential`, delay starting 30s, `max_delay` 1h, `unlimited = true`. Batch defaults: 1 attempt per 24h, constant 5s, unlimited false. `delay_function`: constant, exponential (doubling), fibonacci. `max_delay` ignored for constant. Minimum delay 1 s; docs warn very low delay with `unlimited = true` can cause runaway rescheduling. +- With `unlimited = true`: "The `progress_deadline` in the `update` block still applies". + +### 2.3 Deployments and restarts of old allocations + +- Update block (https://developer.hashicorp.com/nomad/docs/job-specification/update): `max_parallel` 1, `min_healthy_time` 10s, `healthy_deadline` 5m, `progress_deadline` 10m ("the deployment is marked as failed"), `auto_revert` false, `canary` 0. The page says nothing on restarts of old allocations. Not found: docs on deployment/restart interaction. +- Restart is client-local ("Restarts happen on the client that is running the task"); deployment is server-side. Reconciler source (scheduler/reconciler/reconcile_cluster.go, `placeAllocs`): when the deployment is not place-ready (paused/failed/canarying), rescheduling of failed allocs still proceeds, with comment about avoiding "odd user experiences", except where the failed alloc is part of the failed deployment (`partOfFailedDeployment` skips it). + +## 3. systemd + +### 3.1 Start limit (https://www.freedesktop.org/software/systemd/man/latest/systemd.unit.html) + +- A unit started more than `StartLimitBurst=` times within `StartLimitIntervalSec=` may not start again until the interval passes. Applies to all starts, including manual, not only `Restart=`; it is checked after condition checks, so failing conditions do not count. Units with `Restart=` that hit it are no longer restarted automatically; `systemctl reset-failed` flushes the counter. `StartLimitAction=` default none. Counters are cleared when the unit is unloaded. +- Source: `unit_test_start_limit` in src/core/unit.c (`ratelimit_below(&u->start_ratelimit)`; on hit logs "Start request repeated too quickly", returns -ECANCELED); `unit_reset_failed` calls `ratelimit_reset`. + +### 3.2 Job result "dependency" and the start limit + +- src/core/unit.c `unit_start` order: conditions (-ECOMM), asserts (-EPROTO), `unit_verify_deps` (-ENOLINK, "The necessary dependencies are not fulfilled"), then `UNIT_VTABLE(u)->test_startable`. For services, `service_test_startable` (src/core/service.c ~6381) calls `unit_test_start_limit`. A start failing on unfulfilled dependencies returns before the start-limit counter is touched. +- src/core/job.c: `-ENOLINK` maps to `JOB_DEPENDENCY` (`job_run_and_invalidate`); `job_fail_dependencies` also finishes dependent START/VERIFY_ACTIVE jobs with `JOB_DEPENDENCY` ("Dependency failed for %s."). `n_failed_jobs` increments only for JOB_FAILED, JOB_INVALID, JOB_FROZEN, JOB_CONCURRENCY. JOB_DEPENDENCY triggers `OnFailure=` handling. +- Requires=: if the required unit fails to activate and the dependent has After= on it, the dependent is not started; without After= it may still start. Wants=: failure "has no impact on the validity of the transaction as a whole." Dependent is stopped when the required unit is explicitly stopped or restarted; not stopped if the required unit stops on its own. + +### 3.3 Restart= and dependency failure + +- systemd.service(5) (https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html): the restart table is by exit cause of the service's own process (exit code, signal, timeout, watchdog, OOM); no row for dependency failure. Page: "service restart is subject to unit start rate limiting." Not found: a doc statement on whether Restart= acts on a dependency failure. From the code above, a dependency-failed start job finishes with JOB_DEPENDENCY and never passes through the service's own failure state (inference, not an explicit doc statement). +- Backoff: `RestartSec=` default 100ms; `RestartSteps=` with `RestartMaxDelaySec=` give exponential growth to a cap (doc example 10s, 20s, 40s, 80s, 160s, 160s...). `RestartMaxDelaySec` default infinity; both must be set. `RestartRandomizedDelaySec=` (v262) adds jitter. + +### 3.4 Transient units across reboot + +- systemd.unit(5) path table (man/systemd.unit.xml ~lines 403, 465): "Dynamic configuration for transient units" lives in `/run/systemd/transient` (system) and `$XDG_RUNTIME_DIR/systemd/transient` (user). That `/run` is volatile is not stated in this file. +- systemd-run docs: failed transient units are "kept in memory until the user explicitly resets their failure state"; `--collect` is `CollectMode=inactive-or-failed`. systemd-run and https://systemd.io/TRANSIENT-SETTINGS/ do not mention reboot. Not found: an explicit sentence on survival across reboot. + +## 4. Other supervisors + +### 4.1 Erlang/OTP supervisor (https://www.erlang.org/doc/apps/stdlib/supervisor.html, https://www.erlang.org/doc/system/sup_princ.html) + +- `intensity` (MaxR) and `period` (MaxT), defaults 1 and 5. If more than MaxR restarts occur within the last MaxT seconds, the supervisor "terminates all the child processes and then itself" (exit reason `shutdown`). The parent supervisor then restarts it or itself terminates; failure escalates up the tree. Stated purpose: prevent a process from repeatedly crashing for the same reason and being restarted indefinitely. +- Restart types: `permanent` (always), `transient` (only abnormal exit), `temporary` (never). Every restart the strategy performs counts toward intensity; the docs describe no per-cause charging and no backoff delay. Multi-level: total restarts before the top gives up is roughly the product of intensities above the failing child. Strategy `rest_for_one` restarts the failed child and those started after it. + +### 4.2 runit and s6 + +- runit `runsv` (https://smarden.org/runit/runsv.8.html): if ./run exits immediately runsv "waits a second" before restarting; no rate limit or retry cap described; keeps restarting unless told otherwise (`o`, `d`, `x`). +- s6-supervise (https://skarnet.org/software/s6/s6-supervise.html): "There is a minimum 1-second delay between two ./run spawns." A ./finish script exiting 125 makes supervise stop restarting; otherwise it restarts indefinitely. The give-up decision is delegated to a user script, not a built-in counter. + +### 4.3 Borg (Verma et al., EuroSys 2015; https://research.google/pubs/large-scale-cluster-management-at-google-with-borg/, PDF text read) + +- Borglet "restarts them if they fail"; Borg "monitors the health-check URL and restarts tasks that do not respond promptly" (sections 2.6, 3.3). +- Section 4 (Availability): Borg "automatically reschedules evicted tasks"; "limits the allowed rate of task disruptions and the number of tasks from a job that can be simultaneously down"; rate-limits finding new places for tasks from unreachable machines because it cannot tell machine failure from network partition; "avoids repeating task::machine pairings that cause task or machine crashes". +- Section 2.3 (updates): "Updates are generally done in a rolling fashion", with a limit on task disruptions an update causes; changes that would exceed it "are skipped". +- Not found in the Borg paper: a crash-loop backoff schedule, a restart budget, or any distinction between charging task faults and dependency faults. Omega paper not fetched. + +## 5. Facts grouped by the brief's sub-topics + +- (a) Charging only body faults: Kubernetes backoff keys on container exit regardless of cause (readiness failure does not exit and does not advance it). systemd runs dependency verification (-ENOLINK, JOB_DEPENDENCY) before the start-limit test, and JOB_DEPENDENCY is not in `n_failed_jobs`. Nomad's tracker counts any recoverable task failure or start error, with no dependency classification. OTP counts every restart. Borg avoids repeating task::machine pairings that cause crashes (placement, not budget). +- (b) Give up vs capped forever: OTP gives up and escalates. Kubernetes caps at 300 s with no give-up. Nomad restart fail-mode gives up locally, then reschedule continues (service default unlimited, exponential to 1 h); delay-mode never fails. systemd start limit stops auto-restart until the interval passes or reset-failed; `RestartMaxDelaySec` default infinity. runit restarts forever at 1 s; s6 restarts forever unless finish exits 125. +- (c) Rollout vs restart supervisor: Kubernetes splits by layer (kubelet restarts containers; ReplicaSet maintains pod count; Deployment controller scales old RS during rollout). Nomad splits by node vs server (client restart tracker vs server reconciler; reconciler skips rescheduling allocs of a failed deployment). Borg bounds update-induced disruptions with a rate limit. + +## 6. Memory claims vs sources + +budget-charge: + +1. "Kubernetes separates readiness from liveness; only liveness feeds CrashLoopBackOff": CORRECTED. Readiness never restarts and never advances backoff (confirmed). Backoff keys on any container exit (crash, liveness kill, startup-probe kill), so it is not fed only by liveness. +2. "CrashLoopBackOff caps at 5 min": CONFIRMED (docs 300 s; initial 10 s, doubling). Reset: docs say 10 min healthy; code says 600 s since the entry's last update. 60 s cap only under alpha gate. +3. "systemd does not run a unit whose required dependency failed (job result 'dependency')": CONFIRMED with qualifier: Requires= plus After= blocks the start; Requires= without After= may still start; Wants= does not block. +4. "Nomad reschedules with capped exponential delay": CONFIRMED for service jobs (exponential from 30 s, max_delay 1 h, unlimited by default); batch default is constant 5 s, 1 attempt per 24 h. +5. Dependency failure vs start limit (not in the question's claims, material): systemd checks dependencies before counting a start (confirmed by code order). Whether Restart= acts on dependency failure: UNSUPPORTED by docs. +6. Whether restarts after a node reboot count toward CrashLoopBackOff: UNSUPPORTED by docs; code read shows the kubelet backoff map in memory. Nomad's restart tracker is likewise in memory and rebuilt on client restart. + +handover-pre-fencing: + +1. "In a Kubernetes rolling update the old ReplicaSet keeps recreating its dying pods until the rollout scales it down": PARTLY CONFIRMED, PARTLY CORRECTED. ReplicaSets do create pods to reach their desired count, and the Deployment controller scales old RS down, unhealthy first. Container restarts are the kubelet's, not the ReplicaSet's; the RS recreates only pods that are gone. +2. "Nomad restarts old allocations by their own restart policy, independent of the deployment": PARTLY SUPPORTED. Restart is client-local using the alloc's task-group policy (code), so independent of the server-side deployment; no doc statement on the interaction. Server-side, failed allocs are rescheduled even when the deployment is not place-ready, except allocs of a failed deployment. +3. Any source saying an aborted or stalled rollout automatically hands the slot back to the old generation: UNSUPPORTED (Kubernetes takes no action on a stalled Deployment; Nomad `auto_revert` defaults false). diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index 512ac09..a278165 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -1,6 +1,6 @@ # Route continuity and admission: cut map -Status: cut map, Imagination pass 0b (Opus), 2026-09-29. Nothing has landed. +Status: cut map, Imagination pass 0b (Opus), 2026-09-29. Nothing has landed. 2026-09-30: the Odin-lifecycle authority audit is folded in (sections 3, 4 and 5); see the Cut status block. Ends are owned by route-continuity-target.md. **2026-09-29, Self (Eureka session "Codebase audit"; the campaign was handed over from the StreamPixels deployment session with the operator's @@ -11,6 +11,319 @@ on CultLib `codex/fix-node24-ajv-esm`. Behaviour cuts re-take their `file:line` anchors in their own briefs. The operator has asked for no live mitigation on Yggdrasil yet. +**B3 status, 2026-09-30 (Self).** Hands pushed B3 to `idunn/route-b3` at `5ca7760`; it is based on +B1's merge `a61540b`, not stacked on B2. The Opus Soul pass found: +- Full suite green: 252 passed, 2 ignored. +- `cargo-mutants --in-diff`: 31 caught, 4 missed. +- Soul's probe P1 drove a stateful route-proof target through the Engine, so the stateful path works; it + was only untested. + +Verdict: safe to merge for the StreamPixels web ship. The fix batch, in Hands, covers these findings: +- the evidence tag is a second class authority, and a pre-B3 transaction wedges Routing; +- the driver's rejections of a bad answer become gate waits instead of errors; +- the shortfall is prose; +- `OdinSelf` is decorative, and "is Odin" is still decided by name in five places; +- Warming is never driven past one step; +- seven unpinned checks. + +Before `idunn up`, one read-only decode of a *copy* of the live `control.cc` checks for non-terminal +transactions of routed targets with no Odin declaration (operator approved, 2026-09-30). + +**B3 fix batch, 2026-09-30.** Pushed `6ea824b` and `e11b457`; 275 tests pass. +- `ReadinessClass::of` is the only class authority. The evidence tag is demoted to `Voucher`. +- `ChallengeFailure::{Silent, Refused}` separates silence from a refused answer. +- `PresenceDisagrees` makes the capacity shortfall a typed value. +- The recipes are on `route/b3-declare`: Heimdall `d5acc94`, Muninn `2260853`. +- Still open, for a fresh Hands: three survivors (boot re-proof by tag, config error as silence, first-Odin + check by name) and `cargo mutants --in-diff`. + +**Live store, read-only decode of a copy, 2026-09-30 14:56 (operator-approved).** +- `control.cc` holds 5 envelopes, all `admitted_generation.v2` (pre-F0), and **no transactions**. +- `history.cc` holds 957 transaction v3 envelopes and 641 command v2 envelopes. +- Admitted generations, all on Odin receipts: + - ghostlight: routed, declares Odin, stateful. + - odin: routed rudp, provides the rendezvous. + - raven-muninn: unrouted, no declaration, stateless. B3 class `Undeclared`, so **held**. + - streampixels-service: routed, declares Odin, stateful. + - streampixels-web: routed, declares Odin, needs `streampixels.service.api`. +- **Heimdall is not admitted.** It drops out of the ship window. Its declaring recipe still merges. Its + first deploy will hit Soul pass 2's F2: a routed, Odin-correlated target must pass the stable-route + proof, and Heimdall answers no challenge. Record that for its own deploy. + +**B3 pass 2 (Soul, 86ee21d):** safe for the web ship. One same-window blocker: +- **F1:** supervision mints a held continuity over a dead held generation, and that continuity owns the + target, so the declaring redeploy can never freeze. The fix is in Hands. +- F5, bad answers read as Silent over HTTP keep-alive and TCP, is in the same batch. + +**B3 fix batch (`86ee21d..e8b4487`), Soul pass 3, 2026-09-30: merge with follow-ups.** Nothing here hurts a +live host on install. 294 passed, 2 ignored. `cargo mutants --in-diff`: 53 caught, 0 missed. +- **F1 holds.** There is one continuity mint site, and the hold check comes before it. A declaring redeploy + replaces a dead held incumbent end to end. +- **F5 holds.** Zero bytes is Silent. Any byte then a stall or stop is Refused. Non-2xx responses and + non-chunked Transfer-Encoding are Refused. +- **Follow-ups, in Hands on the same branch:** + - the P1b test never reaches the yield branch (mutant M7 survives); + - the chunked trailer loop is unbounded; + - `+N` lengths and whitespace before the colon are accepted; + - exact 1 MiB and one-byte-then-stop are unpinned; + - the `Silent` doc is stale; + - the dead-hold report goes to stderr only and never clears. + +**Ruling, 2026-09-30 (operator): cancel accepts a held pre-fence Deploy.** Idunn still never aborts a held +record on its own. An operator can discard a held Deploy that has not reached the fence with `idunn cancel`. +Nothing is fenced, so the incumbent is untouched. + +**B3 follow-ups landed on `idunn/route-b3` at `5c49646`, 2026-09-30.** +- Cancel accepts a held pre-fence Deploy. It shares `pre_fencing_abort_intent` with the scheduler. +- The P1b test is now in direct-supervision form, with a positive control. +- The reader is strict: trailers share the 32 KiB header budget; digits only; header names without whitespace. +- The `Silent` doc is corrected. +- Tests: 301 pass. `a_failing_step_is_retried_after_a_backoff_not_every_tick` is flaky under a full parallel + run, on the base too. +- `cargo mutants --in-diff`: 24 caught, 0 missed. +- A narrow Soul pass gates the install. + +**Ship log, 2026-09-29 (host clock).** +- B3 merged at `d32395a` and installed on Yggdrasil at 18:36 UTC, with sha `854aa351`. It migrated 5 control + records. The store backup is `/root/idunn-store-backup-20260929T183604Z-pre-route-b3`. +- Nginx reloads fell from 82 per 10 minutes to 0. Odin continuity timeouts fell from 92 in 6 hours to 0. +- **Raven's actuator (built 2026-09-11) could not decode the new plan types.** A rebuild then found that the + Windows `idunn-host` has not compiled since S1: `FROZEN_SOURCE_SYMLINK_TARGET_LIMIT` was defined for unix + only. That is fixed at `d3db582`. + - The new actuator (sha `f36bf1fc`) is swapped in on Raven. The old one is kept as + `idunn-host.exe.prev-20260929T190006Z`. + - Follow-up: nothing builds the Windows actuator in verification. +- raven-muninn was admitted at 19:02 by Odin correlation, replacing the held generation. +- **The streampixels-service deploy failed before fencing.** The S1 `bulk_fetch_objects` asks for bare blob + ids with `git fetch --stdin`. In a `blob:none` clone, the connectivity check then fails with "bad revision + ". The fix, with a test, is in Hands on `fix/bulk-blob-fetch`: a promisor-style fetch with + `--filter=blob:none` and `fetch.negotiationAlgorithm=noop`. + +**Ship completed, 2026-09-29 (host clock).** +- **Bulk-fetch fix.** Merged at `583b2a7` and installed at 19:17. The Idunn sha is `01bde676`. +- **The first streampixels-service redeploy failed after fencing.** The incumbent was restored, and every restart + was refused by Odin with "runtime presence publisher sequence was reordered". Cause: Odin compared publisher + sequences across incarnations, and the TS publisher counts from 1 per process. + - The operator ruled: fix Odin's rule. + - Odin `db041ec` compares sequences only while the stored presence still authenticates under the current + activation. Soul said deploy, with follow-ups; Odin was deployed at 19:4x. +- **Admissions.** + - streampixels-service was admitted at 19:50 on s3-c2. + - The web binding dropped `STREAMPIXELS_ODIN_CULTMESH_RUDP`, and gamecult-ops `37b5ed9` does the same. + - streampixels-web was admitted with Odin-free readiness. +- **Public cutover at 19:54.** The vhost upstreams moved to `:8833` (web) and `:8832` (service). The backup is + `/root/streampixels.gamecult.org.conf.bak-20260929-195415-pre-idunn-upstreams`. + - `https://streampixels.gamecult.org/api/catalog` returns 200 (120,893 B), after being 502 since migration 013. + - The overlay SSE stream answers `text/event-stream`, and the ops checker passes. +- **Legacy units.** `streampixels-web.service` is disabled and stopped. `streampixels-service.service` is + disabled, and was already stopped. +- **Follow-ups.** + - The C2 acceptance hour: sample the nginx reload counter at T0 and T0+60, with no rejected challenges. + - Nothing builds the Windows `idunn-host` in verification. + - Odin presence: pin the stored-at re-authentication time (Soul's M3 survived), and consider checking activation + identity directly rather than any failure to authenticate. + - The dead-hold report field (ruled). + - B2 rework on B3. + - Odin's CultLib pin bump, after the stray-packet merge. + +**C2 acceptance hour passed, 2026-09-29 19:55-20:55 UTC (21:55-22:55 CEST).** +- 1 nginx reload, which was Odin's own route promotion during its `5c37860` deploy. +- 0 rejected StreamPixels challenges, 0 Odin continuity timeouts, and Idunn `NRestarts=0`. +- One running unit each for service, web and Odin. Public `/` and `/api/catalog` return 200. +- Also deployed in the window: Odin `5c37860`, carrying CultLib `3bf1c0c` (RUDP stray-packet hardening in every + runtime), Sleipnir removed, and presence ordering decided by activation identity. +- Still open for Odin: it exits when its 64-slot RUDP session table fills, because its own heartbeat publish + fails. Publishers that never Disconnect hold slots for 30 s. The fix is in Hands. +- The second run, with Odin broken on purpose, remains the operator's to schedule. + +**Ruling, 2026-09-30 (operator): the dead-hold report is stored on the generation.** +- Add an optional `last_error` to `AdmittedGeneration` (new key 21), and make `status` render admitted + generations. +- It is cleared when the generation is replaced or its hold is released. +- The `generation:` readiness report gets the same home. +- It is queued after the ship, with the B2 rework. + +**Odin diagnosis (Eyes, read-only, 2026-09-30).** +- The 109 Odin route timeouts are self-inflicted. Live Idunn (`8ae00a1`, pre-S1) reloads nginx inside + `restore_admitted_membership` before every Odin challenge. The reload moves the reuseport UDP flow, so + the challenge misses. All 109 have a reload 3.05-3.12 s before them. +- 2,977 reloads in 6 h. 479 shutting-down nginx workers, because the stream has no `proxy_timeout`. +- The route timeouts do not block Ready correlation. +- Odin has crashed twice right after a reload. A stray RUDP packet with a foreign connection id escapes + `cultnet-rs` `rudp.rs:1217` (`require_connection`) and ends the process. Each crash costs about 75 s + without a lease. The CultLib fix is in Hands on `hands/rudp-stray-packets`. Odin takes it at its next pin + bump, after the ship. + +**Operator ruling: Idunn first.** Install the new Idunn (B3 carries S1), then watch for about 15 minutes that +reloads stop and Odin stays up. Only then redeploy raven-muninn and StreamPixels. + +**Ship sequence:** +1. B3 fix, then narrow Soul, then merge B3. +2. Merge StreamPixels `route/s3-c2` and the Muninn and Heimdall `route/b3-declare` recipes. +3. Build the Idunn release on its deploy path. +4. Back up `control.cc` and `history.cc`. F0's lift is irreversible. +5. Install Idunn. +6. `idunn up` raven-muninn with the declaring recipe. +7. `idunn up` streampixels (service, then web). + +**Rulings, 2026-09-30:** +- A stored record whose evidence disagrees with its declared class is **held and reported, never aborted** + by Idunn. B5's deadlines or the operator resolve it. +- **Heimdall and raven-muninn are redeployed with their declaring recipes in the same window as Idunn B3 + and the StreamPixels ship.** + +**Ruling, 2026-09-30: the recipe declares readiness; Idunn infers nothing.** This settles the fork +Hands raised against a literal Q1(b). An unrouted target whose recipe declares neither a route nor an +Odin dependency is refused at admission with a typed error. It does not fall back to Odin correlation +because it has no route. `raven-muninn` and Heimdall both publish to Odin, so their recipes declare it. +The rejected alternative let Odin's availability gate Muninn's crash recovery through an undeclared, +inferred dependency. + +**B2 status, 2026-09-30 (Self).** Hands pushed `idunn/route-b2` at `1786ddf`. The Opus Soul pass rejected +it for merge. +- **F1 (critical):** `ContinuityBackoff.deferral_reason` was added without a schema bump. B2 fails the + canonical re-encode of every three-field v3 generation that F0, B1 or B3 wrote, so Idunn cannot read its + own store. +- **F2:** the ceiling refuses a rollback, which leaves an unproven candidate on the stable route. +- **F3:** a failed challenge writes, and continuity's CAS then loses against a stale envelope. +- **F4:** neither window rolls; both are anchored at the first charge. +- **F5:** a clock stepped backwards stalls everything for the length of the step. +- **F6:** `record_proved_challenge` is unpinned. +- **F7:** the mount preflight is not gated. +- B2 also does not compile on top of B3, and after a one-line fix two tests fail, because the fixture target + is now route-proof. + +The rework is a fresh branch from `583b2a7`, not a rebase of `1786ddf`. Its map is under B2 in section 4. + +**Rulings, 2026-09-30.** +- **The ceiling never refuses a rollback.** Restoring the admitted or incumbent route is survival, not + deployment. It is always allowed and always counted. Only installs and other forward changes can be + refused. A rollback that keeps failing is spaced by backoff, never refused. +- **First routed deployments are metered now.** The actuation window lives on the target, not on the + admitted generation, so a target with no admitted generation is charged too. That schema change is folded + into the store-version bump that F1 needs: a v4 generation, with a typed lift from v3. + +**Rulings, 2026-09-30 (operator, on the B2 rework map).** +- **3d A (2026-09-30, morning handoff):** separate budgets. The 12/h route actuation ceiling counts only + Forward actuations (deploys). Survival actuations (continuity restarts, rollbacks, repairs) are still recorded + for visibility but do not charge the ceiling; each stays bounded by its own mechanism (the 6/h per-target + restart log, the challenge backoff). A crash-looping target can therefore always be rescued by a deploy. + Option D (reset the restart log when a new release is admitted) was not chosen: a new release still inherits + its predecessor's restart history. B2 is merged (`46d9c2f`) but not installed; this follow-up (B2-3d) lands + before the install. +- **B2-3d status (Self, 2026-09-30).** Built on `hands/b2-3d` (`e9c718e`, `057bdd5`). Soul re-ran `verify.sh` + clean (341) and killed every one of its own mutants on the Forward/Survival split. **But the premise the operator + ruled on is false:** "every Survival path has its own bound" does not hold. + - **F1:** a continuity transaction whose route proof keeps failing retries inside one transaction under the + 60 s resume cap, reloading nginx twice per retry. That is about 120 reloads/h per target, with no end until B5. + - **F2:** a failing post-fence withdrawal reloads about 60/h. `restore` reloads even when the fragment is + already correct. + - **F3:** nothing bounds reloads across the host. + - None of these is new in 3d: Survival was never refused before either. The live Idunn, which predates B2, has + no reload bound at all. What 3d adds is the false claim, at `control_plane.rs:76-81`, plus the admitted + Deploy doubling to 24/h. + - The fix belongs to B5 ("every post-fencing phase ends"), plus a restore/install that skips the reload when + the fragment is already correct. The operator was told on 2026-09-30. +- **B2 + 3d installed (2026-09-30).** The operator accepted recommendation A: install now, with B5 next. + - Installed at 15:22 CEST: `a96ad9d`, `idunn` sha256 `848c08e5...`; the replaced binary was `01bde676`. + - The migration moved 5 control records to `deployment_transaction.v4` / `admitted_generation.v4`. + - Pre-install backup, taken with Idunn stopped and byte-compared: `/root/idunn-store-backup-20260930T132204Z-pre-route-b2/` + (control.cc `5f879f55...`, history.cc `bafa76ab...`, the other stores, and the old binaries). + - Preflight: 690 transactions, all terminal. They were identical after the install. + - No managed target's PID changed. Raven reattached after 15 s. + - After 10 min: no odin route-continuity rejections (about 95 in the previous 3 h), 0 nginx reloads, and the + Odin post-deploy check `fails=0`. + - B5 and skip-unneeded-reloads are in Hands (`hands/b5`). +- **B5 Soul pass 1 (2026-09-30, on `a96ad9d..c06ff2b`): merge after the batch-2 fixes; no install before them.** + Soul reran verify: 352 passed. Batch 2, now in Hands, fixes: + - **A (high).** A continuity aborted from `Prepared` leaves a lease on disk that `driver.grant` wrote before + `Granted` was durable (`control_plane.rs:6916-6930`, `:9081`). The next fencing refuses, and the target is + wedged. + - **B (medium, a regression from `6ff7d5a`).** The reload skip hides a crash between the write and the reload on + the `restore`/`install` transaction paths, leaving nginx proxying to a dead port. The skip goes back to + supervision only. + - **E.** The resolver's command filter is unpinned. + - **F.** Odin-correlated adoption has no committed test; mutants M6 and M7 survive. + - **J.** A flaky backoff test (`:14200`). + + Recorded: + - **C** (the abort has no deadline) is closed by the Q3-ii cut. + - **D.** There is no host-wide reload bound. 170/h is about a third of the F6 storm, and 4830af8 lowers the real + worst case. + - **G.** A newer Ready can rewrite `lease_adoption`. It names the same lease, so it is harmless. + - **H.** AwaitingReady and Routing keep the F16 shape. It is unreachable at today's heartbeats. + - **I.** An `idunn expire` request written while Idunn is stopped lingers, and there is no verb to withdraw it, + which is a downgrade hazard. +- **B5 MERGED and INSTALLED (2026-09-30).** Merged to main at `fcb8ba1` (Soul passes 1-3). Built on Yggdrasil: + `/srv/build/idunn-fcb8ba1`, 360 passed. `idunn` sha256 `e96bcefa...`, `idunn-provision` `127902ef...`, unit unchanged. + Installed at 22:35 CEST. + - **Backup**, taken with Idunn stopped and byte-compared: + `/root/idunn-store-backup-20260930T203515Z-pre-route-b5/`. It holds `control.cc` `fab5b66b...`, `history.cc` + `a2627ea4...` and `idunn.running` `848c08e5...` (a96ad9d). + - **Preflight** found one live transaction: the raven-muninn continuity `990dfc25` in AwaitingReady for about 40 + min. Raven had rebooted, and its actuator timed out at 21:36 and 21:49 CEST. The operator ruled "Feel free to + install". + - **After install:** + - No managed target's PID changed. Raven reattached after 15 s, and nginx made 0 reloads. + - B5 aborted the stale continuity at its passed deadline, and supervision opened `continuity-26737b91`. + - A single log line, "could not resume transaction tx-bcfbb693...: transaction disappeared while checking + scheduler progress", appeared at 20:35:38 UTC. It looks like a same-tick ordering quirk between the + resolver and resume. It goes to the next Soul pass. + - **Rollback rule:** stop Idunn and check `control.cc` for any `idunn.expiry_request` envelope. If one exists, + restore the backup instead of swapping the binary. + - **Named tradeoff (Soul batch-3 F1):** a persistent `ufw delete` failure while aborting a first routed deploy now + leaves the aborted candidate routed until ufw recovers. Before B5 the cost was reload churn. Q3-ii's abort + deadline bounds it. + - Test pins F2-F4 are in Hands (`hands/b5-pins`). +- **B5 batch 2 (`c06ff2b..aef0da3`) and Soul pass 2 (2026-09-30): merge and install after batch 3.** + - A, B, E, F and J hold. Verify: 356 passed. cargo-mutants caught 14 of 14 viable mutants, and Soul's hand + mutants killed 16 of 20. + - Batch 3, in Hands: test pins for MB2 (`changes_route` swapped for "no incumbent"), ME2/ME3 (the resolver filter + by id length or first byte) and MJ2 (the computed backoff unpinned). `restore` withdraws the firewall rule + before reloading when there is no prior route, so a failing `ufw delete` no longer reloads nginx each retry. + - Hands corrected Self's brief: the continuity path's zero reloads came from `6ff7d5a`'s skip, not `4830af8`. The + narrow no-change case (`changes_route() == false`) keeps it without a write, so there is no crash window. + - Recorded, not in B5: + - **P1:** a crash between `atomic_replace` and the reload in supervision's `restore_admitted_membership` + (`drivers.rs:5543-5545`). Idunn has no SIGTERM handler, so a stop in that window leaves nginx on the drift + until something else reloads. This also exists in live `a96ad9d`. Fix with Q3-ii or a durable + "reload owed" marker. + - **The abort revokes before it stops the candidate** (`control_plane.rs:8758` before `:8783`). Q3-ii owns the + abort order. +- **Q-B2-1 (a):** the continuity restart log moves into `TargetSupervision`; the restart ceiling reads the + target's own log; the carry compensator is deleted. +- **Q-B2-2: fail at once.** A deploy refused by the route/actuation ceiling before the fence fails immediately, + with the reopen time in the error. +- **Q-R1 (a):** the route declaration is the challenge declaration. Every routed target answers the + stable-route challenge regardless of readiness class. R1 checks it before the fence; H1 gives Heimdall a + responder. Operator note: the question's first framing was confusing; the cause is that Heimdall has no + responder, not its class. +- **Q-B4: Idunn's own proof.** Routed providers by current route proof and not degraded; unrouted providers by + admission plus a live workload observation. One clause is **held open**: "a current Odin non-Ready word + still excludes a provider" is not ruled, pending the Odin-lifecycle authority audit (a separate Imagination + pass). Operator: "the last few rulings on Odin are making my authority sense tingle, it feels like Odin is + doing lifecycle work when that's very much Idunn's wheelhouse." B4 is blocked on that audit. + +**Odin-lifecycle audit rulings, 2026-09-30 (operator).** The audit landed as section 5. Q-O1: Idunn proves +(supersedes Q1 (b); readiness classes end at A1 + A3). Q-O2: a private challenge endpoint for unrouted targets +(A2 + a Muninn responder). Q-O4: the Odin dependency is optional and does not gate a compile. Q-O5: restart on bad +auth (amends Q5 (a)). The held Q-B4 clause is struck; B4 becomes B4′. Q-O3 (redeploy order odin -> +streampixels-service -> ghostlight -> web with the operator present, vs a one-shot relabel) is **not yet asked** +and is needed before A1 installs. Odin keeps three permanent self-publication failures fatal until A3 + Q-O5 land +(temporary rule, section 4). + +**Cut status, 2026-09-30.** +- **Ready for Hands:** B2 (rework map in section 4); D1 (after B2). **R1 is never to be written**, superseded by A1. +- **In progress:** `hands/idunn-b2-rework` (B2 rework); `hands/idunn-w1-t1-t2`: W1, T1, T2; X1 on Odin + `hands/odin-x1-residue`; the temporary fatal rule on Odin `hands/odin-session-table`. +- **Next, in order:** A1 (after B2, and after Q-O3 is ruled), B4′, A2 with M1, A3 (irreversible store migration, + back up first), O-R, L-R (section 4, Odin-authority cuts). +- **Open for Soul:** the continuity-restart stall probe and the journal check of the ghostlight/raven-muninn + exclusion (section 5.5). +- **Unblocked:** B4 (now B4′). **Blocked:** H1 (Heimdall repo), on CultLib C1. A supervision change for Q-O5 has + no cut yet. + + Heads read for this map: | Repo | Ref | Notes | @@ -267,7 +580,7 @@ An empty or "none" cell is a finding. It is marked **(gap)**. **Q1. Readiness authority for a CultMesh-aware service such as `streampixels-service`.** -**RULED (b) by the operator, 2026-09-29.** +**RULED (b) by the operator, 2026-09-29. HISTORY, not live design: superseded 2026-09-30 by Q-O1 (Idunn proves; readiness classes end at A1 + A3). The text below is kept as the record of the ruling.** Options: - (a) Every non-Odin target proves Warming and Ready directly through Idunn's @@ -333,6 +646,37 @@ have written. (c) trusts a declaration over evidence. Depends on it: B5, the recovery variants of the terminal record, and the replacement for `idunn cancel`. +**Q3 follow-ups from B5 Hands (`hands/b5` `c06ff2b`), 2026-09-30:** +- **Q3-i: does an issued lease with no adoption evidence count as possibly + written?** **RULED yes** by the operator, 2026-09-30. A candidate cannot prove + it holds the lease while warming, and streampixels-service migrates in exactly + that window. So any issued lease on a deploy is treated as possibly adopted. +- **Q3-ii: after an adopted-lease deploy expires, what stops continuity from + restarting the incumbent on state the candidate may have migrated?** + **RULED 2026-09-30: D before Ready, E after.** Operator, verbatim: "D before + Ready, E after; this actually closes the hole, and is a nice feature for Idunn + to have". + - **D, before the candidate reports Ready:** after the fence and before the + write lease is issued, Idunn snapshots the target's state through a + per-target snapshot/restore declared in its binding. If the deadline expires + before Ready, Idunn restores the snapshot and brings back the incumbent. A + failed restore is the only terminal `OperatorRequired`. + - **E, after Ready:** the candidate is known-good and may have served traffic, + so an expiry rolls forward. The candidate stays the release supervision keeps + alive, and Idunn completes or retries routing and commit. + - Assumption Soul must falsify: the incumbent cannot write between the fence + and the lease, so the snapshot is exact. + - Mapped: section 4, "Q3-ii cut: snapshot before Ready, roll forward after" (Imagination, 2026-09-30); + it establishes the assumption for Idunn-managed writers and names what breaks it (Q3-ii.2). + + History, the question as first put: + The operator on the offered hold-until-next-`idunn up` option, verbatim: "I + don't like any of these options, unless a stateful deploy running out of time + is an extreme edge case, in which case A". It is not extreme: the default + `awaiting_ready` deadline is 300 s (`deployment_plan.rs:474`), so any stateful + release that migrates and then fails to reach Ready lands here. Alternatives + are being put to the operator. + **Q4. The operator's recovery verb after `8ae00a1` is deleted.** **RULED (b) by the operator, 2026-09-29.** @@ -351,7 +695,7 @@ Depends on it: S2 scope and the CLI surface. **Q5. Whether a sustained route-proof failure on an admitted target restarts its unit.** -**RULED (a) by the operator, 2026-09-29.** +**RULED (a) by the operator, 2026-09-29. Amended 2026-09-30 by Q-O5: an answer that does not authenticate as the admitted incarnation restarts the process under the continuity meter; silent answers stay degraded-only.** Options: - (a) No. The route is marked degraded, dependents stop selecting it (Q2b), @@ -379,6 +723,171 @@ it. This is a CultLib foundation merge, so the operator should rule on it. Depends on it: C1 and the StreamPixels gitlink in C2. +**Q-B2-1. Does the continuity restart log move into `TargetSupervision` too?** + +**RULED (a) by the operator, 2026-09-30.** + +Options: +- (a) Move it. `ContinuityBackoff`'s own doc (`:907-908`) already says it "belongs to the target, not the + generation". On the generation it survives only through the copy in `from_transaction` (`:2074-2076`). + Moving it deletes that carry and key 20. The mint CAS pins the target record instead of racing the + generation, which is the F3 shape. One record holds both of a target's meters. +- (b) Keep it on the generation, reshaped to the sliding log in place. The route meter still needs + `TargetSupervision` for first deploys, so the meters split across two records and the carry stays. + +Outcome: the continuity restart log lives in `TargetSupervision`; the restart ceiling reads the target's own +log; the carry compensator in `from_transaction` is deleted. + +Depends on it: B2 (see the B2 rework map in section 4). + +**Q-B2-2. When the route ceiling refuses a first or forward deploy before the fence, does `idunn up` fail or +wait?** + +**RULED: fail at once, by the operator, 2026-09-30.** + +Options: +- (a) Fail. The deploy is aborted pre-fence with "route actuation ceiling reached for ; reopens at +