Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 140fbc3161 | |||
| d13d98a083 | |||
| 6c34758a30 | |||
| 17edea299e | |||
| 506df0a81f | |||
| eace76870a | |||
| c30f89f0df | |||
| 47a9c3d89e | |||
| cdf987730e | |||
| 8dd9f8d503 | |||
| 7eae1c4ee1 | |||
| 74c04bbcc7 | |||
| c13e175bbf | |||
| c5a0ce3d23 | |||
| 37622db430 | |||
| 58e744b745 | |||
| b230703a5c | |||
| a20568ef70 | |||
| 114803c63b | |||
| 7ba27c362e | |||
| 01b6016193 | |||
| 9f15cee11f | |||
| be98521120 | |||
| dc86c6326c | |||
| 74d6ef31e1 | |||
| d5ea779096 | |||
| 6c8fd12335 | |||
| 46ec8241a2 | |||
| 4b6a58c692 | |||
| 157610adb7 | |||
| 1c23b6e217 | |||
|
|
64d63cb417 | ||
|
|
77e2295e97 | ||
|
|
fb1e7b5dd9 | ||
|
|
8ebe0208a6 | ||
|
|
a859914b0b | ||
|
|
4894ad20df | ||
|
|
879b76a842 | ||
|
|
631fb7836b | ||
|
|
d29bf93be6 | ||
|
|
d9710c988a | ||
|
|
caacbb4bba | ||
|
|
9653d270e5 | ||
|
|
8a1beb5c87 | ||
|
|
7e15bf1a7b | ||
|
|
572f9e6aa4 | ||
|
|
56edad95c1 | ||
|
|
ca439a29ff | ||
|
|
27ca875194 | ||
|
|
be3b95bc21 | ||
|
|
cea17c06a7 | ||
|
|
ec6222e2a1 | ||
|
|
7d5cd85728 | ||
|
|
1a3634e046 | ||
|
|
e50a721dbc | ||
|
|
024f2d7a09 | ||
|
|
8400d0712f | ||
|
|
1518cfb13c | ||
|
|
549f4be093 | ||
|
|
4a68df520f | ||
|
|
134c1fb5a2 | ||
|
|
54a75ef9b0 | ||
|
|
7ef31ee699 | ||
|
|
22bd904ed9 | ||
|
|
ef82355bab | ||
|
|
78086705a0 | ||
|
|
804ed759c9 | ||
|
|
dfdc226c25 | ||
|
|
ae6a1bea55 | ||
|
|
def0b78378 | ||
|
|
69eb3e6d6f | ||
|
|
112a7a5ebb | ||
|
|
a3c6386aa4 | ||
|
|
d3222e5724 | ||
|
|
e8c4c69728 | ||
|
|
68d6f96808 | ||
|
|
07616697af | ||
|
|
fd040f09c1 | ||
|
|
76cec0b540 | ||
|
|
04ff2c8ce3 | ||
|
|
022dbece5b | ||
|
|
332dbd0149 | ||
|
|
d74ac8da5c | ||
|
|
d111986093 | ||
|
|
06782d38f2 | ||
|
|
5d8f7e18c5 | ||
|
|
b5d13652e1 | ||
|
|
6b1d14d159 | ||
|
|
2ea0572a4c | ||
|
|
1207f21948 | ||
|
|
60b64e283d | ||
|
|
d6ebd328e1 | ||
|
|
c38bb36560 | ||
|
|
e5294db40c | ||
|
|
36114b8b7b | ||
|
|
2218ffb8a8 | ||
|
|
b379fbd700 | ||
|
|
9173a0e8e6 | ||
|
|
7805c0aa9a | ||
|
|
e2dee7c16a | ||
|
|
8246de7c37 | ||
|
|
d36786f76c | ||
|
|
9c69d00641 | ||
|
|
0e09488c19 | ||
|
|
d94150d396 | ||
|
|
6866d75536 | ||
|
|
b9f27ce1e7 | ||
|
|
491459e6a0 | ||
|
|
6e2a31cf31 | ||
|
|
2f8671ac35 | ||
|
|
dbbdbf12fe | ||
|
|
8760a4fab3 | ||
|
|
c4edc66c48 | ||
|
|
b993615c69 | ||
|
|
b119e500ad | ||
|
|
04389e6afc | ||
|
|
1beca61ac4 | ||
|
|
35d9c7fcf8 | ||
|
|
11f7fdbf09 | ||
|
|
81bfb03712 | ||
|
|
4e66a65758 | ||
|
|
09bd127643 | ||
|
|
f79deecdb7 | ||
|
|
fd6e3e65af | ||
|
|
a868321259 | ||
|
|
196e0dc884 | ||
|
|
4f68809f0b | ||
|
|
a7ad3de559 | ||
|
|
e5298584a4 | ||
|
|
ec45906ead | ||
|
|
8f00df079f | ||
|
|
d9846e8847 | ||
|
|
bff308aea7 | ||
|
|
18c5b643a7 | ||
|
|
cc4a72ad43 | ||
|
|
81ab69046b | ||
|
|
063409489e | ||
|
|
58f213fe15 | ||
|
|
8230c544ac | ||
|
|
9844806eb1 | ||
|
|
95e8076618 | ||
|
|
2ff57533b1 | ||
|
|
c6805d48a7 | ||
|
|
8b6c539a6a | ||
|
|
a254a3e6a5 | ||
|
|
21849eb097 | ||
|
|
0bfadcfc71 | ||
|
|
126d710de3 | ||
|
|
a14055115c | ||
|
|
86926c4e42 | ||
|
|
ffb22e368b | ||
|
|
3265ccb575 | ||
|
|
9ea6c71921 | ||
|
|
94d3b53b55 | ||
|
|
5b421c3cc7 | ||
|
|
6ea0d127d8 | ||
|
|
dd8707b5a2 | ||
|
|
1953bf233b | ||
|
|
a7bfbc2296 | ||
|
|
278cc5c408 | ||
|
|
9139d7edb8 | ||
|
|
755306d654 | ||
|
|
b4bb158707 | ||
|
|
d91ecba759 | ||
|
|
61383297b2 | ||
|
|
9a0a6a2ce9 | ||
|
|
1da150a309 | ||
|
|
ffb42936ca | ||
|
|
da9d246956 | ||
|
|
f6d2943fd3 | ||
|
|
a75e31330a | ||
|
|
07a1d41692 | ||
|
|
32ff879790 | ||
|
|
b1dfe829a1 | ||
|
|
2fae3af133 | ||
|
|
69464b11ef | ||
|
|
61fc89bbd7 | ||
|
|
9a6b4a128c | ||
|
|
6eb7931a8c | ||
|
|
e36cad32f0 | ||
|
|
f079389be0 | ||
|
|
74a700817f | ||
|
|
f54c776417 | ||
|
|
c2477d43d4 | ||
|
|
6667f0f9a6 | ||
|
|
cb6096512b | ||
|
|
424300d672 | ||
|
|
827cfad0bb | ||
|
|
102c129891 | ||
|
|
a909f931ea | ||
|
|
7e4163645d | ||
|
|
8d0786dcaf | ||
|
|
9549d00fb9 | ||
|
|
4ac96e959d | ||
|
|
bca3d85394 | ||
|
|
b69a8e81ba | ||
|
|
ddc031a363 | ||
|
|
f1b45666c6 | ||
|
|
1cfbeff083 | ||
|
|
fcb8ba1c29 | ||
|
|
7793b5bb89 | ||
|
|
889e23629c | ||
|
|
85964d0f00 | ||
|
|
f85e74377f | ||
|
|
1f7c8e8f42 | ||
|
|
3ed5f262ba | ||
|
|
9ad7e4209b | ||
|
|
32cf1f7bf4 | ||
|
|
aef0da34af | ||
|
|
dcd0aa3f2b | ||
|
|
8b9e8dfe11 | ||
|
|
224f006983 | ||
|
|
64da79402a | ||
|
|
c04f24f518 | ||
|
|
e0b31eee76 | ||
|
|
47067cce08 | ||
|
|
4ee70f90ba | ||
|
|
6854b0f825 | ||
|
|
5f232a6120 | ||
|
|
3fad441917 | ||
|
|
e00ff5ec40 | ||
|
|
448f554af5 | ||
|
|
c06ff2b84c | ||
|
|
855ccf8247 | ||
|
|
6394100a93 | ||
|
|
7f7528b33e | ||
|
|
202e5a2e10 | ||
|
|
436d6445fc | ||
|
|
487f4f14b8 | ||
|
|
153af6ac40 | ||
|
|
4830af8eb1 | ||
|
|
495112565b | ||
|
|
0c80a444f6 | ||
|
|
3f4d7561d9 | ||
|
|
fa830c5f6f | ||
|
|
6ff7d5a9c7 | ||
|
|
a96ad9d4e6 | ||
|
|
5939abd87e | ||
|
|
057bdd5c6c | ||
|
|
e9c718e073 | ||
|
|
08d346dfae | ||
|
|
ecf163ae1c | ||
|
|
46d9c2f9dc | ||
|
|
ed07241f34 | ||
|
|
8b9ed0e8d9 | ||
|
|
eceb742ea3 | ||
|
|
c127b53211 | ||
|
|
cccfe8e174 | ||
|
|
d83b1a9c95 | ||
|
|
b6e78e5366 | ||
|
|
0703691790 | ||
|
|
05881ea8d8 | ||
|
|
74e7ed495e | ||
|
|
757e55be8f | ||
|
|
012853529c | ||
|
|
76aaff1e71 | ||
|
|
8447c22201 | ||
|
|
bc0f82e4f1 | ||
|
|
bb0d23c780 | ||
|
|
ef6cfb177f | ||
|
|
1afe0466a4 | ||
|
|
b55bb97f46 | ||
|
|
bdcd23768b | ||
|
|
55f433eb2d | ||
|
|
e7605efe26 | ||
|
|
e7ee053427 | ||
|
|
0bb791d913 | ||
|
|
8212b9d15d | ||
|
|
88ce30905f | ||
|
|
916caed9dd | ||
|
|
61842a3952 | ||
|
|
36467153b5 | ||
|
|
9f17a733dc | ||
|
|
525856e9cd | ||
|
|
3cedfef50a | ||
|
|
76c7b13b33 | ||
|
|
97fb2b8871 | ||
|
|
bf5347c370 | ||
|
|
5f8417589d | ||
|
|
94c7691cb6 | ||
|
|
7480ee9d1c | ||
|
|
583b2a7796 | ||
|
|
08d94ee61b | ||
|
|
f9cebb33d0 | ||
|
|
7f776ec3a6 | ||
|
|
28662cb807 | ||
|
|
d3db58227a | ||
|
|
d32395ac99 | ||
|
|
ae3788909a | ||
|
|
5c49646b81 | ||
|
|
d34f5dc3f9 | ||
|
|
8432d45bbe | ||
|
|
137e120ce9 | ||
|
|
bf2c645a60 | ||
|
|
e8b44878b7 | ||
|
|
4b3b86d863 | ||
|
|
d5669af352 | ||
|
|
3803725bf2 | ||
|
|
2f22dcac96 | ||
|
|
477927509a | ||
|
|
7dc3a4bb8c | ||
|
|
86ee21dac5 | ||
|
|
aa1889b288 | ||
|
|
8d1dc9a87e | ||
|
|
771138a164 | ||
|
|
e11b457458 | ||
|
|
6ea824b5a9 | ||
|
|
6501cafda3 | ||
|
|
5ca7760fdd | ||
|
|
4dcdd3608c | ||
|
|
54507060a5 | ||
|
|
b9864b5041 | ||
|
|
66fda86d8f | ||
|
|
cfe71f680c | ||
|
|
a7b602ae65 | ||
|
|
a61540b93f | ||
|
|
5743a25389 | ||
|
|
287aa92382 | ||
|
|
8998cfcf02 | ||
|
|
81478e6ddd | ||
|
|
81ec1c7f40 | ||
|
|
b75fbdf96c | ||
|
|
3d4c9d34f1 | ||
|
|
ff3cd51910 | ||
|
|
9e5b7f84fd | ||
|
|
4b5e60fb12 | ||
|
|
6e5f87676b | ||
|
|
c03d646ae6 | ||
|
|
4d8ac7a7e8 | ||
|
|
945cb0b322 | ||
|
|
fd75014b6e | ||
|
|
b570114ae6 | ||
|
|
35e4fad8ab | ||
|
|
1450bf9513 | ||
|
|
d795e09a9e | ||
|
|
5d8eb533ab | ||
|
|
09222ad8eb | ||
|
|
6fcc1ba6f3 | ||
|
|
4a9fe87787 | ||
|
|
44277698c3 | ||
|
|
87fa57dd8c | ||
|
|
b619f9875c | ||
|
|
9001b58f4b | ||
|
|
5bbb4616dd | ||
|
|
d719485825 | ||
|
|
8ae00a1244 | ||
|
|
29946bbe66 | ||
|
|
ae8ac45df4 | ||
|
|
5e94eb82f0 | ||
|
|
e0f514745a | ||
|
|
4200330fcc | ||
|
|
e976d32243 | ||
|
|
865190611f | ||
|
|
9f00e7ab84 | ||
|
|
f0465e5c9a | ||
|
|
7a61c77420 | ||
|
|
55e5d559d6 | ||
|
|
a148802cb6 | ||
|
|
02e27a1048 | ||
|
|
022096c45e | ||
|
|
9ede4cd8a0 | ||
|
|
99d3e86d03 | ||
|
|
b029cd29b7 | ||
|
|
098febb996 | ||
|
|
bcf9de3837 | ||
|
|
9d86ca1f60 | ||
|
|
7de13c8c1d | ||
|
|
a79d6282a3 | ||
|
|
38502d1c21 |
62 changed files with 51924 additions and 9848 deletions
1
Cargo.lock
generated
1
Cargo.lock
generated
|
|
@ -482,6 +482,7 @@ dependencies = [
|
|||
"libc",
|
||||
"rmp-serde",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"tempfile",
|
||||
"toml",
|
||||
|
|
|
|||
|
|
@ -18,6 +18,10 @@ path = "src/bin/idunn-provision.rs"
|
|||
name = "idunn-host"
|
||||
path = "src/bin/idunn-host.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "idunn-launch"
|
||||
path = "src/bin/idunn-launch.rs"
|
||||
|
||||
[lib]
|
||||
name = "idunn_daemon"
|
||||
path = "src/lib.rs"
|
||||
|
|
@ -34,6 +38,7 @@ serde = { version = "1", features = ["derive"] }
|
|||
ed25519-dalek = "2.2"
|
||||
toml = "0.8"
|
||||
uuid = { version = "1", features = ["v4"] }
|
||||
serde_json = "1"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
|
|
|
|||
146
README.md
146
README.md
|
|
@ -58,6 +58,64 @@ idunn status --command <id>
|
|||
the bindings, so `idunn up profile:full-gamecult` brings up everything bound
|
||||
into that profile.
|
||||
|
||||
### Steward orders
|
||||
|
||||
A second root unit from the same binary, `idunn-orders` (`idunn orders serve`),
|
||||
takes typed orders from one steward Mind over `/run/idunn-orders/steward.sock`.
|
||||
It has its own store, `orders.cc`, and its own binding, `steward.toml`; the
|
||||
survival daemon takes no order. There is still no command string:
|
||||
|
||||
```bash
|
||||
idunn order package-install <pkg>[=version]... --reason TEXT # apt-get update, install named packages, check them
|
||||
idunn order unit <start|stop|restart|reload> <unit> --reason TEXT
|
||||
idunn order deploy <service|profile:name> --reason TEXT # the same commit as `idunn up`
|
||||
idunn order show <id> # status and output tails
|
||||
idunn order list
|
||||
```
|
||||
|
||||
A unit order submits exactly one job, for the unit it names: every verb runs
|
||||
`systemctl <verb> --job-mode=ignore-requirements -- <unit>`, so systemd builds a
|
||||
transaction of one job. It does not start what the unit requires or restart what
|
||||
is part of it: order the dependency first, and order `postgresql@16-main.service`
|
||||
rather than the `postgresql.service` wrapper. Targets are not orderable. The job is one; what the unit does afterwards is
|
||||
not: systemd still runs full transactions of its own for `Restart=` after a
|
||||
failure, for a `StopWhenUnneeded=` dependency left without a user, and for a
|
||||
`Conflicts=` partner of a unit that fails, so an ordered stop of a unit
|
||||
that something restarts does not last, and a start of a `Restart=always`
|
||||
unit can report failure and be active seconds later. A unit
|
||||
order may act on any unit except Idunn's own (`idunn-*`) and the minds' own
|
||||
(`mind-self@*`, `mind-persona@*`), and the units that end or take the host
|
||||
(`systemd-suspend`, `systemd-hibernate`, `systemd-hybrid-sleep`,
|
||||
`systemd-suspend-then-hibernate`, `rescue`, `emergency`). That set is compiled
|
||||
into the binary and read from no configuration. An order is refused, recorded
|
||||
and posted when the ordered unit, or an alias it resolves from, is one of them;
|
||||
when it has an emergency action (`SuccessAction=`, `FailureAction=`,
|
||||
`StartLimitAction=`, `JobTimeoutAction=`) other than `none` or an `OnFailure=`
|
||||
or `OnSuccess=` unit (not for a reload); or, for a stop or restart, when another
|
||||
unit is `BoundBy=` it, because systemd stops that unit with it whatever the job
|
||||
mode; for a start or restart, when it has an `Upholds=` unit, which systemd
|
||||
starts in a full transaction whatever the job mode; or, for a stop, restart or
|
||||
reload, when it has a `ConsistsOf=` member, a wrapper such as
|
||||
`postgresql.service` whose verbs would exit 0 and touch nothing. A unit
|
||||
`systemctl show` cannot resolve fails the order. Slices, scopes,
|
||||
targets and globs are not units an order can name.
|
||||
|
||||
A package order installs the packages it names and nothing else: apt runs with
|
||||
`--no-remove`, each name is passed as an exact-name pattern so `.` and `+`
|
||||
match themselves, and a last step fails the order unless every named package
|
||||
is installed.
|
||||
|
||||
An order runs once, after the steward's uid is checked and the instance's
|
||||
brake check (`mind-control --instance <principal> check`) exits 0. The check
|
||||
that the order's principal and uid are the bound steward's runs again at every
|
||||
execution, so an order accepted before `steward.toml` changed is refused and
|
||||
recorded, not run. The socket belongs to the steward user alone (mode 0600);
|
||||
root may `show` and `list` orders but never issue one. An order that had
|
||||
started when the service died is recorded as lost and never run again. No order releases a brake. Acceptance and result are posted to
|
||||
`#operator` as a retried projection of `orders.cc`: a chat outage delays the
|
||||
posts and not the orders. A refusal names the field and a code, never the value
|
||||
it refused.
|
||||
|
||||
## Authority
|
||||
|
||||
**Owner.** Deployment admission, and the continuity of installed bodies.
|
||||
|
|
@ -150,6 +208,79 @@ omits one. Unknown fields are rejected on both sides. A repository that gets
|
|||
compromised can therefore change what is built, but not what it is allowed to
|
||||
touch on the host.
|
||||
|
||||
An upstream daemon that does not speak Idunn's runtime contract is deployed
|
||||
with `idunn-launch` as its executable artifact and the upstream tree as a
|
||||
required adjacent artifact; the launcher finds the child inside the release that
|
||||
holds it. The launcher is the unit's process: it binds the
|
||||
candidate endpoint and answers Idunn's challenge there, renders the child's
|
||||
config, starts a stateful child only once it holds the write lease, and signs
|
||||
presence. Members never pass through it: the binding's `[route.member]` makes
|
||||
Idunn route a second listener straight to the child, which binds the address
|
||||
Idunn gives it in `GAMECULT_IDUNN_MEMBER_BIND`.
|
||||
|
||||
```toml
|
||||
# recipe fragment — the upstream tree is built or fetched as an artifact
|
||||
[service]
|
||||
executable_artifact = "idunn-launch"
|
||||
arguments = [
|
||||
{ kind = "literal", value = "--child" },
|
||||
{ kind = "literal", value = "upstream/bin/daemon" },
|
||||
{ kind = "literal", value = "--template" },
|
||||
{ kind = "literal", value = "config.tmpl" },
|
||||
{ kind = "literal", value = "--health-path" },
|
||||
{ kind = "literal", value = "/health" },
|
||||
{ kind = "literal", value = "--health-contract" },
|
||||
{ kind = "literal", value = "example.presence-launcher.v1" },
|
||||
]
|
||||
required_environment = [
|
||||
"GAMECULT_IDUNN_RUNTIME_BUNDLE",
|
||||
"GAMECULT_IDUNN_CANDIDATE_BIND",
|
||||
"GAMECULT_IDUNN_MEMBER_BIND",
|
||||
]
|
||||
```
|
||||
|
||||
A directory artifact is sealed read-only: its files are 0444 and its
|
||||
directories 0555, whatever mode the build or git gave them. A file the tree
|
||||
must run, such as the launcher's `upstream/bin/daemon` child, is declared in
|
||||
`executable_members` and sealed 0555. The members are part of the sealed release
|
||||
address, so changing the list makes a new release. Idunn refuses the release if a
|
||||
member is absolute, climbs with `..`, is missing, is a symlink or sits behind
|
||||
one, or is not a regular file; `executable = true` on a directory output and
|
||||
`executable_members` on a file output are refused too.
|
||||
|
||||
A tree may hold relative symlinks whose `..` steps all come first and climb no
|
||||
higher than the artifact's top (`share/terminfo/4/4025ex -> ../t/tek4025ex` is
|
||||
kept). An absolute link, a `..` after a name, or a climb out of the artifact
|
||||
refuses the release when it is sealed. Host targets (Windows) record
|
||||
`executable` and `executable_members` in the receipt and do not apply them.
|
||||
|
||||
```toml
|
||||
# recipe fragment — the upstream artifact declares the child it launches
|
||||
[[artifacts]]
|
||||
id = "upstream"
|
||||
source_kind = "runner-output"
|
||||
runner = "build"
|
||||
source = "dist/upstream"
|
||||
destination = "upstream"
|
||||
executable_members = ["bin/daemon"]
|
||||
```
|
||||
|
||||
```toml
|
||||
# binding fragment — members connect to the member endpoint, which Idunn
|
||||
# forwards to the private port derived from the candidate: member
|
||||
# private_port_start + (candidate port - route private_port_start)
|
||||
[route]
|
||||
stable_endpoint = "tcp://10.77.0.1:8008"
|
||||
private_port_start = 18831
|
||||
private_port_end = 18839
|
||||
# ...
|
||||
|
||||
[route.member]
|
||||
stable_endpoint = "tcp://10.77.0.1:8448"
|
||||
private_port_start = 18841
|
||||
# max_connections = 128 # nginx limit_conn on the member listener (1-65535)
|
||||
```
|
||||
|
||||
## Contracts
|
||||
|
||||
Idunn's typed contracts are published by
|
||||
|
|
@ -174,11 +305,12 @@ the all-seer's view, not Idunn's contract, and the two sets do not overlap.
|
|||
| `idunn-yggdrasil` | operator wrapper |
|
||||
| `idunn-yggdrasil.sudoers` | the narrow sudo grant the wrapper needs |
|
||||
|
||||
The unit runs as a dedicated `idunn` user under `ProtectSystem=full`,
|
||||
`ProtectHome=yes`, `PrivateTmp=yes`. `ReadWritePaths` is Idunn's own state plus
|
||||
the build root; the operator anchor, the bindings, and Odin's correlation store
|
||||
are all `ReadOnlyPaths`. Idunn admits against operator authority — it does not
|
||||
author it.
|
||||
The unit runs as root under `ProtectSystem=full`,
|
||||
`ProtectHome=yes`, `PrivateTmp=yes`. `ReadWritePaths` contains Idunn's own state,
|
||||
the build root, and only the runtime, state, and release roots explicitly
|
||||
granted to each managed target. The operator anchor, bindings, and Odin's
|
||||
correlation store remain read-only. Idunn admits against operator authority —
|
||||
it does not author it.
|
||||
|
||||
Note `Wants=docker.service`, not `Requires=`. Only the build step is
|
||||
containerized; warm, fence, promote, drain and restart run on systemd, and
|
||||
|
|
@ -203,7 +335,9 @@ purpose; Idunn discovers no authority implicitly.
|
|||
|
||||
Idunn targets Linux and does not compile on Windows (`drivers.rs` uses
|
||||
`std::os::unix` unconditionally). A local `cargo check` on a Windows
|
||||
workstation proves nothing. Test and build where it runs, in the same image
|
||||
workstation proves nothing. `scripts/verify.sh` is the verification command:
|
||||
the library tests plus a Windows-target check of `idunn-host`. Run it on Linux
|
||||
with the `x86_64-pc-windows-gnu` std installed. Test and build where it runs, in the same image
|
||||
Odin uses, with the working tree shipped to the host's build root:
|
||||
|
||||
```bash
|
||||
|
|
|
|||
47
deploy/idunn-orders.service
Normal file
47
deploy/idunn-orders.service
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
[Unit]
|
||||
Description=Idunn steward orders: typed package, unit and deploy orders from the steward
|
||||
# Deliberately no After= or Requires= on any managed target, Synapse or
|
||||
# idunn-yggdrasil: orders are how an operator repairs those, so none of them
|
||||
# may be a condition of this unit starting. Posts are a retried projection and
|
||||
# wait for Synapse on their own.
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
|
||||
# Root, from the same binary as idunn-yggdrasil, with its own store and socket.
|
||||
# idunn-yggdrasil keeps its no-imperative-input invariant and its write set:
|
||||
# no order code runs in `idunn serve`.
|
||||
#
|
||||
# The unit name matches idunn-*, which is in the always-refused set, so no
|
||||
# order can stop or restart this service.
|
||||
User=root
|
||||
# No Group= and no UMask=: the service gives its socket to the steward user
|
||||
# (mode 0600) itself and sets Idunn's own umask for the files it writes, so
|
||||
# nothing here reaches control.cc. The steward user is the only peer the
|
||||
# service answers: it checks SO_PEERCRED on every connection.
|
||||
ExecStart=/usr/local/bin/idunn orders serve \
|
||||
--store /var/lib/gamecult/idunn-orders \
|
||||
--state-store /var/lib/gamecult/idunn/control.cc \
|
||||
--steward-binding /etc/gamecult/idunn/steward.toml \
|
||||
--socket /run/idunn-orders/steward.sock
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
TimeoutStopSec=20
|
||||
|
||||
RuntimeDirectory=idunn-orders
|
||||
RuntimeDirectoryMode=0755
|
||||
|
||||
# The Synapse access token of @idunn, copied from a root-only file, and read
|
||||
# through $CREDENTIALS_DIRECTORY; the service names it in steward.toml.
|
||||
LoadCredential=idunn-matrix-token:/etc/gamecult/idunn/matrix-token
|
||||
|
||||
PrivateTmp=yes
|
||||
ProtectHome=yes
|
||||
ProtectSystem=strict
|
||||
# Its own order store, and the control store a deploy order commits a command
|
||||
# to. Transient units started with systemd-run run in the manager's namespace,
|
||||
# so packages and units are not limited by this list.
|
||||
ReadWritePaths=/var/lib/gamecult/idunn-orders /var/lib/gamecult/idunn
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -41,10 +41,10 @@ ExecStart=/usr/local/bin/idunn serve \
|
|||
--source-root /var/lib/gamecult/idunn/sources \
|
||||
--staging-root /var/lib/gamecult/idunn/staging \
|
||||
--topology-store /var/lib/gamecult/idunn-projection/topology.cc \
|
||||
--incident-store /var/lib/gamecult/idunn-projection/incidents.cc \
|
||||
--idunn-identity-store /var/lib/gamecult/idunn/idunn-service-identity.cc \
|
||||
--odin-correlation-store /var/lib/gamecult/odin/topology.cc \
|
||||
--odin-trust-anchor /etc/gamecult/idunn/odin-topology-anchor.cc \
|
||||
--deployment-brake-operator-anchor /etc/gamecult/idunn/deployment-brake-operator-anchor.cc \
|
||||
--host-actuator-bind 10.77.0.1:17890 \
|
||||
--source-uid 986 \
|
||||
--source-gid 978
|
||||
|
|
@ -76,12 +76,14 @@ ReadWritePaths=/etc/ufw
|
|||
# idunn-projection is deliberately outside Idunn's own 0750 directory: the
|
||||
# topology store is Idunn's *published* surface, and every managed target must
|
||||
# read it to verify its own Expected incarnation against the Idunn anchor. It
|
||||
# was inside the private root, where no workload could traverse to it.
|
||||
# was inside the private root, where no workload could traverse to it. The
|
||||
# published topology store is read lock-free (Odin) and incidents.cc is read by
|
||||
# Bifrost's reader; managed targets get Expected from their runtime bundle. Only
|
||||
# the stores are readable: each .lock beside them is Idunn's alone, 0600.
|
||||
ReadWritePaths=/var/lib/gamecult/idunn /var/lib/gamecult/idunn-authority /var/lib/gamecult/idunn-projection
|
||||
ReadWritePaths=/srv/build
|
||||
# One entry per managed target: its runtime root, state root and release root.
|
||||
# A target Idunn cannot write is a target it cannot deploy, so this list grows
|
||||
# by exactly three paths per migration and is the honest blast radius.
|
||||
# Each managed target gets only its bound roots. A target Idunn cannot write is
|
||||
# a target it cannot deploy; keep this list aligned with operator bindings.
|
||||
ReadWritePaths=/etc/gamecult/heimdall /var/lib/gamecult/heimdall /srv/heimdall/app
|
||||
# Odin is a managed target like any other now, so its three roots are writable
|
||||
# on the same terms. This used to be ReadOnlyPaths, on the reasoning that Idunn
|
||||
|
|
@ -93,6 +95,19 @@ ReadWritePaths=/etc/gamecult/heimdall /var/lib/gamecult/heimdall /srv/heimdall/a
|
|||
# target it cannot deploy, and Odin has to be deployable: it is the bootstrap
|
||||
# target every other target's warming presence is observed through.
|
||||
ReadWritePaths=/etc/gamecult/odin /var/lib/gamecult/odin /srv/odin
|
||||
# Ghostlight Dungeon: its runtime root only, not /etc/gamecult/ghostlight,
|
||||
# which also holds the credential originals Idunn reads but never writes.
|
||||
# The release root sits beside the legacy v1 actuator's /srv/ghostlight.
|
||||
ReadWritePaths=/etc/gamecult/ghostlight/runtime /var/lib/gamecult/ghostlight-world-v2 /srv/ghostlight-idunn
|
||||
# StreamPixels service: runtime bundle, rebuildable state boundary, and releases.
|
||||
ReadWritePaths=/etc/gamecult/streampixels-service/runtime /var/lib/gamecult/streampixels-service /srv/streampixels-idunn/releases/service
|
||||
# StreamPixels web is stateless; it needs only its runtime bundle and releases.
|
||||
ReadWritePaths=/etc/gamecult/streampixels-web/runtime /srv/streampixels-idunn/releases/web
|
||||
# Vili (motion worker): runtime bundle, state, and releases. Prefixed "-" so a
|
||||
# target whose roots are not created yet cannot stop Idunn itself from starting
|
||||
# (Idunn startup never depends on a managed target). A root created after Idunn
|
||||
# starts is writable only from Idunn's next restart.
|
||||
ReadWritePaths=-/etc/gamecult/vili/runtime -/var/lib/gamecult/vili -/srv/vili-idunn/releases
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
|
|||
|
|
@ -4,11 +4,16 @@
|
|||
kept because they are what is *running*, and a migration needs both sides.
|
||||
|
||||
`idunn-yggdrasil` is a root shell actuator reached through the narrow sudo grant
|
||||
in `idunn-yggdrasil.sudoers`. It hardcodes eleven targets, their repositories
|
||||
in `idunn-yggdrasil.sudoers`. It hardcodes ten targets, their repositories
|
||||
and their admitted refs, then delegates to executable manifests under
|
||||
`/srv/odin/deploy-manifests/<target>`. It calls `idunn validate-release-authority`
|
||||
and `idunn validate-minimum-source-revision` — subcommands the current binary
|
||||
does not have.
|
||||
`/srv/odin/deploy-manifests/<target>`. It calls `idunn validate-minimum-source-revision`,
|
||||
a subcommand the current binary does not have.
|
||||
|
||||
Checked on yggdrasil 2026-09-30: `/usr/local/libexec/idunn-yggdrasil` is installed
|
||||
and matches this file byte for byte (before the retirement below), but
|
||||
`idunn-yggdrasil.sudoers` is not installed. `sudo -l -U idunn` says the `idunn`
|
||||
user may not run sudo, so the daemon cannot reach this actuator; only an
|
||||
operator running it as root can.
|
||||
|
||||
## What replaced it
|
||||
|
||||
|
|
@ -16,10 +21,10 @@ Everything this script does by convention, the current Idunn does by contract:
|
|||
|
||||
| Legacy mechanism | Current owner |
|
||||
|---|---|
|
||||
| `deployment_target_policy()` — a `case` over eleven target names | one operator binding per target, in `--bindings-dir` |
|
||||
| `deployment_target_policy()` — a `case` over ten target names | one operator binding per target, in `--bindings-dir` |
|
||||
| `upstream_ref=refs/heads/...` in shell | `[repository] admitted_ref` |
|
||||
| `minimum_source_revision_required` | `[repository] minimum_revision` |
|
||||
| `requires_bifrost_authority` + `--release-authority-store` | `[repository] selection = "signed-release"`, per target |
|
||||
| `requires_bifrost_authority` + `--release-authority-store` (retired with its only target) | `[repository] selection = "signed-release"`, per target |
|
||||
| `/srv/odin/deploy-manifests/<target>` shell scripts | recipe `[[steps]]` in the target's own repository |
|
||||
| `run_manifest_while_holding_authority_locks` | the deployment transaction and process write-lease |
|
||||
| `flock` on a brake store | `[brakes] deployment_store` and `lifecycle_store` |
|
||||
|
|
@ -45,15 +50,26 @@ This is the real migration scope, read out of `deployment_target_policy()`:
|
|||
| `heimdall` | GameCult/Heimdall | `main` | restart is `docker compose restart` |
|
||||
| `epiphany` | GameCult/Epiphany | `codex/epiphany-shakedown-live` | |
|
||||
| `epiphany-capstone-17` | GameCult/Epiphany | `codex/epiphany-shakedown-live` | second target, same repo |
|
||||
| `bifrost-persona-feedback` | GameCult/Bifrost | `main` | **only** target requiring Bifrost release authority |
|
||||
| ~~`bifrost-persona-feedback`~~ | GameCult/Bifrost | `main` | **retired 2026-09-30**, see below |
|
||||
| `repixelizer` | GameCult/repixelizer | `main` | |
|
||||
| `streampixels` | GameCult/StreamPixels | `main` | two units, one target |
|
||||
| `gjallar` | GameCult/Gjallar | `codex/yggdrasil-aggregate-daemon` | |
|
||||
|
||||
Seven of eleven are pinned to a non-`main` branch. That is its own problem — the
|
||||
Seven of the ten live targets are pinned to a non-`main` branch. That is its own problem — the
|
||||
swarm goal is everyone on `main` and current with each other — and migrating a
|
||||
target is the moment to fix its ref, not to copy it forward.
|
||||
|
||||
## Retired targets
|
||||
|
||||
- `bifrost-persona-feedback`, 2026-09-30. Its manifest ran the Bifrost compose lane
|
||||
(`docker compose up persona-feedback`), while the live provider is the native
|
||||
`bifrost-persona-feedback.service`, so firing it would have started a container
|
||||
beside the native unit. The operator ruled "delete the Bifrost lane". Its grant,
|
||||
dispatch case and the Bifrost release-authority check it alone used are gone; a
|
||||
deploy request that claims a Bifrost authority is refused. It was the only
|
||||
caller of `idunn validate-release-authority`. Rebinding persona-feedback to a
|
||||
current Idunn binding is a separate follow-up.
|
||||
|
||||
## When these files die
|
||||
|
||||
When every target above has a binding and a recipe, and `idunn serve` owns
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ target="${2:-}"
|
|||
compose=/srv/compose/yggdrasil-apps.yaml
|
||||
brake_store=/var/lib/gamecult/idunn-authority/deployment-brake.cc
|
||||
brake_lock=/var/lib/gamecult/idunn-authority/deployment-brake.cc.lock
|
||||
brake_anchor=/etc/gamecult/idunn/deployment-brake-operator-anchor.cc
|
||||
generation_lock=/var/lib/gamecult/idunn-authority/idunn-generation.lock
|
||||
idunn_provision=/usr/local/bin/idunn-provision
|
||||
|
||||
|
|
@ -96,14 +95,12 @@ deployment_target_policy() {
|
|||
daemon_id=yggdrasil-odin
|
||||
repository_full_name=GameCult/Odin
|
||||
upstream_ref=refs/heads/codex/ygg-idunn-independent-bootstrap
|
||||
requires_bifrost_authority=false
|
||||
manifest_takes_action=true
|
||||
;;
|
||||
codex-connector)
|
||||
daemon_id=yggdrasil-codex-connector
|
||||
repository_full_name=GameCult/CodexConnector
|
||||
upstream_ref=refs/heads/codex/ghostlight-release-binding
|
||||
requires_bifrost_authority=false
|
||||
minimum_source_revision_required=true
|
||||
manifest_takes_action=true
|
||||
;;
|
||||
|
|
@ -111,7 +108,6 @@ deployment_target_policy() {
|
|||
daemon_id=yggdrasil-ghostlight
|
||||
repository_full_name=GameCult/Ghostlight
|
||||
upstream_ref=refs/heads/codex/ghostlight-dungeon-mvp
|
||||
requires_bifrost_authority=false
|
||||
minimum_source_revision_required=true
|
||||
manifest_takes_action=true
|
||||
;;
|
||||
|
|
@ -119,58 +115,45 @@ deployment_target_policy() {
|
|||
daemon_id=yggdrasil-voidbot
|
||||
repository_full_name=GameCult/VoidBot
|
||||
upstream_ref=refs/heads/main
|
||||
requires_bifrost_authority=false
|
||||
manifest_takes_action=true
|
||||
;;
|
||||
heimdall)
|
||||
daemon_id=yggdrasil-heimdall
|
||||
repository_full_name=GameCult/Heimdall
|
||||
upstream_ref=refs/heads/main
|
||||
requires_bifrost_authority=false
|
||||
;;
|
||||
epiphany-capstone-17)
|
||||
daemon_id=yggdrasil-epiphany-capstone-17
|
||||
repository_full_name=GameCult/Epiphany
|
||||
upstream_ref=refs/heads/codex/epiphany-shakedown-live
|
||||
requires_bifrost_authority=false
|
||||
manifest_takes_action=true
|
||||
;;
|
||||
epiphany)
|
||||
daemon_id=yggdrasil-epiphany
|
||||
repository_full_name=GameCult/Epiphany
|
||||
upstream_ref=refs/heads/codex/epiphany-shakedown-live
|
||||
requires_bifrost_authority=false
|
||||
manifest_takes_action=true
|
||||
;;
|
||||
bifrost-persona-feedback)
|
||||
daemon_id=yggdrasil-bifrost-persona-feedback
|
||||
repository_full_name=GameCult/Bifrost
|
||||
upstream_ref=refs/heads/main
|
||||
requires_bifrost_authority=true
|
||||
;;
|
||||
repixelizer)
|
||||
daemon_id=yggdrasil-repixelizer
|
||||
repository_full_name=GameCult/repixelizer
|
||||
upstream_ref=refs/heads/main
|
||||
requires_bifrost_authority=false
|
||||
;;
|
||||
streampixels)
|
||||
daemon_id=yggdrasil-streampixels
|
||||
repository_full_name=GameCult/StreamPixels
|
||||
upstream_ref=refs/heads/main
|
||||
requires_bifrost_authority=false
|
||||
;;
|
||||
gjallar)
|
||||
daemon_id=yggdrasil-gjallar
|
||||
repository_full_name=GameCult/Gjallar
|
||||
upstream_ref=refs/heads/codex/yggdrasil-aggregate-daemon
|
||||
requires_bifrost_authority=false
|
||||
;;
|
||||
*) echo "deployment target has no root actuator policy: $target" >&2; exit 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
require_signed_deployment_grant() {
|
||||
require_deployment_admission() {
|
||||
source_commit="$3"
|
||||
deployment_request_id="$4"
|
||||
release_authority_id="$5"
|
||||
|
|
@ -190,43 +173,23 @@ require_signed_deployment_grant() {
|
|||
*) echo "signed deployment request is not for root target $target" >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
case "$requires_bifrost_authority" in
|
||||
true)
|
||||
[ -n "$release_authority_id" ] || { echo "missing Bifrost release authority id" >&2; exit 1; }
|
||||
case "$release_authority_sha256" in
|
||||
*[!0-9a-f]*|'') echo "invalid or missing Bifrost release authority digest" >&2; exit 1 ;;
|
||||
esac
|
||||
[ "${#release_authority_sha256}" -eq 64 ] || { echo "Bifrost release authority digest must be 64 lowercase hex characters" >&2; exit 1; }
|
||||
;;
|
||||
false)
|
||||
[ -z "$release_authority_id" ] || { echo "source-observed release cannot claim Bifrost authority id" >&2; exit 1; }
|
||||
[ -z "$release_authority_sha256" ] || { echo "source-observed release cannot claim Bifrost authority digest" >&2; exit 1; }
|
||||
;;
|
||||
esac
|
||||
# No target carries Bifrost release authority: its only one, bifrost-persona-feedback,
|
||||
# was retired on 2026-09-30. A request that claims one is refused.
|
||||
[ -z "$release_authority_id" ] || { echo "source-observed release cannot claim Bifrost authority id" >&2; exit 1; }
|
||||
[ -z "$release_authority_sha256" ] || { echo "source-observed release cannot claim Bifrost authority digest" >&2; exit 1; }
|
||||
|
||||
require_root_owned_resolved_executable "$idunn_provision"
|
||||
require_root_owned_regular_path "$brake_store"
|
||||
require_root_owned_regular_path "$brake_lock"
|
||||
require_root_owned_regular_path "$brake_anchor"
|
||||
|
||||
# This validation is the deployment-admission point. The signed expiry is a
|
||||
# start-before deadline: it denies a later transaction, but does not become a
|
||||
# timer-shaped veto over one already admitted. The root actuator remains the
|
||||
# sole shared-lock holder through the synchronous consequence, so an explicit
|
||||
# engage/release transition cannot race or rewrite that transaction.
|
||||
exec 9<"$brake_lock"
|
||||
# This validation is the deployment-admission point. An absent or clear stop
|
||||
# admits; an engaged or unreadable one makes the status verb exit nonzero.
|
||||
# The root actuator remains the sole shared-lock holder through the
|
||||
# synchronous consequence, so an engage or clear cannot race or rewrite it.
|
||||
# The lock file is created here when no store has ever been written.
|
||||
exec 9>>"$brake_lock"
|
||||
/usr/bin/flock --shared 9
|
||||
now_unix_millis="$(/usr/bin/date +%s%3N)"
|
||||
case "$now_unix_millis" in
|
||||
''|*[!0-9]*) echo "root actuator clock did not produce Unix milliseconds" >&2; exit 1 ;;
|
||||
esac
|
||||
"$idunn_provision" deployment-brake-status \
|
||||
--store "$brake_store" \
|
||||
--operator-anchor "$brake_anchor" \
|
||||
--runtime-id yggdrasil \
|
||||
--release-id "$source_commit" \
|
||||
--deployment-id "$deployment_request_id" \
|
||||
--now-unix-millis "$now_unix_millis"
|
||||
--runtime-id yggdrasil
|
||||
|
||||
if [ "$minimum_source_revision_required" = true ]; then
|
||||
require_root_owned_resolved_executable /usr/local/bin/idunn
|
||||
|
|
@ -234,18 +197,6 @@ require_signed_deployment_grant() {
|
|||
--daemon "$daemon_id" \
|
||||
--source-revision "$source_commit"
|
||||
fi
|
||||
|
||||
if [ "$requires_bifrost_authority" = true ]; then
|
||||
require_root_owned_resolved_executable /usr/local/bin/idunn
|
||||
/usr/local/bin/idunn validate-release-authority \
|
||||
--store /var/lib/gamecult/idunn/yggdrasil.keepalive.cc \
|
||||
--release-authority-store /srv/bifrost/state/repository-release-authority.cc \
|
||||
--repository-full-name "$repository_full_name" \
|
||||
--upstream-ref "$upstream_ref" \
|
||||
--source-revision "$source_commit" \
|
||||
--release-authority-id "$release_authority_id" \
|
||||
--release-authority-sha256 "$release_authority_sha256"
|
||||
fi
|
||||
}
|
||||
|
||||
manifest_group_exists() {
|
||||
|
|
@ -355,11 +306,11 @@ case "$action:$target" in
|
|||
run_manifest_while_holding_authority_locks "$manifest" restart
|
||||
exit $?
|
||||
;;
|
||||
deploy:voidbot|deploy:epiphany|deploy:epiphany-capstone-17|deploy:odin|deploy:codex-connector|deploy:ghostlight|deploy:heimdall|deploy:repixelizer|deploy:streampixels|deploy:bifrost-persona-feedback|deploy:gjallar)
|
||||
deploy:voidbot|deploy:epiphany|deploy:epiphany-capstone-17|deploy:odin|deploy:codex-connector|deploy:ghostlight|deploy:heimdall|deploy:repixelizer|deploy:streampixels|deploy:gjallar)
|
||||
manifest="/srv/odin/deploy-manifests/$target"
|
||||
require_root_owned_regular_path "$manifest"
|
||||
[ -x "$manifest" ] || { echo "root actuator manifest is not executable: $manifest" >&2; exit 1; }
|
||||
require_signed_deployment_grant "$@"
|
||||
require_deployment_admission "$@"
|
||||
export IDUNN_ACTUATOR=1
|
||||
export IDUNN_COMMAND_AUTHORITY=idunn-daemon
|
||||
export IDUNN_SOURCE_COMMIT="$source_commit"
|
||||
|
|
@ -368,7 +319,8 @@ case "$action:$target" in
|
|||
export BIFROST_RELEASE_AUTHORITY_ID="$release_authority_id"
|
||||
export BIFROST_RELEASE_AUTHORITY_SHA256="$release_authority_sha256"
|
||||
export IDUNN_DEPLOYMENT_REQUEST_ID="$deployment_request_id"
|
||||
export IDUNN_REQUIRES_BIFROST_AUTHORITY="$requires_bifrost_authority"
|
||||
# Installed manifests (Epiphany's) still require this to read false.
|
||||
export IDUNN_REQUIRES_BIFROST_AUTHORITY=false
|
||||
if [ "$manifest_takes_action" = true ]; then
|
||||
run_manifest_while_holding_authority_locks "$manifest" "$action"
|
||||
exit $?
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# Narrow root actuator grant for the Idunn Yggdrasil deployment/continuity path.
|
||||
# One line per live target. codex-connector and gjallar were retired 2026-09-06;
|
||||
# their grants are removed here first, because the privileged half of a retired
|
||||
# target must not outlive its consumer.
|
||||
# One line per live target. codex-connector and gjallar were retired 2026-09-06,
|
||||
# and bifrost-persona-feedback on 2026-09-30; their grants are removed here first,
|
||||
# because the privileged half of a retired target must not outlive its consumer.
|
||||
Cmnd_Alias IDUNN_YGGDRASIL_DEPLOY = \
|
||||
/usr/local/libexec/idunn-yggdrasil deploy odin *, \
|
||||
/usr/local/libexec/idunn-yggdrasil deploy ghostlight *, \
|
||||
|
|
@ -9,7 +9,6 @@ Cmnd_Alias IDUNN_YGGDRASIL_DEPLOY = \
|
|||
/usr/local/libexec/idunn-yggdrasil deploy heimdall *, \
|
||||
/usr/local/libexec/idunn-yggdrasil deploy epiphany-capstone-17 *, \
|
||||
/usr/local/libexec/idunn-yggdrasil deploy epiphany *, \
|
||||
/usr/local/libexec/idunn-yggdrasil deploy bifrost-persona-feedback *, \
|
||||
/usr/local/libexec/idunn-yggdrasil deploy repixelizer *, \
|
||||
/usr/local/libexec/idunn-yggdrasil deploy streampixels *
|
||||
Cmnd_Alias IDUNN_YGGDRASIL_RESTART = \
|
||||
|
|
|
|||
|
|
@ -55,7 +55,10 @@ A deployable repository publishes one strict declaration, normally
|
|||
- immutable artifact outputs, optionally pinned to an expected SHA-256;
|
||||
- a constrained launch contract;
|
||||
- one health-contract identity;
|
||||
- an optional typed state contract with state schema generation and slots;
|
||||
- an optional typed state contract with state schema generation and slots
|
||||
(kinds `cultcache-file`, `external-file` and `external-directory`; the
|
||||
external kinds are process-owned state an external runtime imposes, which
|
||||
Idunn preserves and fences but never interprets);
|
||||
- provided capabilities, dependency kinds, startup requirements, capacity, and
|
||||
conflicts.
|
||||
|
||||
|
|
@ -67,6 +70,52 @@ recipe fields.
|
|||
Stateless services declare no state block. Their operator binding therefore has
|
||||
no state root, state-transition policy, or process write-lease record.
|
||||
|
||||
## Foreign workloads: idunn-launch
|
||||
|
||||
An upstream daemon that cannot speak Idunn's runtime contract (Synapse, MAS)
|
||||
runs under `idunn-launch`, the unit's own process. Idunn's two-witness rule is
|
||||
unchanged: Idunn observes the unit, and the launcher, inside the unit and
|
||||
supervising the child, signs presence with the provider identity and the
|
||||
activation key.
|
||||
|
||||
- **Owner:** the launcher owns presence, config rendering and the child's
|
||||
lifetime inside the unit, and the route answers it signs for that child.
|
||||
Idunn owns admission, lease, route and continuity, as before.
|
||||
- **Inputs:** the runtime bundle, the activation credential and provider
|
||||
identity descriptors, the process write lease, the systemd credentials the
|
||||
binding names, and the child's health over TCP at the member bind.
|
||||
- **Outputs:** signed `gamecult.runtime_presence_health.v2` (Warming, Active
|
||||
and failed to Odin; route observations on the candidate endpoint), one child
|
||||
process, and a rendered config in the unit's private tmp that is never read
|
||||
back.
|
||||
- **Route:** Idunn owns both listeners of the route: the Expected route's
|
||||
stable endpoint (presence) and, when the binding has `[route.member]`, the
|
||||
member endpoint, rendered as a second server block in the one nginx fragment
|
||||
with a `limit_conn` on one per-route zone (`max_connections`, default 128), so
|
||||
members cannot exhaust the connections the presence block shares with them.
|
||||
The launcher binds the candidate endpoint and answers Idunn's challenge there,
|
||||
one connection at a time within a one-second deadline; any other first byte
|
||||
closes the connection. The answer never waits on the child: it reports the
|
||||
lifecycle's last probe, and a failed, missing or stale probe reads `degraded`.
|
||||
The child binds the member bind Idunn sets in
|
||||
`GAMECULT_IDUNN_MEMBER_BIND`, derived from the candidate port. Member bytes
|
||||
never touch the launcher.
|
||||
- **Lease:** with `write_lease_required` the launcher starts the child only
|
||||
after it holds the lease (bound to this Expected, this activation and one of
|
||||
its own Warming proofs) and keeps the lease store share-locked while the
|
||||
child lives. Losing the lease stops the child.
|
||||
- **Config:** one template from the release is rendered with the closed
|
||||
placeholder set `{{child.listen}}`, `{{child.listen_host}}`,
|
||||
`{{child.listen_port}}` (all three the member bind), `{{credential:NAME}}`,
|
||||
`{{credential_path:NAME}}` and `{{state_root}}`. Any other placeholder
|
||||
refuses to start.
|
||||
- **Forbidden:** member traffic through the launcher; the child binding the
|
||||
candidate endpoint; a child of a lease-requiring target running without a held lease; any presence not signed
|
||||
by both keys.
|
||||
- **State kind `external-directory`:** a process-owned directory imposed by an
|
||||
external runtime contract. Same writer, recovery and startup rules as
|
||||
`external-file`: it is lease-gated, preserved and fenced, never interpreted.
|
||||
|
||||
## Operator binding
|
||||
|
||||
The root-admitted operator binding supplies the physical policy a repository
|
||||
|
|
@ -173,9 +222,49 @@ health plus Ready correlation. It changes and observes route membership only
|
|||
afterward. Only after the proxy has adopted the candidate may Idunn clear the
|
||||
promotion fence and drain the incumbent.
|
||||
|
||||
The deployment brake gates changes to source, release, configuration, schema,
|
||||
unit, or authority binding. The lifecycle brake separately gates restart of an
|
||||
already-admitted incarnation. Neither target brake may gate Idunn itself or an
|
||||
A target has two slots: the incumbent (the admitted generation) and a
|
||||
transaction's candidate. Continuity owns the incumbent slot and keeps the
|
||||
admitted generation alive; it never stops. A live `DeploymentTransaction` owns
|
||||
its private candidate before Fencing and takes the target itself at Fencing (a
|
||||
stop-then-start rollout, at Starting) through one `take_target` CAS, so of two
|
||||
transactions entering from one view exactly one succeeds. A continuity
|
||||
restart is itself a transaction and takes the target the same way.
|
||||
|
||||
Phases from Warming through Committing carry deadlines, as do aborts and
|
||||
drains; an expired deadline never waits silently. A stateless deployment
|
||||
aborts to its incumbent. A deployment whose candidate was issued a write lease
|
||||
is left to the operator (`operator-required` incident, the second incident
|
||||
condition beside `continuity-exhausted`; its subject is the transaction id and
|
||||
it closes when the record ends or its step goes on). A continuity after
|
||||
Fencing retries its phase forward on a capped backoff and never stops its live
|
||||
candidate, since that is the only running copy of the admitted body; the
|
||||
deadline opens the same incident. The one exception is yielding: a continuity
|
||||
whose route proof has not succeeded gives way, through its own post-fencing
|
||||
abort, to a deployment the deployment stop admits and that awaits the
|
||||
handover; a continuity that has proven its route never yields. The
|
||||
`ContinuityExhausted` incident (six charged attempts in an hour,
|
||||
`restarts_page`) pages and gates nothing. When continuity next acts is derived
|
||||
only from its own record in `TargetSupervision` (schema v2): the attempt
|
||||
streak, the backoff anchor (written by the dead-and-free pass, the only
|
||||
writer), and the once-per-admission re-materialization bound. The workload
|
||||
driver alone answers whether a lost unit was a host reboot
|
||||
(`WorkloadPort::lost_to_host_reboot`). Operator-facing incidents and
|
||||
`idunn status` are projections that decide nothing. A boot lift returns a
|
||||
continuity abort left by an older Idunn, one that had not stopped its running
|
||||
candidate, to forward retry; if it cannot decide, every continuity abort is
|
||||
held until a lift succeeds. It is a migration with a deletion line (follow-up
|
||||
`c2c-delete-boot-lift`). Cut line: cuts 3 and 4 of `reincarnation-map.md`
|
||||
still owe dependency failures not growing the streak and the dead-generation
|
||||
route refresh.
|
||||
|
||||
The deployment brake is a stop, not a per-deploy gate: it gates changes to
|
||||
source, release, configuration, schema, unit, or authority binding only while
|
||||
engaged. Its one reader and writer is `src/brake.rs` (`idunn.deployment_stop.v1`:
|
||||
engaged or clear, who, why, when, sequence). An absent or clear store admits;
|
||||
a store that is present but unreadable, foreign, for another runtime, a symlink
|
||||
or owned by another uid fails closed. Nothing is signed, bound to a release or
|
||||
consumed, and one clear admits every deploy until it is engaged again. The
|
||||
lifecycle brake separately gates restart of an already-admitted incarnation. Neither target brake may gate Idunn itself or an
|
||||
unrelated service.
|
||||
|
||||
If Odin is unavailable, Idunn preserves existing routes and continuity for
|
||||
|
|
|
|||
394
docs/deploy-provenance-map.md
Normal file
394
docs/deploy-provenance-map.md
Normal file
|
|
@ -0,0 +1,394 @@
|
|||
# Deployment provenance: map
|
||||
|
||||
Status: map, Imagination, 2026-10-23 (campaign `idunn-deploy-provenance`,
|
||||
session `self-2026-10-23-eureka-b`). The ends are in
|
||||
[`deploy-provenance-target.md`](deploy-provenance-target.md). Questions,
|
||||
rulings, cut specs and follow-ups live in Eureka's mind; this page holds what
|
||||
has no kind: body facts, rationale and sequencing. Line numbers in T1-T12 are
|
||||
at `d74ac8d` (origin/main on 2026-10-23); T4, T9 and T13-T16 are at `fd040f0`,
|
||||
the head of Hands attempt 1, which revision 2 of the spec builds on.
|
||||
|
||||
## Cut `tree-executable-members`
|
||||
|
||||
Spec `idunn-deploy-provenance:cut_spec:cut-tree-executable-members.r2`
|
||||
(revision 2, after Soul pass 1 found the real interpreter tree refused). It
|
||||
serves `motion-pipeline:follow_up:idunn-tree-executable-members` (Vili's
|
||||
admission waits on it) and will serve gamecult-chat C3 too: the presence
|
||||
launcher runs a child such as `upstream/bin/daemon` from inside a tree artifact
|
||||
(README, launcher fragment), and Synapse needs a Python interpreter there.
|
||||
|
||||
### Body facts
|
||||
|
||||
- T1. `ArtifactOutput` (`src/deployment.rs:106-117`) carries one `executable`
|
||||
bool. `ArtifactReceipt` (`src/deployment_plan.rs:731-737`) copies it and
|
||||
`validate` (`:740-760`) requires it to equal the declaration.
|
||||
- T2. `harden_installed_release` (`src/drivers.rs:6848-6855`) builds its
|
||||
executable set from `root.join(receipt.destination)` only. `harden_root_tree`
|
||||
(`:6858-6892`) seals every directory 0555 and every file 0444 unless that
|
||||
exact path is in the set. A directory artifact with `executable = true` is
|
||||
therefore silently meaningless today: its files are all 0444. Nothing refuses
|
||||
it. No live recipe declares one (CodexConnector, Ghostlight, Heimdall, Odin,
|
||||
StreamPixels service and web surveyed): `executable = true` is on files only.
|
||||
- T3. A tree output is copied by `copy_artifact`/`copy_tree`
|
||||
(`src/drivers.rs:8053-8126`), which recreate symlinks verbatim. The digest
|
||||
(`digest_tree`, `:8142-8175`) covers names, file bytes and link targets, not
|
||||
mode bits. So runner or git exec bits never reach the release: harden is the
|
||||
only mode authority.
|
||||
- T4. `validate_release_symlink` (`src/drivers.rs:6897-6916`) keeps a symlink
|
||||
only if its target is relative with no `..`. It runs only inside
|
||||
`harden_root_tree`, at install, so a tree with a `..` or absolute link copies,
|
||||
digests and seals, then fails at `install_release` (Soul s1: copying and
|
||||
digesting the real tree twice took 160 s before harden refused it). The rule
|
||||
is sound but too narrow: it refuses the real interpreter tree (T13). The
|
||||
frozen-source path has a physical resolver (`resolve_frozen_source_symlink`,
|
||||
`:6617`) that walks the filesystem with a 40-link budget and a memo. Its
|
||||
first version was exponential (Soul S4-1, recorded in `d562a12`), and it needs
|
||||
the tree on disk to answer. Revision 2 does not reuse it (T14).
|
||||
- T5. The sealed release id hashes `(schema, plan_id, artifacts,
|
||||
external_inputs)` (`src/deployment_plan.rs:1000-1011`), and `plan_id` hashes
|
||||
the whole plan including `recipe_blob`. A changed member list is therefore
|
||||
already a new plan and a new release. The receipt must still carry the list,
|
||||
because `harden_installed_release` reads receipts only; its two callers
|
||||
(`install_release` `:2788`, `validate_installed_release` `:2822`) re-harden
|
||||
every installed release on each observation.
|
||||
- T6. Encoding is `rmp_serde::to_vec`, positional. A trailing field with
|
||||
`#[serde(default, skip_serializing_if = "BTreeSet::is_empty")]` keeps every
|
||||
existing receipt's bytes and release id: the plan's `deadlines` field
|
||||
(`:574-577`) is the precedent.
|
||||
- T7. `digest_artifact` and `digest_tree` (`:8128-8175`) `fs::read` each file
|
||||
whole. A torch CPU tree's largest file (`libtorch_cpu.so`, hundreds of MB)
|
||||
is held in Idunn's memory three times per install (seal, install, every
|
||||
validate). `hash_frozen_source_tree` (`:6788-6840`) already streams in 64 KiB
|
||||
chunks (S9). The cut collapses both onto one streaming helper.
|
||||
- T8. `require_leaf_path` (`src/deployment.rs:1951-1958`) only keeps the
|
||||
artifact destination a single filename. Members are paths inside that
|
||||
destination, so it does not change. `require_relative_path` (`:1936-1949`)
|
||||
with `allow_current = false` already refuses absolute, `..`, root and prefix
|
||||
components.
|
||||
- T9. The host actuator re-spells the seal (`src/host.rs:1050-1086`): copy,
|
||||
mode check, digest, pinned-digest check and receipt, the same steps as
|
||||
`collect_artifact` (`src/drivers.rs:2305-2326`). Soul s1: deleting its mode
|
||||
check survives every test. Revision 2 collapses both onto one `seal_artifact`
|
||||
in `drivers.rs`. Each caller keeps only how it finds the source path.
|
||||
- T10. The transient unit runs with `ProtectSystem=strict` and the release in
|
||||
`ReadOnlyPaths` (`src/drivers.rs:3675-3698`), `/tmp` mounted `noexec`, and no
|
||||
`MemoryDenyWriteExecute`. Executing a 0555 file from the release works.
|
||||
Shared libraries need no exec bit.
|
||||
- T11. Live body on Yggdrasil: `/usr/local/bin/idunn` sha256 `44cf8a14...`,
|
||||
installed 2026-10-20 02:34 CEST, before `6e2a31c` (02:40 CEST). `src/` is
|
||||
identical between `8760a4f` and `6e2a31c`, so the live code is the
|
||||
reincarnation c2 merge. `origin/main` is 26 commits ahead (+4194/-112 in
|
||||
`src/`: the presence launcher and member route). Installing this cut ships all
|
||||
of them. Idunn's own body changes only through gamecult-ops
|
||||
`scripts/install-idunn-yggdrasil-release.sh` (staged binaries, unit,
|
||||
bindings). The stage at `/home/gamecultadmin/idunn-stage` is stale (2026-09-11).
|
||||
- T13. Link census of the real interpreter tree (Yggdrasil probe of
|
||||
`cpython-3.12.15+20261021-x86_64-unknown-linux-gnu-install_only.tar.gz`,
|
||||
sha256 `5ff9f060...77191124`; torch adds no links, as Soul s1 counted 1,049
|
||||
after pip): 1,049 links. 0 are absolute, 749 have no `..`, and 300 have all
|
||||
their `..` components before the first name (every one is
|
||||
`share/terminfo/<c>/<name> -> ../<c2>/<name2>`, one `..` each). 0 have a `..`
|
||||
after a name, 0 climb above the tree, 0 dangle and 0 resolve outside it.
|
||||
Blender 5.2.2 has 79 links, none absolute or with `..` (Soul s1).
|
||||
- T14. Lexically normalising `<link's directory>/<target>` does not check
|
||||
containment. The kernel resolves a target one component at a time, so a `..`
|
||||
after a symlinked name climbs from where that link points, not from where its
|
||||
name sits. Probe on Yggdrasil: `root/a/s -> ..` and `root/a/L -> s/../x`
|
||||
normalise to `root` and `root/a/x`, both inside, yet `L` reads the file `x`
|
||||
beside `root`. `root/a/d -> .` with `root/a/M -> d/../../x` escapes the same
|
||||
way. Idunn has met this before: verify-transaction-cut F5 (`D -> .`,
|
||||
`L -> D/D/../../../etc/passwd` passed a lexical check).
|
||||
The sound lexical rule constrains the target's shape: all its `..` components
|
||||
come before its first name, and they number no more than the link's depth
|
||||
below the artifact's top. The climb then starts from the link's own
|
||||
directory, which the walk reached through real directories only (copy and
|
||||
digest never descend through a link), so it lands on a real ancestor inside
|
||||
the artifact. After that the target only descends, and any link it passes
|
||||
through obeys the same rule. By induction on the kernel's hops (capped at 40,
|
||||
then ELOOP), every hop stays inside. The check is one pass over the target's
|
||||
components with no filesystem reads, so its cost is linear in the tree and no
|
||||
resolver is needed. The shapes it refuses (`s/../x`) occur in neither real
|
||||
tree (T13).
|
||||
Prior art, from documentation as known to Imagination (high confidence for Go
|
||||
and securejoin, medium for the tar details):
|
||||
- Go `path/filepath.IsLocal` (1.20) is lexical and says so: it does not account
|
||||
for symlinks. It would admit `s/../x`. Used alone, it is the T14 bug.
|
||||
- Go `os.Root` (1.24) and `cyphar/filepath-securejoin` (used by runc,
|
||||
containerd and umoci) resolve each component inside the root when a path is
|
||||
opened, refusing or clamping an escape. They guard one later access, need
|
||||
the tree on disk, and check again on every open. Idunn instead certifies a
|
||||
sealed tree once, so that the kernel's own resolution stays inside on every
|
||||
later access.
|
||||
- Zip-slip defences and archive extractors check entry names lexically and
|
||||
protect against link targets by refusing to write through links
|
||||
(libarchive's `ARCHIVE_EXTRACT_SECURE_SYMLINKS`, and `SECURE_NODOTDOT` for
|
||||
names; GNU tar creates such links last). They protect the extraction, not
|
||||
what a kept link points at afterwards.
|
||||
- OCI layer extraction keeps link targets verbatim, absolute ones included,
|
||||
because the runtime resolves them inside the container's root. Idunn runs a
|
||||
release in the host's mount namespace, so absolute targets stay refused.
|
||||
- Nix keeps link targets verbatim and tracks dependencies by scanning for
|
||||
references. The whole store is its root, so it has no per-artifact
|
||||
containment to compare with.
|
||||
None of these is a static per-link rule for a tree that runs unconfined. The
|
||||
T14 shape rule fits that case.
|
||||
- T15. Host targets: `idunn-host` observes, starts and stops processes on
|
||||
Windows only (the non-Windows `platform` module in `src/host.rs` bails), and
|
||||
`host.rs` sets no file modes. On Windows a file runs by its type, not by a
|
||||
mode bit, so `executable` and `executable_members` have nothing to set there.
|
||||
The file `executable` flag has always been carried and never applied on host
|
||||
targets. A recipe does not know which workload kind its binding will choose.
|
||||
- T16. The release root holds only artifact destinations. Runner workspaces,
|
||||
where external inputs land, are removed before `SealedRelease::new`
|
||||
(`src/drivers.rs:2421-2423`). `install_release` (`:2781-2788`) and
|
||||
`validate_installed_release` (`:2815-2822`) digest every receipt's
|
||||
destination before calling harden. So a rule in the digest walk runs at seal,
|
||||
at install and at every validate, over every link harden would see.
|
||||
- T12. Disk: `/` on Yggdrasil has 1.3 TB free. A tree release costs about three
|
||||
copies (runner workspace, staging, installed). Nothing collects old releases or
|
||||
staging (`ghostlight-verse:follow_up:idunn-generation-retention`), so every
|
||||
Vili deploy leaves about 7 GiB behind.
|
||||
|
||||
### Contract, corrected against the body
|
||||
|
||||
- Any directory artifact may declare `executable_members` (a set of relative
|
||||
paths). The follow-up limited it to runner-output; that limit protects no
|
||||
invariant, and the launcher's worktree or fetched trees need the same feature.
|
||||
- `executable = true` names a file artifact. On a directory output the release
|
||||
is refused, and `executable_members` on a file output is refused (T2).
|
||||
- A member is refused at recipe parse if it is empty, absolute, or holds `.`,
|
||||
`..`, a root or a prefix. At seal it is refused if it is missing, is a
|
||||
symlink, passes through a symlink or non-directory, or is not a regular file.
|
||||
Refusal happens before `SealedRelease::new`, so no sealed release can name a
|
||||
member it cannot seal.
|
||||
- Harden seals each member 0555 and fails if any declared executable path was
|
||||
not visited as a regular file. "Every declared executable is sealed 0555, or
|
||||
the install fails" holds structurally, not by a lookup that can miss.
|
||||
- Symlinks: a link in a tree artifact is kept when its target is relative, its
|
||||
`..` components all come before its first name, and they number no more than
|
||||
the link's depth below the artifact's top (T14). One function,
|
||||
`check_release_link`, decides this. Only the digest walk calls it, so a bad
|
||||
link refuses the release at seal, before `SealedRelease::new`, and again at
|
||||
install and validate (T16). Harden no longer judges links. The climb is
|
||||
bounded by the artifact's top, not the release root: no live release has a
|
||||
`..` link (the old rule refused them all), so nothing installed is newly
|
||||
refused, and each artifact's digest describes a closed tree.
|
||||
- Host targets carry `executable` and `executable_members` in the receipt,
|
||||
shape-checked at seal by the shared `seal_artifact`, and do not apply them
|
||||
(T15). Refusing members there would make a recipe's validity depend on its
|
||||
binding.
|
||||
- Vili's trees must
|
||||
not come from `python -m venv`: its `bin/python` is an absolute link and its
|
||||
`pyvenv.cfg` names the build workspace. Install packages into the relocatable
|
||||
interpreter tree instead (`motion-pipeline:follow_up:vili-relocatable-python-tree`).
|
||||
|
||||
### Authority
|
||||
|
||||
Owner: the seal (`seal_artifact`, called by `collect_artifact` for systemd
|
||||
targets and by `materialize` in `host.rs` for host targets) decides
|
||||
membership from the recipe and writes it into the receipt. Harden enforces the
|
||||
receipt and decides nothing. Runner exec bits and git `100755` modes are dead
|
||||
inputs for release modes. Link containment belongs to `check_release_link`,
|
||||
which only `digest_tree` calls. `validate_release_symlink` is deleted, and
|
||||
harden's symlink arm leaves links as they are.
|
||||
|
||||
### Acceptance sizing
|
||||
|
||||
The big-tree job (spec `verification.operator`) needs about 12 GiB of scratch
|
||||
on Yggdrasil: a 3.5 GiB tree, its staging copy, downloads and pip cache. It
|
||||
removes it at the end. Revision 2 runs it on the trees as shipped, with the 300
|
||||
terminfo links (T13). If a tree holds a link the T14 rule refuses, the job
|
||||
prints it and stops, and the link is reported to Self, not deleted.
|
||||
|
||||
## Cut `isolation-credential-evidence`
|
||||
|
||||
Spec `idunn-deploy-provenance:cut_spec:cut-cut-isolation-credential-evidence.r1`,
|
||||
at `112a7a5` (origin/main, live on Yggdrasil since 2026-10-24 11:29 UTC). It
|
||||
serves Vili's admission (`motion-pipeline:cut_report:cut-vili-admission.h4`,
|
||||
first deviation): every `idunn up vili` is refused before fencing.
|
||||
|
||||
### Body facts (probed read-only on Yggdrasil, 2026-10-24)
|
||||
|
||||
- I1. `idunn up vili` runs 6 and 7 failed with
|
||||
`systemd workload isolation properties differ from the Idunn contract`, the
|
||||
message of `observe_unit` (`src/drivers.rs:2975-3005`). Of its conjuncts,
|
||||
only one differs on the failed Vili unit: `LoadCredential`. Idunn expects
|
||||
`""` because Vili has no service credential besides the presence identity
|
||||
(`service_credential_names` empty); systemd shows `[unprintable]`. Every
|
||||
other conjunct matches, InaccessiblePaths included (one value across all
|
||||
five idunn-vili/odin units).
|
||||
- I2. systemd 257.13 `systemctl show -p LoadCredential` prints
|
||||
`LoadCredential=[unprintable]` for every unit, whatever the credentials:
|
||||
`ssh.service` and `systemd-journald.service` (not DynamicUser, zero
|
||||
LoadCredential), the four failed `idunn-vili-*` units (zero), and the Odin
|
||||
unit (one). The display form carries no information, so the conjunct was
|
||||
vacuous whenever credentials existed and always false when none did. Odin
|
||||
and StreamPixels passed only because each carries a regular credential.
|
||||
- I3. The exact value is readable: D-Bus
|
||||
`org.freedesktop.systemd1.Service.LoadCredential` (`busctl get-property`)
|
||||
answers `a(ss) 0` for the Vili units and ssh, `a(ss) 1
|
||||
"ODIN_TOPOLOGY_IDENTITY" "<path>"` for Odin.
|
||||
- I4. What the workload can actually read is the delivered directory. On the
|
||||
host, `/run/credentials/` holds a directory only for units with credentials
|
||||
(getty, journald, Odin, StreamPixels service and web). Inside Odin's MainPID
|
||||
root, `/run/credentials/<unit>/` holds exactly `ODIN_TOPOLOGY_IDENTITY`
|
||||
(`-r--r-----+ root root`). The presence identity is a parent-only `OpenFile`
|
||||
descriptor, never a credential.
|
||||
- I5. The rule arrived whole in `1fc0dad` (2026-09-03, "Rebuild Idunn
|
||||
deployment transaction authority"), with its twin clause in
|
||||
`validate_launch_observation` (`:3468-3472`, `:3513-3516`); no later commit
|
||||
touched it. No ruling or finding in any campaign concerns workload isolation
|
||||
verification; gamecult-chat's foreign-workloads cut touched
|
||||
`CREDENTIALS_DIRECTORY` reservation, not this check.
|
||||
- I6. `SystemdWorkloadObservation` is persisted by previous Idunns in both
|
||||
MessagePack encodings, compact (positional) included
|
||||
(`persisted_systemd_observations_decode_as_the_systemd_variant`,
|
||||
`:9915-9930`), so deleting the `load_credential` field shifts every later
|
||||
slot of a live incumbent's record. The field is also in the drift list
|
||||
(`:2462`) that `stop` (`:4064-4070`) uses to refuse stopping a changed
|
||||
workload: changing what the field holds without leaving that list would make
|
||||
the next Idunn refuse to stop the live Odin and StreamPixels incumbents.
|
||||
- I7. Fixtures: the verbatim 257.13 `systemctl show` output of base's 27
|
||||
properties for a Vili unit (presence identity only) and the Odin unit
|
||||
(presence identity plus one service credential), captured 2026-10-24 into
|
||||
this Imagination's scratchpad `fixtures/`.
|
||||
- I8. The new rule holds for every live systemd incumbent: inside each
|
||||
MainPID root the delivered entries equal its binding's `secret_files` keys
|
||||
minus `GAMECULT_RUNTIME_PRESENCE_IDENTITY` (Odin 1, StreamPixels service 11,
|
||||
StreamPixels web 1; key names compared, no values read). An Idunn restart onto
|
||||
this cut re-observes them (`:3955-3970`) and passes.
|
||||
|
||||
### Decision
|
||||
|
||||
A property systemd cannot display does not belong in the comparison. The
|
||||
invariant it reached for, that the workload holds exactly the credentials Idunn
|
||||
delivered, is observed where it is decided: the entries of
|
||||
`/run/credentials/<unit>` in the MainPID's root equal Idunn's service
|
||||
credential names, absent directory equal to none. That covers
|
||||
`LoadCredential`, `SetCredential`, `ImportCredential` and drop-ins alike, and
|
||||
needs no new program. The D-Bus read (I3) is exact but sees only one of those
|
||||
properties and adds a second systemd client and parser; rejected.
|
||||
|
||||
### Authority
|
||||
|
||||
Owner: `observe_unit` decides the credential rule, from the delivered
|
||||
directory. `LoadCredential`'s display form is no longer read or compared; the
|
||||
`load_credential` field is a retired wire slot, written empty, out of the drift
|
||||
list, kept only so persisted records decode. Forbidden writers: the
|
||||
`observe_unit` conjunct and the `validate_launch_observation` clause. Dropping
|
||||
the slot waits on the next schema revision of the observation.
|
||||
|
||||
### Deploy
|
||||
|
||||
Idunn's body changes only through gamecult-ops
|
||||
`scripts/install-idunn-yggdrasil-release.sh` (stage `idunn` only; unit and
|
||||
bindings unchanged). After it, Vili's Hands reruns from step 4 of
|
||||
`motion-pipeline:cut_report:cut-vili-admission.h4`'s undone list.
|
||||
|
||||
## Cut `runner-failure-evidence`
|
||||
|
||||
Spec `idunn-deploy-provenance:cut_spec:cut-runner-failure-evidence.r1`, at
|
||||
`804ed75` (origin/main, live on Yggdrasil). It serves the StreamPixels-area
|
||||
Self: `idunn up vili` (command `up-4866467c-ee98-4a2b-b4a9-a008d7c44acd`,
|
||||
transaction `tx-a04b68c7-c742-4d7d-8220-8aa08b9febae`, 2026-10-25 00:50-00:55
|
||||
UTC) failed in Sealing with `Docker runner exited with exit status: 1: ` and
|
||||
nobody could tell which recipe step failed.
|
||||
|
||||
### Body facts (source read at `804ed75`; host read-only, 2026-10-25)
|
||||
|
||||
- R1. The runner is `docker run --rm` (`src/drivers.rs:2088-2090`), launched by
|
||||
`DockerRunnerDriver::docker` (`:2173-2195`) with stdin null and
|
||||
`Command::output()`: stdout and stderr are each buffered whole in the
|
||||
daemon's memory, unbounded. On a non-zero exit the error is
|
||||
`"Docker runner exited with {status}: {stderr}"`; stdout is dropped. The
|
||||
container is gone the moment it exits, so `docker ps -a` and `docker logs`
|
||||
hold nothing.
|
||||
- R2. The empty message: the failing process wrote nothing to stderr, and
|
||||
stdout, the only stream that carried its report, is discarded at
|
||||
`:2182-2193`. The same interpolation echoes whatever a step does write to
|
||||
stderr into `last_error`, `pre_fencing_abort.error` and
|
||||
`completion.error` (`bounded_detail`, 2048 bytes), and from there into
|
||||
`idunn up`'s `deployment failed:` line and `idunn status`.
|
||||
- R3. No step identity reaches the error. `materialize` (`:2368-2391`) loops
|
||||
over `declaration.steps` and calls `run_in_workspace` with argv only;
|
||||
`step.id` is used only in the environment precondition. External inputs
|
||||
(`curl` in the runner, `:2247-2258`) fail through the same helper with the
|
||||
same anonymous message.
|
||||
- R4. Which step failed on 2026-10-25, inferred from the failed transaction's
|
||||
staging root, which survives (926 MiB,
|
||||
`/var/lib/gamecult/idunn/staging/tx-a04b68c7-.../`): in `.runner-node-build`,
|
||||
`node_modules/.package-lock.json` was written at 00:51:24 (so
|
||||
`install-vili-dependencies`, `npm ci`, finished), nothing outside
|
||||
`node_modules` is newer, and `release/vili` (the next step's output) is
|
||||
absent; the python workspace holds only fetched inputs. The failing step was
|
||||
`test-daemon` (`node --test ... scripts/test-vili-daemon.mjs`), whose test
|
||||
reporter writes to stdout: exactly the stream R1 discards. Inference from
|
||||
residue, not from output.
|
||||
- R5. The failure path: `resume_one_transaction` (`src/control_plane.rs:6291-6320`)
|
||||
sends a pre-fence error to `begin_pre_fencing_abort` (`:9916-9935`), which
|
||||
persists `bounded_detail(format!("{error:#}"))`. Nothing reaches the journal
|
||||
on this path: the idunn-yggdrasil journal for 00:45-01:10 holds only
|
||||
provider-exclusion lines.
|
||||
- R6. Stores. `control.cc` (root 0640) holds commands, transactions
|
||||
(`idunn.deployment_transaction.v4`, keys 0-36), generations, supervision and
|
||||
expiry requests; `ControlSnapshot::read` refuses any other type
|
||||
(`:3420`). Records must re-encode byte-exactly (`decode_record`,
|
||||
`:3717-3727`), so a new transaction field is a schema bump and a v4 lift
|
||||
(`:4100-4110`). Terminal transactions move verbatim to `history.cc`
|
||||
(`:4366-4396`; 39.8 MB on the host), which status reads for `--command`.
|
||||
`incidents.cc` lives in the published projection directory and by its own
|
||||
doc (`src/incident.rs:1-11`) carries no free text and is never `control.cc`,
|
||||
because a type added there stops continuity for every target on a rollback.
|
||||
- R7. The daemon runs as root (`User=root`). Runner secrets reach the
|
||||
container as read-only file mounts named by env (`:2041-2047`), so any text
|
||||
a step derives from them can land in its output.
|
||||
- R8. Staging residue: 475 transaction roots, 55 GiB under
|
||||
`/var/lib/gamecult/idunn/staging`; a failed Sealing leaves its workspaces
|
||||
because `materialize` returns before `remove_tree_inside`.
|
||||
|
||||
### Decision
|
||||
|
||||
The failure becomes a typed value at the runner, `RunnerStepFailure`: the
|
||||
subject (recipe step or external input, by id), the runner id, the exit code or
|
||||
signal, and the last 8 KiB of stdout and of stderr with each stream's total
|
||||
byte count. Its `Display` carries the subject and status, never output.
|
||||
Capture streams both pipes through bounded tails, so a verbose build no longer
|
||||
buffers whole in the daemon.
|
||||
|
||||
The tail lives in a new private store, `runner-evidence.cc`, beside
|
||||
`control.cc` (mode 0600), one record per transaction
|
||||
(`idunn.runner_failure.v1`), newest 32 kept by the writer. Not in the
|
||||
transaction record: that is a v5 bump whose rollback makes every older Idunn
|
||||
refuse its control store (R6), the scar `incident.rs` already names. Not in
|
||||
`incidents.cc`: published, and free text is barred there. Not in the journal:
|
||||
journal readers are a wider set than root, so the journal line names the
|
||||
transaction, the subject, the status and the store, and no output.
|
||||
|
||||
No redaction. Matching secret bytes would make the daemon read every secret,
|
||||
and still misses any encoding, prefix or transform of one; a claimed redaction
|
||||
that leaks is worse than a stated rule. The rule instead is the reader set: the
|
||||
store's only readers (root) can already read every secret the runner was
|
||||
given.
|
||||
|
||||
Keeping the failed container (dropping `--rm`) is worse: its output then lives
|
||||
in Docker's json-file log, unbounded, untyped, outside Idunn's state and
|
||||
readable by the docker group; containers accumulate with no owner to reap them;
|
||||
`docker system prune` erases the evidence; and nothing ties a container to the
|
||||
transaction in `idunn status`.
|
||||
|
||||
### Authority
|
||||
|
||||
Owner: the runner decides what failed and keeps the bounded tail; the engine's
|
||||
pre-fence failure path is the one writer of `runner-evidence.cc`.
|
||||
`last_error`, `pre_fencing_abort.error` and `completion.error` keep only the
|
||||
failure's `Display`. Forbidden writers: the `stderr` interpolation in
|
||||
`DockerRunnerDriver::docker`. Retention is by count inside the writer, so the
|
||||
store needs no external owner; staging roots (R8) stay with
|
||||
`ghostlight-verse:follow_up:idunn-generation-retention`.
|
||||
|
||||
### Deploy
|
||||
|
||||
Idunn's body changes only through gamecult-ops
|
||||
`scripts/install-idunn-yggdrasil-release.sh` (stage `idunn` only; unit and
|
||||
bindings unchanged). The next failed Sealing then shows its step and tail on
|
||||
`sudo idunn status --command <id>`.
|
||||
75
docs/deploy-provenance-target.md
Normal file
75
docs/deploy-provenance-target.md
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
# Deployment provenance instead of an approval gate: target
|
||||
|
||||
Status: target, Self, 2026-10-19 (campaign `idunn-deploy-provenance` in
|
||||
Eureka's mind). It states the ends. The map will live in
|
||||
`deploy-provenance-map.md`.
|
||||
|
||||
Live as of cut `brake-is-a-stop` (2026-11-13): the deployment brake is the
|
||||
stop of invariant 3 and the signed per-deploy release is gone. `src/brake.rs`
|
||||
reads and writes it; `idunn-provision deployment-brake-engage|clear|status`
|
||||
take only `--store`, `--runtime-id` and `--reason`; the writer's login name and
|
||||
the verb's clock are recorded; an absent or clear store admits. The
|
||||
transaction stamps `deployment_admitted_at_unix_millis` (record v6). Still
|
||||
planned: the typed authority provenance on the transaction (invariant 2), and a
|
||||
steward-scoped store.
|
||||
|
||||
## Why
|
||||
|
||||
Operator, 2026-10-19:
|
||||
|
||||
> The rule about the deployment brake was instituted by a cautious agent, but
|
||||
> it doesn't actually suit how GameCult operates. The whole point of Eureka is
|
||||
> to give the Mind, Persona and Self agency over its own Body. It's requiring
|
||||
> my approval for every deployment that violates our principles. Yes, we
|
||||
> certainly want to know which agent approved the deployment and why (this is
|
||||
> the authority provenance which we might want to add to the transaction), but
|
||||
> that's not the same as gating every deploy on my approval.
|
||||
|
||||
On the same day a StreamPixels deploy (tx-b40c7240) waited on the brake
|
||||
because only the root-held operator identity could release it. The agent
|
||||
running the deploy could neither proceed nor record why it should.
|
||||
|
||||
`F:\Projects\CLAUDE.md` says a deployment brake *may* gate artifact, revision,
|
||||
configuration, schema, unit or authority changes. It does not say that
|
||||
operator approval is the gate.
|
||||
|
||||
## Invariants
|
||||
|
||||
1. **No deployment waits on a per-transaction operator approval.** An agent
|
||||
acting for the Mind, Persona or Self may admit a deployment of a target it
|
||||
is authorized for.
|
||||
2. **Every transaction carries typed authority provenance.** That is: who
|
||||
approved it (agent or instance, and faculty), under which session or
|
||||
ruling, and why. It is written before Fencing, it is immutable, and
|
||||
`idunn status` and history show it. It is part of the transaction
|
||||
document, not a log line.
|
||||
3. **A brake that survives is a stop, not a gate.** The operator or an
|
||||
authorized agent can halt deployment of a target, and the halt is itself
|
||||
provenance-bearing. The deployment brake and the continuity lifecycle brake
|
||||
stay separate (`F:\Projects\CLAUDE.md` Infrastructure). Neither gates Idunn
|
||||
itself, and the deployment brake never gates continuity.
|
||||
4. **Existing invariants hold.** These are `idunn-reincarnation` target r2,
|
||||
`route-continuity-target.md` 1-7 and `authority-map.md`.
|
||||
|
||||
## Scope
|
||||
|
||||
In:
|
||||
|
||||
- The deployment-brake release path in Idunn and `idunn-provision`.
|
||||
- The provenance record on the transaction.
|
||||
- `idunn status` and history projection of the provenance.
|
||||
- The gamecult-ops runbooks that describe a per-transaction operator release
|
||||
(`heimdall-discord-launch.md`, `idunn-host-raven.md`, the StreamPixels
|
||||
runbook).
|
||||
|
||||
- A static release (a sealed tree published by repointing a link, with no
|
||||
workload): its transaction records `flip` (release, previous release, the
|
||||
sha256 of the carried per-site file) beside the brake bytes. See
|
||||
`deployment-authority.md`, "Static release".
|
||||
|
||||
Out:
|
||||
|
||||
- `idunn-reincarnation`. This campaign is mapped after that campaign's cut 2
|
||||
merges, and coordinates with it rather than bundling.
|
||||
- Harness permission rules on the workstation, which are the operator's.
|
||||
- Heimdall's onboarding failures.
|
||||
|
|
@ -48,8 +48,8 @@ and health. CultMesh carries typed projections between them.
|
|||
scaling, restart, host reboot recovery, and crash recovery use the same
|
||||
source-to-recipe-to-artifact provenance and the same distinct write-lease and
|
||||
route-membership primitives.
|
||||
Deployment may change that chain only while the deployment brake admits the
|
||||
exact transaction. Continuity may restart only the already-admitted chain and
|
||||
Deployment may change that chain only while the deployment stop admits it
|
||||
(absent or clear; an engaged stop halts). Continuity may restart only the already-admitted chain and
|
||||
is governed by a separate lifecycle brake.
|
||||
- **Deletion line:** hard-coded target copies, raw `deploy_command` and
|
||||
`restart_command` state, `sh -c` actuation, root Git inspection, staged
|
||||
|
|
@ -135,6 +135,13 @@ deterministically lowers the constrained launch contract through the configured
|
|||
workload driver; repository-owned raw unit or container-runtime text is not an
|
||||
input.
|
||||
|
||||
For systemd `DynamicUser` workloads, systemd's reported primary user and group
|
||||
names are allocated runtime identities. When a binding declares no fixed state
|
||||
group, Idunn validates that the process primary UID and GID match instead of
|
||||
requiring an empty `Group` property. A declared state group remains an exact
|
||||
name and numeric GID contract because it grants the process its bounded write
|
||||
lease.
|
||||
|
||||
## Operator binding
|
||||
|
||||
An operator binding is static root-admitted configuration, not observed state.
|
||||
|
|
@ -164,6 +171,96 @@ different semantic capability.
|
|||
`gamecult-ops` publishes these fleet bindings and provisions host substrate. It
|
||||
does not own application build or deployment programs.
|
||||
|
||||
## Static release
|
||||
|
||||
A target whose product is a sealed tree, with no process to run (a built web
|
||||
front end), declares `[static_release] artifact = "<id>"` in place of
|
||||
`[service]`, and its binding declares `driver = "static-release"` with a
|
||||
`release_root`, a `current_link` and one `[workload.carry]`. It is the same
|
||||
transaction as any deployment, in three phases: Sealing (source, build, seal,
|
||||
install: unchanged), Starting (the one external change, behind the deployment
|
||||
brake) and Complete. It never enters Warming through Committing, has no
|
||||
Expected projection, no activation and no workload, publishes nothing to
|
||||
Odin, and its Complete creates no admitted generation, so continuity never
|
||||
supervises it. It declares no state, capabilities, dependencies, conflicts,
|
||||
route or write lease, and its rollout is `candidate-then-promote` with at least
|
||||
two retained releases.
|
||||
|
||||
What Starting does, in three durable steps recorded as `flip` on the
|
||||
transaction: the release to name and the release the link names now are
|
||||
written first; then Idunn copies the operator's carry source into
|
||||
`<release_root>/<sealed release id>/<carry.destination>` (mode 0444) and
|
||||
renames a fresh symlink over `current_link`; then it observes the link naming
|
||||
the release and completes. A failure before the rename ends the transaction
|
||||
through the ordinary pre-fencing abort with `current_link` untouched. A crash
|
||||
resumes at the recorded intent and repeats the flip, which is harmless.
|
||||
|
||||
The carry is a copy of one per-site file (for example `config.json`) taken at
|
||||
flip time. It sits beside the artifacts, never inside one, so it is outside
|
||||
every sealed digest: changing the site file changes no release id, and the
|
||||
sha256 of the bytes carried is provenance on the transaction
|
||||
(`flip.carried_sha256`), beside the brake bytes and the requester. The
|
||||
destination is a single leaf name that no artifact uses. The web server reads
|
||||
`current_link/<artifact destination>` for the tree and `current_link/<carry
|
||||
destination>` for the carried file (a one-time root edit of the vhost, an
|
||||
`alias` for the carried file beside the root for the tree). A missing or
|
||||
non-regular carry source fails the deployment before anything moves.
|
||||
|
||||
What `current_link` names is observed from the symlink and confirmed by the
|
||||
latest Admitted static transaction's `flip.release_id`; it is stored nowhere
|
||||
else. A rollback is a flip back to `flip.previous_release_id`: release
|
||||
directories stay, and Idunn prunes none. The deployment stop is read exactly as for a
|
||||
service, by the binding's runtime id. Release ids are per transaction, so
|
||||
redeploying an unchanged tarball with a changed site file is a new release that
|
||||
carries the new file, never an in-place re-carry of an existing one: the carry
|
||||
is part of a release. If `current_link` is replaced by a directory or by a link
|
||||
that names nothing under the release root after the flip was recorded, the
|
||||
transaction keeps retrying with backoff: `idunn status --command <id>` shows the
|
||||
error as "waiting on ...", operator cancel is refused (Starting, not held), and
|
||||
the fix is by hand (restore the link or remove the directory); it then
|
||||
completes. A bounded retry is follow-up work. A new
|
||||
upstream tarball enters as a recipe `external_input` with a pinned url and
|
||||
sha256, so each release is a recipe commit whose pin is its provenance.
|
||||
|
||||
### Rollback
|
||||
|
||||
`idunn rollback <target> [--state-store PATH] [--no-wait]` writes a
|
||||
`rollback-` command and nothing else. The daemon freezes it into one new
|
||||
transaction (`CommandKind::Rollback`) that reuses the previous release exactly:
|
||||
the plan, sealed release and installed release are copied from the transaction
|
||||
that flipped it, so there is no source, no build and no source cleanup, and it
|
||||
creates no admitted generation. It starts at Sealing with that evidence and
|
||||
reaches the same deployment stop as a deploy, which admits it unless engaged;
|
||||
then `advance_static` flips, with the same three durable steps, the same
|
||||
nothing-aborts-after-the-rename rule and the same stuck-flip behavior.
|
||||
|
||||
The previous release is derived, never stored: the newest Admitted static
|
||||
transaction for the target (control.cc and history.cc) names the release
|
||||
`current` must point at (`flip.release_id`) and the one it displaced
|
||||
(`flip.previous_release_id`); the newest transaction that flipped that release
|
||||
supplies the evidence. A rollback of a rollback therefore flips forward again.
|
||||
The command is refused, recorded as a rejected transaction, when the target has
|
||||
no completed static release, when the newest release is the first one, when
|
||||
`current` does not name the newest flip, when the previous release directory is
|
||||
gone, or when an artifact in it no longer digests to its sealed receipt. A
|
||||
failed check at flip time ends the transaction before fencing with `current`
|
||||
untouched.
|
||||
|
||||
The rollback reuses the old plan verbatim, so its binding snapshot, brake path
|
||||
and carry source are the old ones: if the operator moved the site file the
|
||||
rollback fails before the link moves. The carry is re-applied from the site file
|
||||
as it is at rollback time, and its sha256 is recorded in the rollback's own
|
||||
flip evidence (it may differ from the original release's; the copy inside the
|
||||
rolled-back-to release is replaced). Nothing prunes release directories.
|
||||
|
||||
`CommandKind::Rollback` is stored in the existing v5 command and transaction
|
||||
records with no schema change; an Idunn at 9f15cee or earlier cannot decode
|
||||
them, so this is forward-only, like v5 itself. The newest flip is ordered by
|
||||
`flipped_at` wall clock, and the observed-link check makes a clock step refuse
|
||||
the rollback rather than mis-flip. A rollback cannot be cancelled, like a static
|
||||
deploy. A rollback is also refused when the previous release was deployed under
|
||||
a different release root or current link than the newest one.
|
||||
|
||||
## Typed deployment state
|
||||
|
||||
Mutable control-plane state will be CultCache. Runtime integration must use
|
||||
|
|
@ -202,6 +299,82 @@ names both content addresses and the executable artifact digest. Present and
|
|||
Ready remain Odin-owned correlations over independently verified runtime
|
||||
evidence; the deployment-plan module deliberately does not model them.
|
||||
|
||||
## The deployment stop
|
||||
|
||||
`src/brake.rs` is the only reader and writer of the stop. A stop halts work
|
||||
that has not crossed an irreversible step, and it is read again at each such
|
||||
step: Sealing (`advance_sealing`), the static flip intent (`advance_static`) and
|
||||
the entry into Fencing for a deploy (`advance_warming`). An engaged stop there
|
||||
makes the transaction wait with a note naming who engaged it and why; nothing
|
||||
aborts and nothing moves. After the flip intent is recorded, or once a deploy
|
||||
has fenced, the transaction runs to its end: a stop halts before the
|
||||
irreversible step, never mid-fence or mid-flip. Continuity is never gated.
|
||||
The sequence of a stop record is positive and advances by one per verb; nothing
|
||||
reads it for authority, so it is not validated further.
|
||||
|
||||
Absent means the store file is missing in an existing directory. Anything else
|
||||
Idunn cannot judge (a symlink, a zero-length or undecodable file, another
|
||||
runtime's record, another owner, an unsearchable or non-directory parent) reads
|
||||
as unreadable and fails closed; the wait note carries the
|
||||
`idunn-provision deployment-brake-clear` command that replaces it.
|
||||
|
||||
A binding may name a second stop, the target's steward:
|
||||
|
||||
```toml
|
||||
[brakes.steward]
|
||||
user = "mind-streampixels" # a login, never root
|
||||
store = "/var/lib/gamecult/idunn-steward/streampixels/deployment-brake.cc"
|
||||
```
|
||||
|
||||
Deploys admit only when every stop admits. The steward's store is read as owned
|
||||
by that user's uid: the owner uid proves who wrote the file, and the wait note
|
||||
names the stop by the binding's steward user, never by the record's `by` (which
|
||||
is the verb's own text). Idunn never writes the steward directory and no key,
|
||||
group or ACL is involved. The operator's store is a separate stop the steward
|
||||
cannot clear, and the wait note names each holding stop.
|
||||
|
||||
The steward owns its directory, so it could unlink any file in it. Binding
|
||||
validation therefore refuses a steward store whose directory is shared with,
|
||||
inside or around the directory of any other Idunn authority store (deployment
|
||||
and lifecycle brakes, trust anchor, write-lease record, release authority).
|
||||
Validation cannot see the file system: the root who provisions the directory
|
||||
makes it contain the steward's store and nothing else.
|
||||
|
||||
A steward store owned by another uid, or a steward user that does not resolve
|
||||
to a non-root uid, is unreadable and fails closed. Only the steward (the
|
||||
owner uid) can repair it: a clear run as root leaves a root-owned file that is
|
||||
still unreadable.
|
||||
|
||||
### Provisioning a steward (root, once per target)
|
||||
|
||||
1. Create the store directory, owned by the steward and holding nothing else:
|
||||
`install -d -o mind-streampixels -g mind-streampixels -m 0755
|
||||
/var/lib/gamecult/idunn-steward/streampixels`. It must not be, contain or sit
|
||||
inside any other authority directory, and no workload `ReadWritePaths` may
|
||||
cover it. Idunn (root) reads it without further grants; no group or ACL.
|
||||
2. Add the stanza above to the target's operator binding, after `[brakes]`,
|
||||
with the steward's real login. Deploys planned afterwards see it.
|
||||
3. The steward engages and clears as itself, never through root:
|
||||
`idunn-provision deployment-brake-engage --store <store> --runtime-id <id>
|
||||
--reason <text>` and `deployment-brake-clear` with the same flags.
|
||||
4. The operator reads it with `idunn-provision deployment-brake-status --store
|
||||
<store> --runtime-id <id> --owner-user mind-streampixels`.
|
||||
|
||||
The binding is sealed into each plan. A deploy already planned keeps the
|
||||
binding it was sealed with: editing the stanza does not apply to it. If the
|
||||
steward's login is deleted or its directory is lost, that deploy stays
|
||||
unreadable-held at Sealing (or the static flip intent). `idunn cancel` withdraws
|
||||
only a deploy held for a readiness disagreement before Fencing, and `idunn
|
||||
expire` only one in Warming through Committing, so neither frees a deploy held
|
||||
by the stop. Recovery is to restore the steward's login or recreate the
|
||||
directory owned by it (and have the steward clear its store).
|
||||
|
||||
Record v6 (`deployment_admitted_at_unix_millis`, key 38) is forward-only: an
|
||||
Idunn that predates v6 cannot read a migrated `control.cc`, so, as for v5 (see
|
||||
`route-continuity-cut.md`, "An older Idunn refuses the v5 store"), take a
|
||||
byte-compared pre-install backup of `control.cc` and `history.cc` with Idunn
|
||||
stopped, and roll back by restoring the backup, never by swapping the binary.
|
||||
|
||||
## Initial drivers and replaceable boundary
|
||||
|
||||
The first Yggdrasil implementation uses:
|
||||
|
|
@ -240,7 +413,7 @@ For a singleton incarnation:
|
|||
declaration, binding, launch contract, and input digests.
|
||||
5. Derive the release-bound Expected incarnation from the validated sealed
|
||||
release and its exact compiled plan. Publish it only once the deployment
|
||||
brake has admitted this exact transaction, under the candidate's own
|
||||
stop has admitted this transaction (stamping `deployment_admitted_at_unix_millis`), under the candidate's own
|
||||
incarnation key, beside whatever the target's admitted incarnation projects.
|
||||
6. Issue one opaque runtime-instance activation bound to the Expected digest,
|
||||
pass the exact Expected and activation documents through the actuator's
|
||||
|
|
@ -263,8 +436,8 @@ For a singleton incarnation:
|
|||
and stop the incumbent.
|
||||
|
||||
Sealing resolves the source, freezes it, runs every recipe step, and installs
|
||||
the release before the deployment brake is consulted at all (`advance_sealing`).
|
||||
The brake gates step 5, the first Verse-visible change: publishing the
|
||||
the release before the deployment stop is consulted at all (`advance_sealing`).
|
||||
The stop gates step 5, the first Verse-visible change: publishing the
|
||||
candidate's Expected incarnation on the move into `Starting`. Build and test
|
||||
already run unbraked, inside the sealing phase.
|
||||
|
||||
|
|
@ -295,9 +468,71 @@ identity, and route membership. Every restart receives a new runtime-instance
|
|||
activation and, for a state writer, a new process-bound lease only after the new
|
||||
warming presence is observed. A lease from the dead process is never reused. A
|
||||
continuity restart does not fetch a branch, rebuild, rerun a migration, alter
|
||||
the launch contract, or consume the deployment brake. An explicit lifecycle
|
||||
the launch contract, or pass the deployment stop. An explicit lifecycle
|
||||
brake may suspend this actuation without changing deployment authority.
|
||||
|
||||
Continuity never stops. Six charged attempts in an hour only open the
|
||||
ContinuityExhausted incident (a page; the wire name stays, nothing is
|
||||
exhausted) and gate no mint, wait or yield. When the next attempt is due is a
|
||||
function of continuity's own record: a stored attempt streak, a backoff anchor
|
||||
written when the pass first finds the admitted generation dead and the target
|
||||
free, and a capped doubling wait from that anchor (5 s base, 300 s cap). A run
|
||||
of 600 s or more by the admitted body resets the streak. A generation lost to a
|
||||
host reboot (the workload driver's evidence, `lost_to_host_reboot`) is
|
||||
re-materialized once per admission without charge or wait.
|
||||
|
||||
Each target has two slots, the incumbent and a transaction's candidate. The
|
||||
incumbent is continuity's until a transaction takes the target, at Fencing (at
|
||||
Starting for a stop-then-start rollout), through one `take_target` CAS; of two
|
||||
transactions entering from one view exactly one succeeds. Before that, a
|
||||
deployment owns only its private candidate and the target is free for it
|
||||
between continuity attempts.
|
||||
|
||||
Phases Warming through Committing, post-fencing aborts and drains all carry
|
||||
deadlines (the plan's frozen `PhaseDeadlines`; `WARMING_DEADLINE_SECONDS`, and
|
||||
`PRE_FENCING_ABORT_DEADLINE_SECONDS` for a pre-fencing abort). What an expired
|
||||
post-fencing deadline does depends on the owner. A deployment whose candidate
|
||||
was never issued a write lease aborts to the incumbent. One whose candidate was
|
||||
issued a lease may have written under a new state contract and is left to the
|
||||
operator. A continuity retries its phase forward on a capped backoff and never
|
||||
stops its live candidate, because that candidate is the only running copy of
|
||||
the admitted body. Both operator cases open an `operator-required` incident
|
||||
(subject: the transaction id) that closes when the record ends or its step goes
|
||||
on.
|
||||
|
||||
The one yield: a continuity whose route proof has not succeeded gives way to a
|
||||
deployment the deployment stop admits that awaits the handover of the
|
||||
target. It begins its own post-fencing abort and the deployment fences what is
|
||||
admitted. A continuity that has proven its route is serving never yields.
|
||||
|
||||
At boot, a lift returns to forward retry any continuity abort an older Idunn
|
||||
left over a running candidate. If the lift errs or cannot observe the
|
||||
candidate, the abort is held, not run. The lift is a migration with a deletion
|
||||
line (follow-up `c2c-delete-boot-lift`).
|
||||
|
||||
An admitted generation keeps its Ready and latest Odin receipts as durable
|
||||
proof of the exact activation at their recorded admission times. Dependency
|
||||
selection reauthenticates those receipts at those times; continuity and route
|
||||
observation own current workload health. Historical receipts do not expire
|
||||
merely because their provider remains admitted longer than the observation
|
||||
window.
|
||||
|
||||
A failed continuity restart leaves its own candidate activation projected under
|
||||
the shared Expected key. Its abort demotes that key to Expected-only with the
|
||||
exact activation the transaction issued. Rollback after fencing demotes the
|
||||
admitted generation's exact activation. No path adopts an activation the
|
||||
transaction did not issue.
|
||||
|
||||
A projection can still hold an activation the abort never resolved: a failure
|
||||
between preparing a candidate and publishing its observation, an abort written
|
||||
by an earlier Idunn, or an operator edit. Continuity therefore restarts a
|
||||
generation only from an Expected-only projection: when its own demotion is
|
||||
refused it mints no restart, records the deferral in the generation's
|
||||
continuity backoff, and tries again after it. At boot, Idunn demotes an
|
||||
activation still projected for a failed continuity in history, by that
|
||||
transaction's exact activation, and only reports one no failed transaction
|
||||
issued.
|
||||
|
||||
Idunn starts and recovers from its own durable admitted state. Odin is the first
|
||||
managed semantic daemon, never an Idunn bootstrap dependency. Odin's initial
|
||||
admission is the sole graph-bootstrap exception: Idunn uses its root-admitted
|
||||
|
|
|
|||
|
|
@ -44,7 +44,8 @@ Idunn centralizes that lifecycle work:
|
|||
- record every deployment/restart request and result as typed state;
|
||||
- record release targets, deployment artifacts, state migrations, and rollout
|
||||
results as typed state;
|
||||
- escalate to an operator through Bifrost when human action is needed.
|
||||
- record operator incidents in its published `incidents.cc`, which Bifrost reads
|
||||
and delivers. Idunn never waits on delivery.
|
||||
|
||||
The desired shape is simple:
|
||||
|
||||
|
|
@ -111,22 +112,12 @@ cargo run --release --bin idunn -- --daemon demo --rudp-health-bind <idunn-rudp-
|
|||
`--rudp-health-bind` is explicit. Bare one-daemon Idunn does not open a
|
||||
localhost health ingress.
|
||||
|
||||
Install the local Starfire boot watchdog:
|
||||
|
||||
```powershell
|
||||
.\scripts\install-idunn-startup.ps1
|
||||
```
|
||||
|
||||
That task starts one `idunn.exe` swarm supervisor at user logon. The Rust
|
||||
runtime owns the target catalog for Odin, Stonks, Muninn, the enforced
|
||||
The Rust runtime owns the target catalog for Odin, Stonks, Muninn, the enforced
|
||||
Yggdrasil source artifact lanes, and the Nightwing display services. The
|
||||
current Mimir dashboard is observed but not restarted until its restart
|
||||
authority is named.
|
||||
|
||||
It also starts `idunn-swarm-deployment-coverage`, which watches the deployment
|
||||
target catalog in `scripts/idunn-deployment-targets.ps1`. A repo/service is not
|
||||
allowed to vanish into shrug-space: it is either enforced by Idunn, explicitly
|
||||
blocked with the missing authority named, external-owned, or not a runtime.
|
||||
authority is named. A repo/service is not allowed to vanish into shrug-space:
|
||||
it is either enforced by Idunn, explicitly blocked with the missing authority
|
||||
named, external-owned, or not a runtime.
|
||||
|
||||
To record that a daemon has not published health, run a manual target without a
|
||||
matching `idunn.daemon_health` RUDP publication:
|
||||
|
|
@ -184,12 +175,7 @@ Idunn should not guess private service truth. A daemon should publish:
|
|||
If that information is missing, Idunn should fail closed and create an operator
|
||||
alarm instead of improvising.
|
||||
|
||||
For the repo swarm, deployment ownership begins with catalog coverage. Use:
|
||||
|
||||
```powershell
|
||||
.\scripts\show-idunn-deployment-targets.ps1
|
||||
.\scripts\health-idunn-swarm-deployment-coverage.cmd
|
||||
```
|
||||
For the repo swarm, deployment ownership begins with catalog coverage.
|
||||
|
||||
The enforced targets are Nightwing Gjallar plus the Yggdrasil source artifact
|
||||
lanes whose ops runbooks can produce and verify deployment manifests. Each
|
||||
|
|
@ -257,9 +243,9 @@ surface, then let Idunn run the rollout. Direct script execution without
|
|||
`IDUNN_ACTUATOR=1` fails on purpose; the little hand reaching for the deploy
|
||||
button has been removed from the machine.
|
||||
|
||||
`scripts/start-idunn-local.ps1` launches the local swarm with
|
||||
`--rudp-health-bind 0.0.0.0:17870` so host-local and explicitly configured peers
|
||||
can publish to the configured Idunn health endpoint. It accepts raw
|
||||
A local swarm daemon started with
|
||||
`--rudp-health-bind 0.0.0.0:17870` lets host-local and explicitly configured peers
|
||||
publish to the configured Idunn health endpoint. It accepts raw
|
||||
`idunn.daemon_health` document puts on the canonical CNR0 RUDP `schema` channel
|
||||
and writes them into the keepalive store. That is daemon-owned health
|
||||
publication, not restart/deploy authority. On Windows, UDP reset reports from
|
||||
|
|
@ -305,11 +291,11 @@ layer must call `.ps1` launchers directly instead of `cmdPath` trampolines.
|
|||
Those three live Raven tasks have been repaired and verified. The repair
|
||||
actuator now uploads its Raven PowerShell body with `sftp` and runs a tiny
|
||||
cleanup wrapper, so the hidden-task repair does not hit Windows command-line
|
||||
limits before it can run. The long-running Muninn serve bodies on Raven,
|
||||
Nightwing, and Starfire now also carry their own `--idunn-rudp-health`,
|
||||
`--idunn-daemon`, and `--idunn-health-contract` arguments, and live Idunn
|
||||
accepts those daemon-owned health records directly instead of relying on
|
||||
one-shot health commands for the truth.
|
||||
limits before it can run. The Muninn serve bodies no longer publish a
|
||||
daemon-health document: `--idunn-rudp-health`, `--idunn-daemon` and
|
||||
`--idunn-health-contract` are retired flags that the binary rejects. Raven's
|
||||
liveness is Odin-correlated runtime presence; Starfire and Nightwing are
|
||||
checked by their `scripts/health-*-muninn.ps1` witness scripts.
|
||||
|
||||
From here on out, the work is not "teach another daemon to speak." It is
|
||||
"delete the witness masks and keep ownership where it belongs."
|
||||
|
|
|
|||
|
|
@ -82,8 +82,9 @@ running a script, and a dead `muninn serve` is seen the tick it dies.
|
|||
2. No path on yggdrasil can start, stop or observe a host workload except
|
||||
through the actuator session bound to that host's anchor.
|
||||
3. A host with no attached actuator is a failed observation, which
|
||||
continuity counts and gives up on after three attempts, exactly like a
|
||||
unit that will not start.
|
||||
continuity counts as a charged attempt and retries on its capped backoff
|
||||
(it never gives up; six in an hour only pages), exactly like a unit that
|
||||
will not start.
|
||||
4. The actuator cannot be told to run a program the recipe did not declare;
|
||||
`allowed_programs` on the host-native runner is enforced on the host.
|
||||
|
||||
|
|
@ -96,7 +97,8 @@ Each of these was found by running the pipe against Raven, not by reading.
|
|||
bind time. `rollout.strategy = "stop-then-start"` exists for exactly a
|
||||
route-less host-actuator workload: the admitted incumbent is stopped at
|
||||
Starting, before the candidate starts, and supervision leaves the target
|
||||
alone from Starting on. A systemd-transient workload may not declare it.
|
||||
alone from Starting on: the transaction takes the target there (one
|
||||
`take_target` CAS) rather than at Fencing. A systemd-transient workload may not declare it.
|
||||
- **An unobservable host is silence.** The hub returns a typed
|
||||
`HostUnobservable` for not-attached, detached, reconnected and unanswered.
|
||||
Supervision skips that tick and logs it. Before this, an Idunn restart
|
||||
|
|
@ -147,3 +149,7 @@ builds on a Windows host with the stable toolchain and runs on Raven; it is
|
|||
not built on yggdrasil. The library compiles on both; unix-only code is
|
||||
`cfg(unix)`, and the Linux-shaped test fixtures pass on a Windows checkout
|
||||
except the one parent-only-descriptor test.
|
||||
|
||||
Invoke Cargo directly as the container command or through a non-login shell.
|
||||
A login shell can reset `PATH` and hide the Rust toolchain installed in the
|
||||
build image.
|
||||
|
|
|
|||
|
|
@ -67,23 +67,20 @@ Migrate in dependency order, and migrate the authority's own dependencies last:
|
|||
2. **`repixelizer`** — second. No CultLib dependency at all, but runs a venv
|
||||
interpreter against a checkout, so it needs an artifact story for Python
|
||||
before a recipe means anything. Unit is `repixelizer-gui`.
|
||||
3. **`streampixels`** — **blocked, like Ghostlight.** The product has changed
|
||||
direction: 2D animation could not express what the spec required, so it is
|
||||
moving to a 3D-rendered overlay, and whether that runs in WebGL or needs
|
||||
rendering infrastructure is undecided. Its deployment shape depends on that
|
||||
answer — a WebGL overlay is the current two-unit shape, rendering
|
||||
infrastructure is a new target class with hardware requirements. Do not
|
||||
migrate it, and do not "fix" the 2D animation path: it is being replaced, not
|
||||
repaired. There is a paying client, so coordinate rather than assume.
|
||||
3. **`streampixels-service` and `streampixels-web`** — migrate as two Idunn
|
||||
targets from one repository. The native Unity renderer runs on each creator's
|
||||
machine; Yggdrasil hosts only the TypeScript API and Next.js web app. Build
|
||||
those two artifacts with the pinned Node runner, preserve Postgres and Redis
|
||||
as external state owners, and admit the service before the web target that
|
||||
consumes its API capability. The repository declares separate recipes at
|
||||
`deployment/idunn/service.toml` and `deployment/idunn/web.toml`; each host
|
||||
binding selects one recipe path.
|
||||
|
||||
**Their continuity is already broken, and has been.** The legacy actuator
|
||||
restarts all three with
|
||||
`docker compose -f /srv/compose/yggdrasil-apps.yaml restart <name>`, and that
|
||||
file **does not exist** — `/srv/compose` holds only `odin`, `voidbot` and
|
||||
`voidbot-retrieval`. The path is guarded by `require_root_owned_regular_path`,
|
||||
so every restart of these three has been failing at the guard. They are
|
||||
running only because nothing has asked them to restart. This is the
|
||||
difference between a supervised service and a service that happens to be up.
|
||||
**Retire the old continuity path during cutover.** The preview tarball,
|
||||
checkout-based systemd units, and legacy deploy helper are not an alternate
|
||||
recovery lane. The current host's live status and any existing creator data
|
||||
must be inspected before stopping them. Preserve the public host and path
|
||||
routing, with nginx pointing at Idunn's stable service and web endpoints.
|
||||
4. **`ghostlight`** — **blocked, not merely later.** A world-elaboration and
|
||||
ontology rebuild is running in it right now: commits landed 2026-09-05 across
|
||||
178 branches, with work parked on `codex/ghostlight-dungeon-mvp`. Adding
|
||||
|
|
@ -523,7 +520,7 @@ started, ran, and then died on a file it could not open:
|
|||
| `runtime_root` and every directory above it | traversable by the workload: `o+x`, or group-owned by the state group with `g+x` | the workload is a `DynamicUser` and owns nothing, and `ReadOnlyPaths=` binds the bundle's leaf into the namespace without granting traversal above it. Idunn now checks the whole chain and names the closed directory |
|
||||
| `runtime_root` | setgid, group-owned by the state group (`2750 root:<state_group>`) | `harden_root_authority_file` requires the write-lease record's group to equal its parent's, and only setgid makes a newly created record inherit it. Without it the record lands `root:root` and the workload cannot read its own lease |
|
||||
| the write-lease record | `root:<state_group>`, mode `0640` | follows from the setgid parent; the workload reads it through the state group. Idunn writes it, so fixing an existing wrong-group record means deleting it, and that is safe only when no transaction is live |
|
||||
| `/var/lib/gamecult/idunn-projection` | default ACL `u::rw-,g::r--,o::r--` | Idunn runs with `UMask=027`, so each publish creates the store and its `.lock` at `0640`. The code re-opens them after every write, but that leaves a window; a default ACL sets the mode at creation and closes it |
|
||||
| `/var/lib/gamecult/idunn-projection` | default ACL `u::rw-,g::r--,o::r--` | Idunn runs with `UMask=027`, so a store is created `0640` and each publish re-opens it `0644`; the default ACL sets the mode at creation and closes that window for stores. The `.lock` is not published and no reader opens it: Idunn creates it `0600` (replacing one found wider, so a descriptor another uid already holds is orphaned), and the ACL never decides its mode |
|
||||
| `/etc/nginx/idunn-stream-routes` | `root:root`, not group-writable | "route authority parent is not canonical root-owned and nonwritable". It was `root:idunn 0775` from the generation where Idunn ran unprivileged and needed to write it as a group member |
|
||||
|
||||
Two shapes are worth stating plainly because they are easy to get backwards.
|
||||
|
|
|
|||
600
docs/reincarnation-map.md
Normal file
600
docs/reincarnation-map.md
Normal file
|
|
@ -0,0 +1,600 @@
|
|||
# Reincarnation after reboot: map
|
||||
|
||||
Status: map, Imagination, 2026-10-18 (campaign `idunn-reincarnation`, session
|
||||
`idunn-reincarnation-2026-10-18`). The ends are in
|
||||
[`reincarnation-target.md`](reincarnation-target.md). Questions, rulings, the
|
||||
cut spec and follow-ups live in Eureka's mind; this page holds what has
|
||||
no kind: body facts, the model page, the authority map's rationale and the
|
||||
cut sequence. Line numbers are at `74a7008`; `src/` is byte-identical to
|
||||
`c2477d4`, so the earlier anchors still hold.
|
||||
|
||||
Cut 2 pass, Imagination, 2026-10-19 (same campaign and session): body facts
|
||||
B13-B18, the cut-2 rows of the model page and the cut-2 parts of sections 3-5
|
||||
are anchored at `9a0a6a2`, whose `src/` is byte-identical to `ffb4293` (cut 1's
|
||||
head). The B1-B12 anchors above stay at `74a7008`.
|
||||
|
||||
Read the in-force set with the rulings-in-force and open-follow-ups recipes
|
||||
for root `idunn-reincarnation`; the cut in flight is the specs-with-no-report
|
||||
recipe. The first spec (`cut-c1-durable-stop.r1`) was withdrawn when the
|
||||
operator ruled that continuity never stops; its replacement is cut
|
||||
`c1-never-stop`. Cited from `idunn-watchdog`: `exhaustion-close-rule-op` and
|
||||
`paging-conditions`.
|
||||
|
||||
## 1. Body facts
|
||||
|
||||
Evidence: the `idunn-yggdrasil` journal from 11:40 UTC (94,670 lines), the
|
||||
per-unit journals of the streampixels-service candidates and of Odin
|
||||
(`idunn-odin-7cfe5cd1…`), `idunn status` and `idunn status --command <id>`
|
||||
on Yggdrasil, `journalctl -u nginx`, and the source at `c2477d4`. All times
|
||||
UTC, 2026-10-18.
|
||||
|
||||
**B1. The reboot did not stop continuity from re-materializing anything.**
|
||||
Every streampixels-service continuity attempt minted a fresh transient unit
|
||||
from the installed release `sha256-df663f0a…`: 54 continuity commands from
|
||||
`continuity-85887ef2` (11:46) to `continuity-52bbbd66` (19:57), most listed by `idunn status` as
|
||||
`continuity-… failed systemd unit idunn-streampixels-service-<hash>.service
|
||||
is not running`. A continuity transaction is a full `DeploymentTransaction`
|
||||
built by `from_continuity` (`control_plane.rs:1678`) from the admitted plan,
|
||||
sealed release, installed release and Expected. It runs every phase:
|
||||
Starting, Warming, Fencing, Leasing, AwaitingReady, Routing, Committing. The
|
||||
absent admitted unit `461c6443…` was never the problem. Each new candidate
|
||||
was.
|
||||
|
||||
**B2. Every candidate died of a dependency, not of itself.** The
|
||||
streampixels-service process exits about 30 s after start when Odin does not
|
||||
admit its initial presence. Unit `2bc02905…` at 11:47:21 logged "StreamPixels
|
||||
service failed to enter its Idunn incarnation: Error: Odin did not admit the
|
||||
initial Idunn runtime presence ... Timed out waiting for CultNet RUDP response
|
||||
after 5000 ms." The same is logged by `42ecf363…` at 13:47:13, `03acf44d…`
|
||||
at 19:47:14, `45276008…` at 19:49:54 and `599b705b…` at 19:51:18. Odin's
|
||||
RUDP endpoint is `rudp://10.77.0.1:17871`, which nginx proxies. nginx failed
|
||||
at 11:46:26 ("bind() to 10.77.0.1:17871 failed (99: Cannot assign requested
|
||||
address)") and started at 19:46:24. Between 11:46 and 19:44 Odin's journal
|
||||
shows no presence message from streampixels-service at all: none arrived.
|
||||
After nginx returned, the first one arrived at 19:49:25 and was rejected:
|
||||
"runtime presence has no exact current activation and provider anchor". The
|
||||
attempts minted at about 19:53 and 19:55 got past Fencing and then failed
|
||||
"the Leasing phase passed its deadline" (FailedAfterFencing, RestoreIncumbent).
|
||||
Other early failures: "observed write lease is not the current Idunn grant"
|
||||
(11:46:47, the first attempt after boot), and three "reading process stat
|
||||
/proc/N/stat: No such file or directory" races.
|
||||
|
||||
**B3. Every attempt was charged at mint.** `record_restart(now)` runs inside
|
||||
the mint CAS (`control_plane.rs:6156-6157`). Nothing reads a continuity's
|
||||
outcome back into the log. The only exemption anywhere is `HostUnobservable`
|
||||
(`:5944`), and it applies to an observation, not to a candidate. So the 6-per-
|
||||
hour window (`CONTINUITY_RESTART_ATTEMPTS` = 6, window 3,600,000 ms, `:71-72`)
|
||||
filled at 11:50:52 and refilled every hour as entries slid out. The bursts are
|
||||
in the journal at :47, :48, :49, :51, :52 and :53 of every hour, 39 times
|
||||
"Idunn stopped restarting admitted streampixels-service". This answers Q2:
|
||||
the nginx outage counted against the body's budget, and the place that decides
|
||||
it is the unconditional charge at `:6156`.
|
||||
|
||||
**B4. The deployment was killed by a window slide, not by a live
|
||||
continuity.** `up-3ad6b05b` (release `sha256-67e227b2…`, a new release) was
|
||||
recorded at 20:24:51. It was still before Fencing at 20:46:41, when it was
|
||||
completed `FailedBeforeFencing` with "admitted incumbent failed before
|
||||
candidate fencing; deployment yielded to continuity: systemd unit
|
||||
idunn-streampixels-service-461c6443….service is absent". Two seconds later
|
||||
continuity minted `tx-0804185a` with the OLD release `df663f0a…`, and it was
|
||||
admitted. The restart that left the window was minted at about 19:46:40 (its
|
||||
unit `03acf44d…` started at 19:46:42), exactly one window before the abort.
|
||||
The mechanism, from the source:
|
||||
|
||||
- `:5805-5821`: a live transaction for the target is the `blocker`. From
|
||||
Fencing on (or Starting for stop-then-start) supervision skips the target
|
||||
entirely.
|
||||
- `:6033-6061`: with a blocker present, continuity never restarts and never
|
||||
reports. It may abort the blocker, and the predicate for that is
|
||||
`Deploy && !supervision.restarts_exhausted(now) && phase < Fencing && no
|
||||
abort && not held` (`:6045-6050`).
|
||||
- `:6062-6084`: only with no blocker does an exhausted continuity open the
|
||||
incident and say "The target is free for a deployment to replace it".
|
||||
|
||||
So from 20:24:51 the deployment owned the target (`owns_target_authority`,
|
||||
phase != Complete, `:1772`). Continuity deferred to it through the blocker
|
||||
`continue` at `:6060`. The deployment deferred to continuity whenever the
|
||||
window had a free slot, through `:6046`. "Stopped" existed only as a durable
|
||||
incident (incidents.cc, a projection) and a report. The ownership decision
|
||||
read the window. When the window gained a slot, continuity reclaimed the
|
||||
target with no record that it had resumed. This answers Q3: the two owners are
|
||||
the live `DeploymentTransaction` (it owns while not Complete) and
|
||||
continuity-as-window (it owns whenever the window is not full). Continuity
|
||||
defers to the deployment while any transaction is live. The deployment defers
|
||||
to continuity while the window has room. The "free for a deployment" message
|
||||
names the first ownership, and the code acts on the second.
|
||||
|
||||
**B5. Fencing an absent incumbent already works.** `advance_fencing`
|
||||
(`:6869-6952`) stops the exact incumbent and revokes its lease. The systemd
|
||||
driver's `stop` returns `Ok(())` for an absent unit (`drivers.rs:4003-4007`).
|
||||
A deployment that is not aborted therefore replaces an absent incumbent with
|
||||
nothing to fence, which is target invariant 3's last clause, already true.
|
||||
`exact_incumbent` (`:7556-7569`) fails "target incumbent changed after
|
||||
transaction creation" if a continuity commit replaced the generation during
|
||||
the deployment. That is why a phase-split handover needs a rebase
|
||||
(`handover-pre-fencing`).
|
||||
|
||||
**B6. The two rejection streams are consequences of a dead admitted
|
||||
generation, not separate faults.** "preserved admitted streampixels-service
|
||||
after rejecting topology observation: topology correlation does not bind the
|
||||
current activation" was logged 46,271 times, from 11:46:49 to exactly
|
||||
20:46:48, about 5,300 an hour. Ghostlight's 42,482 identical lines ran to
|
||||
19:56:29. Since 20:47, raven-muninn, newly exhausted, shows the same line
|
||||
621 times. `refresh_admitted_topology` (`:6395-6457`) authenticates Odin's
|
||||
correlation against the admitted generation's activation. When that
|
||||
incarnation is dead and Odin's latest record names another one (a
|
||||
continuity candidate's, or a stale one), authentication fails with an `Err`
|
||||
rather than `Ok(false)`, and `:5834-5840` prints it on every pass. The pass
|
||||
order is what does it: route supervision (`:5822`) and topology refresh
|
||||
(`:5834`) run before the workload is observed (`:5891`). So a dead
|
||||
generation is "supervised" every tick. "rejected admitted
|
||||
streampixels-service route continuity: admitted process write lease is no
|
||||
longer exact" (3,615, from 20:02:23 to 20:46:48) followed the two
|
||||
post-fencing aborts at 19:54 and 19:57. Fencing revoked the dead incumbent's
|
||||
lease, "restore incumbent" cannot re-grant a lease to a process that does not
|
||||
exist, and `supervise_admitted_route` (`:6250-6255`) then refuses the
|
||||
admitted generation on every challenge. Both streams stopped the moment
|
||||
`tx-0804185a` was admitted. This answers Q4: both streams follow from one
|
||||
model gap, that a dead admitted generation is treated as a live incumbent.
|
||||
That gap is cut 4. They are not the deadlock.
|
||||
|
||||
**B7. Ghostlight is a separate fault.** Its continuity `tx-039b91e9` is
|
||||
wedged in a post-fencing abort. It was "lease-not-adopted" at its AwaitingReady
|
||||
deadline, and the abort withdraws the candidate's lease (`:8895-8928`) before
|
||||
it stops the candidate (`:8929-8946`). The candidate unit `6c51e35e…` has
|
||||
been running since 19:55:15 and holds the lease's shared lock, so every
|
||||
attempt fails "process write lease is held by a shared-lock consumer"
|
||||
(`drivers.rs:4179/4224`). The transaction owns the target, so continuity is
|
||||
locked out and the route degraded (53 consecutive failures). This is follow-up
|
||||
`ghostlight-abort-lease-before-stop` (route-continuity invariant 5), not the
|
||||
stopped-owner deadlock.
|
||||
|
||||
**B8. Unexplained, recorded.** Idunn's record of Odin's stable route reads
|
||||
"degraded-since 11:46:31, consecutive-failures 1, next-challenge-at 11:47:01"
|
||||
about nine hours later. It has not been re-challenged since. This is follow-up
|
||||
`odin-route-record-stale`. It blocks cut 3. Under `budget-never-stop`, a
|
||||
failure attributed to a declared dependency must not grow the backoff, and the
|
||||
only dependency-health record Idunn holds for Odin's stable route is this
|
||||
one.
|
||||
|
||||
**B9. Luna's weekend commits on this path (Q5).** None of them caused the
|
||||
deadlock. The yield predicate dates from the route-continuity and watchdog
|
||||
campaigns, not from the weekend. Where they touch:
|
||||
|
||||
| Commit | On the path? | Status |
|
||||
|---|---|---|
|
||||
| `9f00e7a` replay provider receipts at admission time | Yes: the deployment's Sealing compile (`advance_sealing` → `current_ready_provider_tokens`) logged "excluded non-current provider ghostlight / raven-muninn: admitted provider's latest receipt is not Ready at its admission time" at 20:24:53 (`:8588`, `:8609`) | Still live: route-continuity B4 has not landed. It did not block this deployment |
|
||||
| `29946bb` wait for the stable route listener | Yes: the route proofs during the outage were "Connection refused" on 127.0.0.1:8831-8833 | Kept by route-continuity (honest scope) |
|
||||
| `8ae00a1` cancel live stateless candidates | No: no cancel ran | Still present (`:2888`); route-continuity S2 owns its deletion |
|
||||
| `ae8ac45`, `8651906`, `e976d32`, `5e94eb8` identity and launch evidence | Only in `observe` and `stop`; nothing here drifted | Kept |
|
||||
| `a148802` | No | Reverted (`87fa57d`) |
|
||||
|
||||
The target forbids building on them. Cut 1 touches none of their lines.
|
||||
|
||||
**B10. The stop was the only door a deployment had into a crash loop.**
|
||||
Source read at `74a7008`. A queued `idunn up` becomes a transaction only in
|
||||
`freeze_command`, which refuses while any transaction for any of its targets
|
||||
`blocks_new_target_mutation` (`:5741-5747`); the command just stays queued.
|
||||
The scheduler tick runs `retire_one_terminal_transaction` (returns early when
|
||||
it retires), then `resume_one_transaction`, then
|
||||
`supervise_one_admitted_generation`, then `freeze_one_queued_command`
|
||||
(`:5414-5423`). Today's doubling wait is anchored at the previous attempt's
|
||||
**mint** (`next_restart_at`, `:1163-1170`). So once an attempt lasts longer than
|
||||
its wait, the tick after its transaction is retired mints the next attempt
|
||||
before the freeze runs, and the queued deployment never finds the target
|
||||
free. At `c2477d4` that ended when the window filled and continuity stopped.
|
||||
Delete the stop without opening another door and a crash-looping release can
|
||||
never be replaced. The door the cut uses is kubelet's own anchor
|
||||
(`docs/research/restart-budget-prior-art.md` §1.1, the exited container's
|
||||
`FinishedAt`): the wait runs from the first supervision pass that finds the
|
||||
target dead **and free**. That leaves at least one wait (5 s or more) with no
|
||||
live transaction, and the freeze runs every tick. This is reasoning over the
|
||||
source, not a probe. The cut pins it with a tick-level test.
|
||||
|
||||
**B11. The yield cannot outlive the stop.** With the window term gone from
|
||||
`:6046`, the yield aborts every pre-fencing Deploy whenever the incumbent is
|
||||
dead (`:6045-6059`). For a crash-looping release the incumbent is always
|
||||
dead, so the deployment that would fix it is aborted at the first
|
||||
supervision pass after it is frozen. That is the 2026-10-18 failure: on that
|
||||
day the operator's `idunn up` existed to replace a release that would not
|
||||
start. Ruling `handover-phase-split` already says nothing yields before
|
||||
Fencing. So cut 1 deletes the whole yield. Continuity keeps deferring to any
|
||||
live transaction for its target (`:6033`, the `continue`). Cut 2 keeps
|
||||
the incumbent alive alongside the build. Between them, a dead incumbent stays
|
||||
down while a pre-fencing deployment over it builds. That is bounded by the
|
||||
deployment's own deadlines, and the deployment then fences it as nothing
|
||||
(B5).
|
||||
|
||||
**B12. The doubling function already exists once.** `backoff_wait`
|
||||
(`:4000-4007`) doubles from a base up to `RESUME_BACKOFF_CEILING_MILLIS`, and
|
||||
has one caller (`back_off`, `:5621`). Continuity's wait is a second spelling
|
||||
of the same rule (`5_000 << (live.len() - 1)`, `:1169`). The cut gives
|
||||
`backoff_wait` a ceiling parameter and deletes the second spelling.
|
||||
`#[cultcache(key = N, default)]` is how an absent key decodes as its default
|
||||
(precedent `DeploymentTransaction` key 34, `:1552-1557`). New
|
||||
`TargetSupervision` keys use it, because every record on Yggdrasil predates
|
||||
them.
|
||||
|
||||
**B13. Two live transactions for one target make the control store
|
||||
unreadable.** Source read at `9a0a6a2`. `ControlSnapshot::validate` refuses
|
||||
"multiple transactions claim current-incarnation authority for one target"
|
||||
over `owns_target_authority` (`:3245-3254`), and that predicate is just
|
||||
`phase != Complete` (`:1896-1898`). Every reader of the store goes through it:
|
||||
the scheduler tick, `idunn status`, cancel, expire. So if supervision minted a
|
||||
continuity beside a pre-fencing deployment today, the next read would fail
|
||||
the whole engine. Coexistence needs the identity rule changed first, and that
|
||||
is the cut's first commit.
|
||||
|
||||
**B14. One predicate answers four questions, and the phase rule lives in
|
||||
one caller.** `blocks_new_target_mutation` (`:1903-1909`, live or post-commit
|
||||
cleanup pending) is read by the freeze busy check (`:5934-5942`), the
|
||||
supervision blocker (`:5998-6001`, a single `.find`), candidate-port
|
||||
reservation (`:7997`) and status (`:4613`). `owns_target_authority` is read by
|
||||
validation (`:3249`) and by sibling ordering inside one command (`:3384`).
|
||||
Neither looks at the phase. The only phase-aware ownership rule is inline in
|
||||
supervision (`:6002-6012`, owns from Fencing, or from Starting for
|
||||
stop-then-start). The freeze refusal is why a `profile:` command needs every
|
||||
target free in one tick (follow-up `profile-deploy-joint-gap`).
|
||||
|
||||
**B15. What a pre-fencing deployment reads about its incumbent.** Warming's
|
||||
semantic warming check uses the incumbent's lease digest
|
||||
(`:6904-6905`, `is_semantic_warming` `:9530-9560`: Odin reports the candidate's
|
||||
one disagreement as the lease the incumbent holds). Route preflight uses the
|
||||
incumbent's route observation (`:7026-7047`). Isolation is proven against the
|
||||
incumbent's workload (`:7050-7076`). Each goes through `exact_incumbent`
|
||||
(`:7768-7781`), which bails "target incumbent changed after transaction
|
||||
creation" once a continuity commit replaced the generation. Sealing, and
|
||||
Starting for candidate-then-promote, read no incumbent. Stop-then-start's
|
||||
Starting stops it (`:6857-6863`), which is already the owner's act. The
|
||||
Sealing-to-Starting write is a `replace_transaction` at `:6806-6808`, not
|
||||
`transition` (`:8914-8927`), so the two edges at which a transaction can take
|
||||
a target are written by two different functions today.
|
||||
|
||||
**B16. The commit already guards the incumbent; nothing guards the
|
||||
handover.** `advance_committing` expects the admitted envelope it read
|
||||
(`:7701-7702`, CAS `:7743-7764`), so a deployment can never commit over a
|
||||
generation that moved under it. Entering Fencing is an unguarded
|
||||
`replace_transaction` of the transaction alone. Continuity's mint CAS
|
||||
(`:6371-6400`) guards its command, transaction and supervision record, and no
|
||||
other transaction. The store accepts read-only expectations: an expected
|
||||
entry with no replacement only guards (CultLib
|
||||
`packages/cultcache-rs/src/lib.rs:1327-1368`). One engine holds the store
|
||||
(`ProcessLock::acquire`, `:5514`), but the CLI writes transactions too:
|
||||
`cancel` sets a pre-fencing abort on a live deployment (`:3014-3083`). So a
|
||||
handover decided by one CAS over the target's records is exact, and a tick
|
||||
order is not.
|
||||
|
||||
**B17. What coexisting candidates share, and what they do not.** Continuity
|
||||
reuses the admitted plan, so its candidate port is the admitted generation's,
|
||||
already reserved (`:7988-7992`); a deployment picks another. Topology is keyed
|
||||
per incarnation (`:8495-8498`): a continuity publishes under the admitted key,
|
||||
a deployment under its own. No candidate holds a write lease before its own
|
||||
Leasing, which follows Fencing, so two pre-fencing candidates cannot both
|
||||
write. Route actuation by a continuity is always `Survival`, never refused
|
||||
(`:4870-4876`); a deployment's `Forward` cannot starve it. The Odin replay
|
||||
floor `max_odin_sequence` (`:3399-3435`) spans every transaction of the target
|
||||
and the admitted cursor; incumbent refresh and candidate warming already share
|
||||
it during every deployment today, so the replay check (`:9498-9501`) gains no
|
||||
new exposure. Stop-then-start is declared only by route-less host-actuator
|
||||
bindings (`deployment.rs:756-764`, `:1245-1249`), and no control-plane test
|
||||
builds one (grep `StopThenStart` in `src/control_plane.rs`: the definition
|
||||
site only). It is live on Yggdrasil: raven-muninn declares it
|
||||
(`git grep stop-then-start` in `gamecult-ops`:
|
||||
`idunn/yggdrasil/bindings/raven-muninn.toml.in`). So the Starting edge of the
|
||||
handover carries production traffic, and its engine-level proof is owed
|
||||
before the closing install (follow-up `stop-first-handover-fixture`).
|
||||
|
||||
**B18. A pre-fencing error is an abort.** `resume_candidate` turns any `Err`
|
||||
from a pre-fencing step into `begin_pre_fencing_abort` unless it is
|
||||
`LockContended` (`:5768-5783`). So anything the cut makes a deployment wait on
|
||||
must return `Ok` (a gate wait, `record_gate_wait` `:8170-8182`), or the wait
|
||||
becomes the yield that ruling `handover-phase-split` forbids. The same path is
|
||||
how a superseded continuity ends itself, which is wanted.
|
||||
|
||||
## 2. Model page
|
||||
|
||||
One row per persistent kind the campaign touches.
|
||||
|
||||
| Kind | What names it | Lifecycle | Who decides |
|
||||
|---|---|---|---|
|
||||
| Paging log: `TargetSupervision.continuity_restarts` (control.cc, `idunn.target_supervision.v1`, key 3) | key = target | The newest 6 charged attempt times, ascending, read through a sliding 1 h window. Outlives every generation. After cut 1 it is the paging threshold only (`restarts_page`): it opens and holds the ContinuityExhausted incident and is shown in status. It gates no mint, wait or yield | Continuity supervision, pushed in the mint CAS |
|
||||
| Attempt streak (new, cut 1: key 7, `default`) | target | Charged attempts since the admitted body last ran a sustained run. +1 in each charged mint CAS. Reset to 0 when the dead generation was admitted after the last charged attempt and ran ≥ 600 s before it was found dead. Saturating, and the wait is capped, so it can be unbounded | Continuity supervision only |
|
||||
| Backoff anchor (new, cut 1: key 8, `default`) | target | Written on the first pass that finds the admitted generation dead and the target free after an attempt. The next attempt is due at anchor + `backoff_wait(5 s, streak - 1, 300 s)`. Cleared by every mint. Stale, and ignored, once older than the admitted generation's `admitted_at` | Continuity supervision only |
|
||||
| Re-materialization bound (new, cut 1: key 9, `default`) | target | Set by the one attempt that is not a backoff step, when the driver reports a reboot loss. Spent while it is not older than the admitted generation's `admitted_at` | Continuity supervision, from the workload driver's evidence |
|
||||
| Projection deferral (keys 4-5) | target | Unchanged: a failed demotion defers the next mint 30 s. It spaces a re-materialization too | Continuity supervision |
|
||||
| Admitted generation (`idunn.admitted_generation`) | key = target, `generation_id` = `generation-<tx>` | Replaced atomically by every committed deployment or continuity. A dead incarnation keeps its record until replaced | The committing transaction |
|
||||
| Incident `ContinuityExhausted` (incidents.cc) | condition + subject (target) | Opened at 6 charged attempts in the hour while the target is dead and free. Closed when the target runs and the window no longer holds 6 (`exhaustion-close-rule-op`). Retired after retention. The name is the wire name and stays, though nothing is exhausted any more | Idunn's incident writer, a notification projection: never read by ownership |
|
||||
| Transaction and pre-fencing abort (`idunn.deployment_transaction.v4`, history.cc once terminal) | `tx-<uuid>` and its `command_id` (`up-…` or `continuity-…`). Keyed by transaction, so a continuity and a deployment for one target are two records with no shared key | Sealing … Complete. A pre-fencing abort completes `FailedBeforeFencing`, a post-fencing abort `FailedAfterFencing` with a named recovery. Retired to history. After cut 2, per target at any instant: at most one live Deploy, at most one live Continuity, at most one owner (validated by `ControlSnapshot::validate`, B13) | The scheduler advances it. After cut 1, continuity supervision never aborts a transaction; the transaction's own steps do. After cut 2, a continuity whose target was taken, or whose source generation was replaced, aborts itself (superseded) |
|
||||
| Target ownership (cut 2: derived, no record) | target | A transaction owns its target from `takes_target_at()` (Fencing, or Starting for stop-then-start) until Complete; a Complete transaction with pending post-commit cleanup drains it. Entered only through the handover CAS (`Engine::take_target`) | Derived from the phase. The handover CAS is the one writer of a phase entry into `takes_target_at()` |
|
||||
| `incumbent_generation_id` on a transaction (key 9, unchanged format) | the generation's id | Deploy: set at freeze; before the handover it names the generation its incumbent-relative evidence was proven against and is rebased when the admitted generation moves (clearing that evidence); pinned by the handover CAS, exact from then on. Continuity: its source generation, never rebased | Deploy: its own pre-handover rebase step and the handover CAS. Continuity: the mint |
|
||||
| Unit materialization (systemd transient unit `idunn-<target>-<hash>`) | the activation's hash | Created by Starting. Lost on a host reboot (`/run/systemd/transient`). A failed unit stays loaded; a cleanly stopped one is collected | The workload driver actuates. Whether a loss was a reboot is the driver's evidence (`WorkloadPort::lost_to_host_reboot`, cut 1) |
|
||||
|
||||
No cell is open. Both forks are ruled (`budget-never-stop`,
|
||||
`handover-phase-split`). The numbers that `budget-never-stop` left open are
|
||||
the defaulted ruling `continuity-backoff-defaults`. Cut 2 opens one question,
|
||||
`fencing-over-owning-continuity` (does a deployment ready to fence wait for a
|
||||
continuity that already owns the target, or preempt it); the spec is written
|
||||
for the recommended answer, wait.
|
||||
|
||||
## 3. Authority map: continuity and deployment
|
||||
|
||||
The authority map for cut 1 is in its spec. This section is the
|
||||
campaign-level map, and cut 1 is one step of it.
|
||||
|
||||
- **Owner.** Exactly one authority actuates a target at each instant. A live
|
||||
`DeploymentTransaction` owns the target from Fencing on, and a stop-first
|
||||
rollout owns it from Starting. Before Fencing (`handover-phase-split`), a
|
||||
deployment owns only its private candidate, and continuity owns the
|
||||
incumbent slot. Continuity owns the admitted generation's survival and
|
||||
never stops. When it next acts is a function of its own record: the
|
||||
streak, the backoff anchor and the re-materialization bound.
|
||||
- **Inputs.** For continuity: the admitted generation (id, `admitted_at`,
|
||||
workload, plan), the workload observation and the driver's reboot
|
||||
evidence, its own record, and whether a live transaction blocks the
|
||||
target. After cut 3 it also reads Idunn's dependency-health observation.
|
||||
For a deployment: its own plan, its brake, and at Fencing whatever
|
||||
generation is admitted at that moment (cut 2).
|
||||
- **Outputs.** Continuity: mints, either charged or the one re-materialization;
|
||||
backoff anchors; deferrals; and, as notifications only, the
|
||||
ContinuityExhausted incident and its once-report. Deployment: a new admitted
|
||||
generation, or an abort that names its recovery.
|
||||
- **Derived state.** `restarts_page(now)` (6 charged attempts in the hour) is
|
||||
derived from the paging log and is notification-only: incident open and
|
||||
close, and status. The next attempt's due time is derived from the
|
||||
anchor and the streak. The incident and `idunn status` are projections.
|
||||
- **Forbidden writers.** The paging window gates no mint, no wait and no
|
||||
yield. Continuity supervision never aborts a deployment:
|
||||
`begin_pre_fencing_abort` has no caller in it. No deployment path writes
|
||||
continuity's fields. The incident store decides nothing. No stopped state
|
||||
exists: no field, and no branch that ends a pass because a count is
|
||||
reached.
|
||||
- **Shared paths.** A crash restart, a restart after a sustained run, and a
|
||||
reboot re-materialization all mint through the one CAS (`:6158-6183`). The
|
||||
dead-and-free pass is the only writer of the backoff anchor. Every
|
||||
deployment, operator or scheduled, enters through `freeze_command` and
|
||||
advances through `advance_transaction`.
|
||||
- **Deletion line.** Cut 1 deletes the yield (`:6034-6059`), the
|
||||
exhaustion `continue` and its "free for a deployment" text (`:6062-6085`),
|
||||
and the mint-anchored wait (`:1160-1184`). Cut 2 (anchors at `9a0a6a2`)
|
||||
deletes `owns_target_authority` and `blocks_new_target_mutation`
|
||||
(`:1894-1909`), the inline phase rule and the single-blocker `.find` in
|
||||
supervision (`:5998-6012`), the blocker `continue` (`:6229-6233`), the
|
||||
freeze busy check (`:5934-5942`), the one-live-transaction-per-target
|
||||
validation (`:3245-3254`), and the "target incumbent changed after
|
||||
transaction creation" text (`:7779`).
|
||||
|
||||
### Cut 2: the two slots and the handover
|
||||
|
||||
Each target has one **incumbent slot**: its `AdmittedGeneration`, the body
|
||||
running it, its write lease and its stable-route membership. Each live
|
||||
transaction has its own **candidate slot**: its incarnation key, activation,
|
||||
candidate port and unit.
|
||||
|
||||
- **Owner.** Before any transaction takes the target, continuity supervision
|
||||
owns the incumbent slot: it alone decides to mint a restart. From the
|
||||
handover CAS until Complete, the one transaction that took the target owns
|
||||
it, fencing, leasing, routing and committing. Its commit hands the slot
|
||||
back to supervision as a new generation. A transaction owns its own
|
||||
candidate slot from mint or freeze until it is terminal. A pre-handover
|
||||
deployment owns nothing else.
|
||||
- **The handover.** `Engine::take_target` is the only writer of a phase entry
|
||||
into `takes_target_at()` (Fencing, or Starting for stop-then-start), for
|
||||
both kinds. It is one CAS. It expects the transaction's own envelope, the
|
||||
target's admitted envelope (the one `incumbent_generation_id` names, or
|
||||
absent), and every other non-terminal transaction of the target, each
|
||||
unchanged, and it writes only the transaction. Of two transactions
|
||||
entering from one view, exactly one CAS succeeds. The loser's next step
|
||||
re-reads: a deployment gate-waits for the owner, and a continuity aborts
|
||||
itself as superseded.
|
||||
- **Inputs.** Continuity: as cut 1, plus whether any transaction owns or
|
||||
drains the target, or a continuity is live for it (the mint gate). A
|
||||
pre-handover deployment: its plan, its brake, and the admitted generation as
|
||||
it is now, which it rebases onto. The handover: the target's records above.
|
||||
- **Outputs.** The handover writes the new phase. A rebase rewrites
|
||||
`incumbent_generation_id` and clears the evidence proven against the old
|
||||
incumbent: `warming`, `route_preflight`, `isolation`. A superseded continuity
|
||||
writes its own pre-fencing abort.
|
||||
- **Derived state.** `owns_target()` is derived from the phase and
|
||||
`takes_target_at()`. `drains_incumbent()` is derived from the post-commit
|
||||
cleanup. The freeze refusal is derived from "a live Deploy for this
|
||||
target". `incumbent_generation_id` on a pre-handover deployment is the
|
||||
generation its evidence was proven against, not a pin.
|
||||
- **Forbidden writers.** No path enters `takes_target_at()` except
|
||||
`take_target`. Continuity supervision aborts nothing, and mints nothing
|
||||
while another transaction owns or drains the target or a continuity is
|
||||
live. A deployment never aborts because another transaction owns the
|
||||
target: it gate-waits (B18). A continuity is never rebased. Freeze reads no
|
||||
continuity state. No new persistent record decides ownership: it is the
|
||||
phase.
|
||||
- **Shared paths.** Candidate-then-promote's Warming-to-Fencing and
|
||||
stop-then-start's Sealing-to-Starting go through one phase-entry seam,
|
||||
which calls `take_target` exactly when the entered phase is
|
||||
`takes_target_at()`. Deployments and continuities use the same CAS. A
|
||||
single-target and a `profile:` command use the same freeze rule.
|
||||
|
||||
Demotions, named: `incumbent_generation_id` is no longer a pin before the
|
||||
handover; it is derived from the admitted generation by the rebase.
|
||||
`blocks_new_target_mutation` is no longer an owner; it splits into
|
||||
`is_live`, `owns_target` and `drains_incumbent`, each read by the consumer
|
||||
that asks that question. The freeze busy check is no longer a
|
||||
target-ownership gate; it guards only one live deployment per target.
|
||||
|
||||
Demotions, named: `restarts_exhausted` is no longer an owner of anything. It
|
||||
is renamed `restarts_page`, and it opens and holds the incident. The
|
||||
ContinuityExhausted incident was never read as "stopped", and there is now no
|
||||
stopped state for it to stand for. The "free for a deployment" report is
|
||||
deleted: every target is free for a deployment between attempts (B10).
|
||||
|
||||
## 4. Rationale
|
||||
|
||||
The fault is a split brain, not a missing rule. Continuity's "I have stopped"
|
||||
was written in two places, a durable incident and a log message, and neither
|
||||
was what the ownership decision read: the yield re-derived "stopped" from a
|
||||
clock. The withdrawn first spec moved "stopped" into control state. The
|
||||
operator's ruling is better, and it is also smaller: there is no stopped
|
||||
state. The window it was derived from becomes a pager. Every case the stop
|
||||
existed for is then handled by an owner that already exists. A broken release
|
||||
is replaced by a deployment, which gets in through the gap after each
|
||||
attempt (B10) and is never yielded (B11). A host-dependency outage costs one
|
||||
attempt every five minutes or so, and pages.
|
||||
|
||||
Why the wait is anchored at "found dead and free" rather than at the mint:
|
||||
the anchor is the door. A mint-anchored wait can be swallowed by a long
|
||||
attempt, and then no tick ever sees the target free (B10). Kubelet anchors
|
||||
CrashLoopBackOff on `FinishedAt` for the same reason, so this is the
|
||||
established spelling, not a new one. A streak is stored, not recomputed from
|
||||
the paging log. The log holds only six entries and slides by the hour, while
|
||||
the streak must outlive both. Reading the backoff from the window count was
|
||||
today's spelling, and it is exactly the mutant the cut's tests must kill.
|
||||
|
||||
The reset, "admitted after the last charged attempt and ran at least 600 s
|
||||
before it was found dead", needs no new writer. Supervision observes the
|
||||
admitted generation on every pass, so the first pass that finds it dead is
|
||||
within one poll of the death. A death hidden behind a live transaction or an
|
||||
unobservable host overstates the run, and so resets early. That is the safe
|
||||
side, because it shortens one wait.
|
||||
|
||||
The reboot invariant needs evidence of what happened, not a guess. "Unit
|
||||
absent" alone is not a reboot, because a cleanly stopped transient unit is
|
||||
collected too. "Host booted after admission" alone is not a loss, because
|
||||
the unit might have been started this boot. The systemd driver owns both
|
||||
facts (a `systemctl show` that returns nothing, and the `/proc/stat` btime).
|
||||
So it answers one question, `lost_to_host_reboot`, and supervision keeps the
|
||||
budget. The once-per-admission bound is still needed under never-stop. A
|
||||
failing re-materialization leaves the same admitted generation, unit absent,
|
||||
booted after admission. Without the bound it would repeat with no step, no
|
||||
wait and no page, every tick. The clock comparison is safe: admission
|
||||
happens at least one full transaction, tens of seconds, after boot.
|
||||
|
||||
What cut 1 deliberately does not fix. A dependency failure still grows the
|
||||
streak (cut 3). A dead incumbent stays down while a pre-fencing deployment
|
||||
builds over it (cut 2). A dead generation is still route-supervised and
|
||||
topology-refreshed every tick (cut 4). Paging residual: the cap gives 3600 /
|
||||
(300 + D) attempts an hour, where D is the attempt's duration, so a crash
|
||||
loop whose attempts last longer than 300 s settles under 6 an hour. Its
|
||||
incident opens in the first hour's burst and stays open while the release
|
||||
never runs, but a release that runs for one pass after each restart can see
|
||||
it close and not reopen. That is the operator's threshold, applied as ruled.
|
||||
It is recorded here, not changed.
|
||||
|
||||
Replayed against 2026-10-18 with only cut 1. The reboot loss at 11:46 is
|
||||
re-materialized at once, with no step. It fails on the nginx outage. The
|
||||
next attempt resets the streak, because the generation had run for hours,
|
||||
and then waits 5, 10, 20, 40, 80, 160 s, and 300 s after that. The incident
|
||||
opens within the first hour and stays open (the release never runs), so the
|
||||
operator is paged during the outage, not at 20:24. The `idunn up` at 20:24
|
||||
freezes in the next gap. Continuity defers to it, nothing aborts it, and it
|
||||
fences the absent incumbent as nothing and admits the new release.
|
||||
|
||||
Why cut 2 is shaped this way. The handover is the phase, guarded by one CAS
|
||||
over the target's records, rather than a new per-target "owner" record. A
|
||||
record would be a second spelling of what the phase already says, and it
|
||||
would need its own lifecycle, crash repair and migration. A CAS that reads
|
||||
every other live transaction of the target is enough because a newly minted
|
||||
or frozen transaction starts at Sealing and cannot own anything (B16).
|
||||
The rebase clears exactly the evidence that names an incumbent (B15): warming
|
||||
names its lease, preflight its route and isolation its workload. Keeping any
|
||||
of them would let the deployment fence one generation with proofs about
|
||||
another. A deployment waits in Warming while another transaction owns the
|
||||
target, because Warming's steps read the incumbent that owner is fencing, and
|
||||
a failed pre-fencing step is an abort (B18). Sealing and Starting keep
|
||||
running, so the long part, the build, still overlaps. A continuity that loses
|
||||
the target is aborted, not parked: it restarts a generation that is about to
|
||||
be replaced, and while it lives it blocks continuity of the generation that
|
||||
replaces it.
|
||||
|
||||
Prior art on question `fencing-over-owning-continuity` (wait for a
|
||||
continuity that already owns the target, or preempt it), from
|
||||
`docs/research/restart-budget-prior-art.md` at `74a7008`. It leans the other
|
||||
way from the recommendation, and the recommendation says why. Kubernetes'
|
||||
rolling update scales old ReplicaSets down unhealthy pods first, so that
|
||||
"unhealthy replicas will block deployment" does not happen (§1.4,
|
||||
`rolling.go`). That is preemption of a failing old replica. Nomad separates
|
||||
the two by layer: the client restarts old allocations under their own policy,
|
||||
and the server reconciler places the new ones (§2.3). Neither waits on an
|
||||
old replica's restart. But deleting a pod is the kubelet's ordinary teardown.
|
||||
In Idunn, preempting a continuity past its own Fencing means a deployment
|
||||
driving another transaction's post-fencing abort. That abort revokes a lease
|
||||
and restores a projection, and it is the path that wedged Ghostlight (B7).
|
||||
Waiting is bounded by the owner's own phase deadlines (`PhaseDeadline`,
|
||||
`:938-960`) and adds no new abort path. Recommendation: wait, at medium
|
||||
confidence. Preempt is the better answer once post-fencing aborts are known
|
||||
not to wedge, which is follow-up `ghostlight-abort-lease-before-stop`.
|
||||
|
||||
Residuals, recorded and not fixed. A wedged post-fencing abort of a
|
||||
continuity (B7, follow-up `ghostlight-abort-lease-before-stop`) holds a
|
||||
waiting deployment as long as it holds the target. A continuity superseded
|
||||
after its mint stays charged; that lengthens one wait for a generation that
|
||||
is being replaced. Stop-then-start has no control-plane fixture (B17), so in
|
||||
cut 2 its edge is pinned structurally (one seam, one call) and by a unit test
|
||||
of the phase mapping, not by an engine run. raven-muninn uses it, so the
|
||||
engine test is owed before cut 5.
|
||||
|
||||
## 5. Cut sequence
|
||||
|
||||
1. **c1-never-stop**: spec in the mind (specs-with-no-report recipe). Depends
|
||||
on nothing.
|
||||
2. **c2-phase-split** (`handover-phase-split`): spec
|
||||
`cut-c2-phase-split.r1` in the mind, branch `idunn/reincarnation-c2` from
|
||||
`9a0a6a2`. Continuity restarts the incumbent while a pre-fencing
|
||||
deployment builds. The ownership predicates split (B13, B14). The handover
|
||||
is one CAS at `takes_target_at()`. A pre-handover deployment rebases onto
|
||||
whatever is admitted, and a superseded continuity aborts itself. Freeze
|
||||
refuses only a second live deployment for a target, which dissolves
|
||||
follow-up `profile-deploy-joint-gap`. It folds in the test-only follow-up
|
||||
`c1-sustained-reset-test-reach`. Open question:
|
||||
`fencing-over-owning-continuity`. Depends on cut 1 (merged at `1da150a`).
|
||||
3. **Dependency failures do not grow the backoff**
|
||||
(`budget-never-stop`): an attempt that failed because of a declared
|
||||
dependency (Odin's stable route, the target's own stable route
|
||||
listener), as Idunn itself observed it, leaves the streak unchanged.
|
||||
Waits on follow-up `odin-route-record-stale`, because the record it
|
||||
would read is not trustworthy, and on cut 1.
|
||||
4. **A dead generation is not supervised as alive** (target invariant 5):
|
||||
observe the workload before route supervision and topology refresh, so
|
||||
that a dead admitted generation gets no topology authentication and no
|
||||
lease-exactness challenge. Its non-binding correlation is reported once,
|
||||
not every tick. Waits on cut 1, because it rewrites the same function.
|
||||
5. **Closing proof on Yggdrasil**: an operator-approved install of the merged
|
||||
cuts. streampixels-service survives the install, and `idunn status` is
|
||||
checked before and after. Waits on 1-4 and on follow-up
|
||||
`stop-first-handover-fixture` (raven-muninn is stop-then-start, B17).
|
||||
|
||||
Ops, out of scope: nginx boot order (`gamecult-ops`), and retention of the
|
||||
roughly 130 dead transient units.
|
||||
|
||||
## Installed
|
||||
|
||||
Cut 1 (`1da150a`, merge of `idunn/reincarnation` at `ffb4293`) was installed on
|
||||
Yggdrasil on 2026-10-19 at 08:34:45 UTC with the operator's OK. The binary is
|
||||
`/usr/local/bin/idunn` sha256 `f0e58308…`, built in `/srv/build/idunn-1da150a`
|
||||
(`cargo test --lib`: 414 passed). The pre-install store backup is
|
||||
`/root/idunn-store-backup-*-pre-reincarnation-c1`. Restoring it means restoring
|
||||
its stores and `idunn.running` together, because the cb60965 binary cannot read
|
||||
`idunn.target_supervision.v2`. At first open Idunn logged
|
||||
"migrated 5 control record(s) … idunn.target_supervision.v2". streampixels-service
|
||||
kept running through the restart (unit 73ca40ba, PID 968752, NRestarts 0). The
|
||||
catalog returned 200, and 4443/udp was listening.
|
||||
|
||||
## Installed: cut 2
|
||||
|
||||
Cuts c2-phase-split, c2b-owner-deadlines, c2c-post-fence-forward and
|
||||
c2d-continuity-yields (branch tip `8760a4f`) were installed on Yggdrasil on
|
||||
2026-10-20 at 00:34:06 UTC, with the operator's OK.
|
||||
|
||||
- **The hotfix before it.** `ec45906` (`bounded_detail`) was installed on its
|
||||
own on 2026-10-19 at 17:32 UTC. It unwedged tx-f978ab86.
|
||||
- **Build.** `/srv/build/idunn-8760a4f`; `cargo test --lib` passed 471.
|
||||
- **Backup.** `/root/idunn-store-backup-*-pre-reincarnation-c2` holds the
|
||||
stores, `incidents.cc` and the binaries. A rollback must restore them
|
||||
together: older binaries refuse two live transactions on one target, and
|
||||
they refuse the `operator-required` incident condition.
|
||||
- **First boot.** The boot lift returned exactly tx-039b91e9 (ghostlight) and
|
||||
tx-4eaed669 (odin) to forward retry, as the Soul install gate predicted.
|
||||
Odin's continuity, wedged in Routing since the 2026-10-18 reboot, committed
|
||||
4 s later. Generation tx-4eaed669 is admitted, its route is healthy, and the
|
||||
same PID 2571 still runs. Ghostlight's continuity is retrying forward in
|
||||
AwaitingReady. Its daemon's readiness is a ghostlight-side fault
|
||||
(`Permission denied` reading its session-wrapping key).
|
||||
- **Live processes.** Every live MainPID was the same before and after the
|
||||
install. The streampixels catalog returned 200, and 4443/udp was listening.
|
||||
95
docs/reincarnation-target.md
Normal file
95
docs/reincarnation-target.md
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
# Reincarnation after reboot: target
|
||||
|
||||
Status: target, Self, 2026-10-18 (campaign `idunn-reincarnation` in Eureka's
|
||||
mind). It states the ends. The body facts, model page and cut sequence will
|
||||
live in `reincarnation-map.md`.
|
||||
|
||||
## Why
|
||||
|
||||
Yggdrasil rebooted at 11:46 UTC on 2026-10-18. Admitted bodies run as
|
||||
transient systemd units under `/run/systemd/transient`, and the reboot wiped
|
||||
them all. nginx failed at boot, because it bound `10.77.0.1:17871` before
|
||||
`wg-gamecult` carried that address. Every stable route, Idunn's own
|
||||
`127.0.0.1:8831-8833` included, stayed down until a manual
|
||||
`systemctl restart nginx` at about 19:46.
|
||||
|
||||
What Idunn did:
|
||||
|
||||
- From 11:50 onward continuity spent streampixels-service's whole restart
|
||||
budget (6 per hour) and logged "The target is free for a deployment to
|
||||
replace it: systemd unit … is absent". The log repeated in hourly bursts as
|
||||
the window slid.
|
||||
- When nginx returned, ghostlight and streampixels-web were re-incarnated, and
|
||||
streampixels-service was not.
|
||||
- An operator-approved `idunn up streampixels-service` (tx-e69f5402) sat in
|
||||
Sealing for about 15 minutes. It then failed with "admitted incumbent failed
|
||||
before candidate fencing; deployment yielded to continuity". Continuity had
|
||||
freed the target for a deployment. The deployment handed it back to
|
||||
continuity. Neither actuated.
|
||||
- At 20:46:43 continuity re-incarnated the service under generation
|
||||
tx-0804185a, once the window had slid to 4/6. So the outage was not ended by
|
||||
the deployment the operator approved. The restart window bounded it, a full
|
||||
hour after its cause had cleared.
|
||||
- Throughout, several times a second: "preserved admitted … after rejecting
|
||||
topology observation: topology correlation does not bind the current
|
||||
activation", and "rejected admitted streampixels-service route continuity:
|
||||
admitted process write lease is no longer exact".
|
||||
|
||||
## Invariants
|
||||
|
||||
1. **A reboot is not a fault of the body.** If an admitted body's unit is
|
||||
absent because the host lost its transient units, continuity
|
||||
re-materializes it from the installed body. It needs no deployment, no
|
||||
operator, and no part of the restart budget for that absence.
|
||||
2. **The restart budget measures the body.** A failure caused by a shared host
|
||||
dependency, such as nginx, the stable route's listener or Odin, never
|
||||
charges a target's restart budget. A dependency outage can never exhaust
|
||||
the budget of every target behind it.
|
||||
3. **Exactly one authority owns each slot at every instant, and nothing
|
||||
waits on an owner that has stopped.** Continuity never stops (operator,
|
||||
2026-10-18: capped backoff, and the 6 per hour count only pages). It never
|
||||
aborts a deployment. Before Fencing a deployment owns only its candidate,
|
||||
and continuity owns the incumbent slot. At Fencing the deployment takes the
|
||||
target and fences whatever is admitted then; an absent incumbent fences as
|
||||
nothing. Every continuity attempt leaves a free gap in which a queued
|
||||
deployment can be admitted. There is no reachable state in which
|
||||
continuity waits for a deployment while that deployment waits for
|
||||
continuity. A negative test proves it.
|
||||
4. **Continuity is daemon survival; deployment is change** (`F:\Projects\CLAUDE.md`).
|
||||
Restarting the admitted installed body never needs a deployment and is
|
||||
never gated by a deployment brake. A deployment brake gates only artifact,
|
||||
revision, configuration, schema, unit or authority changes.
|
||||
5. **Rejections of stale observations are not a log stream.** An observation
|
||||
that cannot bind the current activation is either resolved by the owner
|
||||
that can refresh it, or reported once as an incident. It is never
|
||||
re-rejected several times a second.
|
||||
6. **Existing invariants hold.** `route-continuity-target.md` 1 to 7 (in
|
||||
particular no actuation from observation, bounded backoff, every
|
||||
post-fencing phase ends) and `authority-map.md` (Idunn's own recovery
|
||||
depends on no managed target).
|
||||
|
||||
## Scope
|
||||
|
||||
In:
|
||||
|
||||
- Idunn's continuity of admitted bodies across a reboot: absent units, the
|
||||
restart budget and what charges it.
|
||||
- The handover between continuity and deployment, in both directions.
|
||||
- Classifying the topology-correlation and write-lease rejections: the same
|
||||
fault, or a separate one with its own cut.
|
||||
- Re-admitting streampixels-service on Yggdrasil. It is running again under
|
||||
tx-0804185a, and the closing proof checks that it survives the fix's
|
||||
install.
|
||||
|
||||
Out, and not consumers of this change:
|
||||
|
||||
- **nginx's boot order on Yggdrasil.** It is host configuration, owned by
|
||||
`gamecult-ops`. It gets its own operator-approved change and record, not an
|
||||
Idunn cut.
|
||||
- **Ghostlight's degraded route** (53 consecutive failures, incident open) is
|
||||
in scope only if the map shows the same fault. Otherwise it is a follow-up.
|
||||
- **Heimdall's onboarding failures and its environment.**
|
||||
- **Retention of the ~130 dead transient units** on Yggdrasil. It is an ops
|
||||
follow-up already recorded under route continuity.
|
||||
- **Luna's 2026-09-27 weekend patches** are not foundation to build on. Where
|
||||
this map meets them, it cuts them.
|
||||
124
docs/research/restart-budget-prior-art.md
Normal file
124
docs/research/restart-budget-prior-art.md
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
# Restart-budget and handover prior art (Eyes facts pass)
|
||||
|
||||
Serves two operator questions in Eureka campaign `idunn-reincarnation` (session idunn-reincarnation-2026-10-18):
|
||||
|
||||
- `idunn-reincarnation:question:budget-charge` (what charges a target's continuity restart budget)
|
||||
- `idunn-reincarnation:question:handover-pre-fencing` (who acts when the incumbent dies while a deployment is before Fencing)
|
||||
|
||||
Facts and evidence pointers only. No recommendation. Gathered 2026-10-18 from official docs and upstream source (main branches at fetch time). Quotes are under 15 words. "Not found" means the primary source read does not state it, not that it is false.
|
||||
|
||||
## 1. Kubernetes
|
||||
|
||||
### 1.1 CrashLoopBackOff schedule, cap, reset (kubelet)
|
||||
|
||||
- Docs, pod lifecycle, "Container restarts": backoff is "exponential backoff delay (10s, 20s, 40s, ...)", "capped at 300 seconds (5 minutes)". Reset: after the container has run "for 10 minutes without any problems". https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/
|
||||
- Source pkg/kubelet/kubelet.go (https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/kubelet.go): `initialCrashLoopBackOff = time.Second * 10`; max is `MaxCrashLoopBackOff = v1beta1.MaxContainerBackOff` (value lives in the kubelet config API package, not re-read; docs say 300 s). Under alpha gate `ReduceDefaultCrashLoopBackOffDecay`: initial 1 s, max 60 s. Gate `KubeletCrashLoopBackOffMax` lets the config set the max (`CrashLoopBackOff.MaxContainerRestartPeriod`).
|
||||
- Reset rule in code: `klet.crashLoopBackOff.HasExpiredFunc` returns `eventTime.Sub(lastUpdate) > 600*time.Second` (kubelet.go ~line 1102). I.e. the entry resets when more than 600 s have passed since the entry was last updated (last restart event). The client-go default is 2*maxDuration; the kubelet overrides it.
|
||||
- Growth: client-go `Backoff.Next` doubles (`delay := entry.backoff * 2`) and caps with `min(delay, p.maxDuration)` (https://github.com/kubernetes/client-go/blob/master/util/flowcontrol/backoff.go).
|
||||
- Window anchor: the exited container's `FinishedAt`; keyed per pod+container; event text "Back-off restarting failed container" (pkg/kubelet/kuberuntime/kuberuntime_manager.go, `doBackOff`).
|
||||
- The backoff table is an in-memory object created in `NewMainKubelet` (`flowcontrol.NewBackOff(base, boMax)`); no persistence found in the code read. Not found: any doc statement that a node reboot does or does not reset or count toward it. Docs only say restart policy "only applies to restarts that make replacement containers inside the same Pod and running on the same node." Docs on kubelet restarts: pods "continue to run even during the restart" (kubelet restart, not node reboot).
|
||||
|
||||
### 1.2 Probes
|
||||
|
||||
Docs (pod lifecycle "Container probes"; https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/):
|
||||
|
||||
- Liveness: past the failure threshold "the kubelet restarts that container" (per pod restartPolicy).
|
||||
- Readiness: the EndpointSlice controller "removes the Pod's IP address from the EndpointSlices"; the container is not restarted.
|
||||
- Startup: until it succeeds "Kubernetes does not execute liveness or readiness probes"; on failure "the kubelet kills the container", subject to restart policy.
|
||||
- Backoff keys on a container having exited (`doBackOff` reads the last exited container), so crash, liveness kill and startup-probe kill all reach it; readiness failure leaves the container running and does not. The docs do not describe backoff per probe type.
|
||||
|
||||
### 1.3 Dependency down
|
||||
|
||||
Docs, same page, "Dependencies and init containers": init containers "run only once (if successful), during Pod startup"; failed init containers are restarted by the kubelet when pod restartPolicy is OnFailure or Always. Readiness gates: pod is ready only when all `readinessGates` conditions are True. No general documented rule for a main container whose external dependency is down: an app that exits is restarted under backoff; an app that stays up and fails readiness is only removed from endpoints.
|
||||
|
||||
### 1.4 RollingUpdate and the old ReplicaSet
|
||||
|
||||
- Deployment docs (https://kubernetes.io/docs/concepts/workloads/controllers/deployment/): "Once new Pods are ready, old ReplicaSet can be scaled down further"; at the end "all old ReplicaSets is scaled to 0". maxUnavailable and maxSurge default 25%. progressDeadlineSeconds default 600; when exceeded, "Kubernetes takes no action on a stalled Deployment" beyond the Progressing=False condition.
|
||||
- ReplicaSet docs: it fulfills its purpose by "creating and deleting Pods as needed to reach the desired number." https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/
|
||||
- Deployment controller source, pkg/controller/deployment/rolling.go (https://github.com/kubernetes/kubernetes/blob/master/pkg/controller/deployment/rolling.go): `reconcileOldReplicaSets` computes `minAvailable = replicas - maxUnavailable`, runs `cleanupUnhealthyReplicas` first (comment: "Clean up unhealthy replicas first, otherwise unhealthy replicas will block deployment"), then `scaleDownOldReplicaSetsForRollingUpdate`. The Deployment controller decides old-RS replica counts during rollout, scaling unavailable ones down first.
|
||||
- Layering: restarts of a dying container are done by the kubelet (restartPolicy with backoff), not the ReplicaSet. The ReplicaSet recreates a pod only when the pod object is gone or terminal. Not found: a statement in those words that the old ReplicaSet keeps recreating its dying pods until scaled down.
|
||||
|
||||
## 2. Nomad
|
||||
|
||||
### 2.1 restart block (https://developer.hashicorp.com/nomad/docs/job-specification/restart)
|
||||
|
||||
- `attempts` restarts allowed within `interval`; interval "starts when the first task starts". Defaults: service/system attempts 2, interval 30m, delay 15s, mode fail; batch attempts 3, interval 24h, delay 15s, mode fail. Delay has up to 25% jitter.
|
||||
- `mode = "fail"` (default): after attempts are used the allocation is marked failed and goes to `reschedule`. `mode = "delay"`: "The task waits another full interval before restarting"; the allocation is never failed by restarts.
|
||||
- Source client/allocrunner/taskrunner/restarts/restarts.go (https://github.com/hashicorp/nomad/blob/main/client/allocrunner/taskrunner/restarts/restarts.go): in-memory `RestartTracker` with `count` and `startTime` set at construction (`time.Now()`); a new interval resets count; `count > Attempts` returns TaskNotRestarting in fail mode, or TaskRestarting with delay to interval end in delay mode. Constructed in `NewTaskRunner` (task_runner.go ~line 468) from the alloc's job type and the task group's policy. `SetKilled` returns TaskKilled without counting. A restart triggered with failure=false (operator/API) returns TaskRestarting without consulting the policy. Unrecoverable start errors return TaskNotRestarting.
|
||||
- Client restart: restarts.go keeps no persisted counter. `TaskRunner.Restore` (task_runner.go ~line 1244) restores local state and the driver handle (`RecoverTask`); if the handle cannot be restored for a non-system job the task "will not run until server is contacted" (`waitOnServers`). The tracker is rebuilt at task-runner construction. Persisted `TaskState` carries restart counts separately; `GetState` does not read them.
|
||||
|
||||
### 2.2 reschedule block (https://developer.hashicorp.com/nomad/docs/job-specification/reschedule)
|
||||
|
||||
- Applies when an allocation fails to set up or its tasks fail more than `restart` allows. Drains trigger migrations and lost nodes trigger replacements, so reschedule is ignored in both cases. Not applied to system/sysbatch.
|
||||
- Service defaults: `delay_function = exponential`, delay starting 30s, `max_delay` 1h, `unlimited = true`. Batch defaults: 1 attempt per 24h, constant 5s, unlimited false. `delay_function`: constant, exponential (doubling), fibonacci. `max_delay` ignored for constant. Minimum delay 1 s; docs warn very low delay with `unlimited = true` can cause runaway rescheduling.
|
||||
- With `unlimited = true`: "The `progress_deadline` in the `update` block still applies".
|
||||
|
||||
### 2.3 Deployments and restarts of old allocations
|
||||
|
||||
- Update block (https://developer.hashicorp.com/nomad/docs/job-specification/update): `max_parallel` 1, `min_healthy_time` 10s, `healthy_deadline` 5m, `progress_deadline` 10m ("the deployment is marked as failed"), `auto_revert` false, `canary` 0. The page says nothing on restarts of old allocations. Not found: docs on deployment/restart interaction.
|
||||
- Restart is client-local ("Restarts happen on the client that is running the task"); deployment is server-side. Reconciler source (scheduler/reconciler/reconcile_cluster.go, `placeAllocs`): when the deployment is not place-ready (paused/failed/canarying), rescheduling of failed allocs still proceeds, with comment about avoiding "odd user experiences", except where the failed alloc is part of the failed deployment (`partOfFailedDeployment` skips it).
|
||||
|
||||
## 3. systemd
|
||||
|
||||
### 3.1 Start limit (https://www.freedesktop.org/software/systemd/man/latest/systemd.unit.html)
|
||||
|
||||
- A unit started more than `StartLimitBurst=` times within `StartLimitIntervalSec=` may not start again until the interval passes. Applies to all starts, including manual, not only `Restart=`; it is checked after condition checks, so failing conditions do not count. Units with `Restart=` that hit it are no longer restarted automatically; `systemctl reset-failed` flushes the counter. `StartLimitAction=` default none. Counters are cleared when the unit is unloaded.
|
||||
- Source: `unit_test_start_limit` in src/core/unit.c (`ratelimit_below(&u->start_ratelimit)`; on hit logs "Start request repeated too quickly", returns -ECANCELED); `unit_reset_failed` calls `ratelimit_reset`.
|
||||
|
||||
### 3.2 Job result "dependency" and the start limit
|
||||
|
||||
- src/core/unit.c `unit_start` order: conditions (-ECOMM), asserts (-EPROTO), `unit_verify_deps` (-ENOLINK, "The necessary dependencies are not fulfilled"), then `UNIT_VTABLE(u)->test_startable`. For services, `service_test_startable` (src/core/service.c ~6381) calls `unit_test_start_limit`. A start failing on unfulfilled dependencies returns before the start-limit counter is touched.
|
||||
- src/core/job.c: `-ENOLINK` maps to `JOB_DEPENDENCY` (`job_run_and_invalidate`); `job_fail_dependencies` also finishes dependent START/VERIFY_ACTIVE jobs with `JOB_DEPENDENCY` ("Dependency failed for %s."). `n_failed_jobs` increments only for JOB_FAILED, JOB_INVALID, JOB_FROZEN, JOB_CONCURRENCY. JOB_DEPENDENCY triggers `OnFailure=` handling.
|
||||
- Requires=: if the required unit fails to activate and the dependent has After= on it, the dependent is not started; without After= it may still start. Wants=: failure "has no impact on the validity of the transaction as a whole." Dependent is stopped when the required unit is explicitly stopped or restarted; not stopped if the required unit stops on its own.
|
||||
|
||||
### 3.3 Restart= and dependency failure
|
||||
|
||||
- systemd.service(5) (https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html): the restart table is by exit cause of the service's own process (exit code, signal, timeout, watchdog, OOM); no row for dependency failure. Page: "service restart is subject to unit start rate limiting." Not found: a doc statement on whether Restart= acts on a dependency failure. From the code above, a dependency-failed start job finishes with JOB_DEPENDENCY and never passes through the service's own failure state (inference, not an explicit doc statement).
|
||||
- Backoff: `RestartSec=` default 100ms; `RestartSteps=` with `RestartMaxDelaySec=` give exponential growth to a cap (doc example 10s, 20s, 40s, 80s, 160s, 160s...). `RestartMaxDelaySec` default infinity; both must be set. `RestartRandomizedDelaySec=` (v262) adds jitter.
|
||||
|
||||
### 3.4 Transient units across reboot
|
||||
|
||||
- systemd.unit(5) path table (man/systemd.unit.xml ~lines 403, 465): "Dynamic configuration for transient units" lives in `/run/systemd/transient` (system) and `$XDG_RUNTIME_DIR/systemd/transient` (user). That `/run` is volatile is not stated in this file.
|
||||
- systemd-run docs: failed transient units are "kept in memory until the user explicitly resets their failure state"; `--collect` is `CollectMode=inactive-or-failed`. systemd-run and https://systemd.io/TRANSIENT-SETTINGS/ do not mention reboot. Not found: an explicit sentence on survival across reboot.
|
||||
|
||||
## 4. Other supervisors
|
||||
|
||||
### 4.1 Erlang/OTP supervisor (https://www.erlang.org/doc/apps/stdlib/supervisor.html, https://www.erlang.org/doc/system/sup_princ.html)
|
||||
|
||||
- `intensity` (MaxR) and `period` (MaxT), defaults 1 and 5. If more than MaxR restarts occur within the last MaxT seconds, the supervisor "terminates all the child processes and then itself" (exit reason `shutdown`). The parent supervisor then restarts it or itself terminates; failure escalates up the tree. Stated purpose: prevent a process from repeatedly crashing for the same reason and being restarted indefinitely.
|
||||
- Restart types: `permanent` (always), `transient` (only abnormal exit), `temporary` (never). Every restart the strategy performs counts toward intensity; the docs describe no per-cause charging and no backoff delay. Multi-level: total restarts before the top gives up is roughly the product of intensities above the failing child. Strategy `rest_for_one` restarts the failed child and those started after it.
|
||||
|
||||
### 4.2 runit and s6
|
||||
|
||||
- runit `runsv` (https://smarden.org/runit/runsv.8.html): if ./run exits immediately runsv "waits a second" before restarting; no rate limit or retry cap described; keeps restarting unless told otherwise (`o`, `d`, `x`).
|
||||
- s6-supervise (https://skarnet.org/software/s6/s6-supervise.html): "There is a minimum 1-second delay between two ./run spawns." A ./finish script exiting 125 makes supervise stop restarting; otherwise it restarts indefinitely. The give-up decision is delegated to a user script, not a built-in counter.
|
||||
|
||||
### 4.3 Borg (Verma et al., EuroSys 2015; https://research.google/pubs/large-scale-cluster-management-at-google-with-borg/, PDF text read)
|
||||
|
||||
- Borglet "restarts them if they fail"; Borg "monitors the health-check URL and restarts tasks that do not respond promptly" (sections 2.6, 3.3).
|
||||
- Section 4 (Availability): Borg "automatically reschedules evicted tasks"; "limits the allowed rate of task disruptions and the number of tasks from a job that can be simultaneously down"; rate-limits finding new places for tasks from unreachable machines because it cannot tell machine failure from network partition; "avoids repeating task::machine pairings that cause task or machine crashes".
|
||||
- Section 2.3 (updates): "Updates are generally done in a rolling fashion", with a limit on task disruptions an update causes; changes that would exceed it "are skipped".
|
||||
- Not found in the Borg paper: a crash-loop backoff schedule, a restart budget, or any distinction between charging task faults and dependency faults. Omega paper not fetched.
|
||||
|
||||
## 5. Facts grouped by the brief's sub-topics
|
||||
|
||||
- (a) Charging only body faults: Kubernetes backoff keys on container exit regardless of cause (readiness failure does not exit and does not advance it). systemd runs dependency verification (-ENOLINK, JOB_DEPENDENCY) before the start-limit test, and JOB_DEPENDENCY is not in `n_failed_jobs`. Nomad's tracker counts any recoverable task failure or start error, with no dependency classification. OTP counts every restart. Borg avoids repeating task::machine pairings that cause crashes (placement, not budget).
|
||||
- (b) Give up vs capped forever: OTP gives up and escalates. Kubernetes caps at 300 s with no give-up. Nomad restart fail-mode gives up locally, then reschedule continues (service default unlimited, exponential to 1 h); delay-mode never fails. systemd start limit stops auto-restart until the interval passes or reset-failed; `RestartMaxDelaySec` default infinity. runit restarts forever at 1 s; s6 restarts forever unless finish exits 125.
|
||||
- (c) Rollout vs restart supervisor: Kubernetes splits by layer (kubelet restarts containers; ReplicaSet maintains pod count; Deployment controller scales old RS during rollout). Nomad splits by node vs server (client restart tracker vs server reconciler; reconciler skips rescheduling allocs of a failed deployment). Borg bounds update-induced disruptions with a rate limit.
|
||||
|
||||
## 6. Memory claims vs sources
|
||||
|
||||
budget-charge:
|
||||
|
||||
1. "Kubernetes separates readiness from liveness; only liveness feeds CrashLoopBackOff": CORRECTED. Readiness never restarts and never advances backoff (confirmed). Backoff keys on any container exit (crash, liveness kill, startup-probe kill), so it is not fed only by liveness.
|
||||
2. "CrashLoopBackOff caps at 5 min": CONFIRMED (docs 300 s; initial 10 s, doubling). Reset: docs say 10 min healthy; code says 600 s since the entry's last update. 60 s cap only under alpha gate.
|
||||
3. "systemd does not run a unit whose required dependency failed (job result 'dependency')": CONFIRMED with qualifier: Requires= plus After= blocks the start; Requires= without After= may still start; Wants= does not block.
|
||||
4. "Nomad reschedules with capped exponential delay": CONFIRMED for service jobs (exponential from 30 s, max_delay 1 h, unlimited by default); batch default is constant 5 s, 1 attempt per 24 h.
|
||||
5. Dependency failure vs start limit (not in the question's claims, material): systemd checks dependencies before counting a start (confirmed by code order). Whether Restart= acts on dependency failure: UNSUPPORTED by docs.
|
||||
6. Whether restarts after a node reboot count toward CrashLoopBackOff: UNSUPPORTED by docs; code read shows the kubelet backoff map in memory. Nomad's restart tracker is likewise in memory and rebuilt on client restart.
|
||||
|
||||
handover-pre-fencing:
|
||||
|
||||
1. "In a Kubernetes rolling update the old ReplicaSet keeps recreating its dying pods until the rollout scales it down": PARTLY CONFIRMED, PARTLY CORRECTED. ReplicaSets do create pods to reach their desired count, and the Deployment controller scales old RS down, unhealthy first. Container restarts are the kubelet's, not the ReplicaSet's; the RS recreates only pods that are gone.
|
||||
2. "Nomad restarts old allocations by their own restart policy, independent of the deployment": PARTLY SUPPORTED. Restart is client-local using the alloc's task-group policy (code), so independent of the server-side deployment; no doc statement on the interaction. Server-side, failed allocs are rescheduled even when the deployment is not place-ready, except allocs of a failed deployment.
|
||||
3. Any source saying an aborted or stalled rollout automatically hands the slot back to the old generation: UNSUPPORTED (Kubernetes takes no action on a stalled Deployment; Nomad `auto_revert` defaults false).
|
||||
2518
docs/route-continuity-cut.md
Normal file
2518
docs/route-continuity-cut.md
Normal file
File diff suppressed because it is too large
Load diff
101
docs/route-continuity-target.md
Normal file
101
docs/route-continuity-target.md
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
# Route continuity and admission: target
|
||||
|
||||
Status: target, Self, 2026-09-29. The cut map will live in
|
||||
`route-continuity-cut.md` once Imagination has produced it. This document states
|
||||
the ends. It does not specify the means.
|
||||
|
||||
## Why
|
||||
|
||||
On 2026-09-27 and 28 an agent spent 24 hours failing to deploy StreamPixels'
|
||||
web app and service through Idunn v2. The live evidence is on Yggdrasil:
|
||||
|
||||
- Admitted route continuity rewrites and reloads nginx whenever a route
|
||||
observation is older than 30 s. The config is usually byte-identical when it
|
||||
does so (`supervise_admitted_route` → `restore_admitted_membership` →
|
||||
`reload`, since `391a7e2` on 2026-09-04). The result is ~115 reloads/h per
|
||||
routed target when healthy. Once the StreamPixels routes were failing it
|
||||
reached ~3,000/h, and 475 nginx workers were left shutting down.
|
||||
- Every reload moves nginx's UDP listener for Odin (`10.77.0.1:17871` →
|
||||
`127.0.0.1:17973`) to new workers. That breaks the CultNet RUDP sessions of
|
||||
Idunn's route observer and of every runtime-presence publisher. The
|
||||
timeouts fail the route checks, and the failed checks cause more reloads.
|
||||
Odin continuity restarts followed on 2026-09-28 at 16:26 and 21:27. Each
|
||||
left Odin refusing every provider until its new lease was picked up.
|
||||
- The web app could not be admitted without the shared Odin RUDP path. It
|
||||
published signed presence to Odin, and its HTTP snapshot route returned 503
|
||||
whenever that publication timed out.
|
||||
- The web transaction stayed in `Committing` for more than 12 hours. After
|
||||
fencing nothing has a deadline: errors are resumable forever,
|
||||
`AwaitingReady` waits forever, and no command recovers a continuity or
|
||||
stateful transaction.
|
||||
- The weekend fixes worked around symptoms. `a148802` relaxes rollback to
|
||||
demote whatever activation is projected, and `9f00e7a` authenticates Ready
|
||||
and provider receipts at admission time instead of now.
|
||||
|
||||
## Invariants
|
||||
|
||||
1. **Observation never actuates routing.** Refreshing a route observation is a
|
||||
challenge through the stable endpoint and nothing else. Idunn writes an nginx
|
||||
fragment or reloads nginx only when the fragment on disk differs from the
|
||||
admitted or candidate membership, or when a route is installed or withdrawn.
|
||||
A healthy steady state performs zero reloads.
|
||||
2. **Failure never accelerates actuation.** Repairing a failed route or unit is
|
||||
bounded and backs off. No failure mode can raise the reload rate or the
|
||||
restart rate above a fixed ceiling per target.
|
||||
3. **A web app proves itself over its own HTTP route** (operator,
|
||||
2026-09-29: "web apps prove over HTTP; Odin can observe a web app, but a web
|
||||
app has no reason to be aware of Odin"). Its admission and continuity depend
|
||||
only on Idunn, its own stable HTTP route, and its declared host inputs. A web
|
||||
app needs no Odin client, no RUDP publisher and no Odin trust anchor. Odin
|
||||
may observe it. Observation is never admission authority.
|
||||
4. **Route proof binds the exact incarnation and is fresh.** Traffic reaches only
|
||||
the admitted incarnation. The proof answers an Idunn challenge and is signed
|
||||
with the credential Idunn issued to that launch. Invariant 3 changes where the
|
||||
proof travels, not what it proves.
|
||||
5. **Every post-fencing phase has a deadline and is never a silent wait.** A
|
||||
deployment's stateless phase that passes its deadline aborts to its
|
||||
incumbent. A continuity past its deadline is left to the operator
|
||||
(operator-required incident) and retries its phase forward on a capped
|
||||
backoff, because aborting it would stop the only running copy of the admitted
|
||||
body (ruling post-fence-forward-retry), until a deployment that the
|
||||
deployment brake has released awaits the handover of its target and the
|
||||
continuity's route proof has not succeeded: then it yields through its own
|
||||
post-fencing abort, and the deployment fences what is admitted (rulings
|
||||
continuity-yields-to-deploy, c2d-yield-only-unproven-to-movable). A
|
||||
continuity that has proven its route is serving and never yields. A
|
||||
stateful deployment is left to the operator.
|
||||
6. **The admitted receipt describes the running incarnation.** Continuity
|
||||
commits update the admitted generation atomically, so the rollback path never
|
||||
meets drift between the receipt and the projection. Rollback demotes the exact
|
||||
admitted activation (`a148802` reverted). Readiness and provider checks
|
||||
authenticate current evidence at the current time (`9f00e7a` reverted).
|
||||
7. **Idunn stays independent** (existing, `authority-map.md`): Idunn starts and
|
||||
recovers its own admitted state without Odin or any managed target.
|
||||
|
||||
## Scope
|
||||
|
||||
In:
|
||||
|
||||
- Admitted route supervision.
|
||||
- nginx route actuation.
|
||||
- The HTTP route proof for web targets.
|
||||
- Post-fencing deadlines and resolution.
|
||||
- Stuck-lease recovery.
|
||||
- Reverting `a148802` and `9f00e7a`.
|
||||
- Reviewing the other weekend commits: `ae8ac45` and `29946bb` in particular.
|
||||
- The app-side cost of invariant 3 in CultLib's Idunn runtime-authority
|
||||
packages.
|
||||
- The StreamPixels web binding that proves invariant 3 end to end.
|
||||
|
||||
Out, and not consumers of this change:
|
||||
|
||||
- **The verify-transaction campaign** (`verify-transaction-cut.md`, branch
|
||||
`idunn/cut1-fix5`). It is a separate foundation. Coordinate merges, do not
|
||||
bundle.
|
||||
- **Heimdall's failed Idunn onboarding.**
|
||||
- **The Windows host actuator.**
|
||||
- **Operator-owned nginx settings** (`worker_shutdown_timeout`) and host
|
||||
retention of dead transient units and activation directories. These are ops
|
||||
follow-ups recorded in the map.
|
||||
- **StreamPixels' public cutover.** It resumes after this lands, through the
|
||||
ordinary path.
|
||||
475
docs/secret-files-cut.md
Normal file
475
docs/secret-files-cut.md
Normal file
|
|
@ -0,0 +1,475 @@
|
|||
# Binding secrets out of every printable surface: cut map
|
||||
|
||||
Status: cut map, Imagination pass 1 (Opus), 2026-09-30. Nothing has landed.
|
||||
There is no separate target document; the invariant in section 1 is the end state.
|
||||
Anchors are against these HEADs:
|
||||
- Idunn `46d9c2f`
|
||||
- gamecult-ops `62e3f05`
|
||||
- Heimdall `e8e2832`
|
||||
- StreamPixels `a49f7d8` (reference pattern only)
|
||||
|
||||
Incident: during overnight work an agent printed Heimdall's deployment-binding
|
||||
secrets into a transcript. The operator's words: "I'll rotate before prod, not too
|
||||
concerned about Anthropic having my oauth; do map the cut that'll ensure it doesn't
|
||||
happen again".
|
||||
|
||||
**Heimdall `hands/secret-files` status (Self, 2026-09-30).**
|
||||
- **Batch 5** (`efd1137..b5d7b6b`) fixed Soul pass 4's four findings:
|
||||
- F1: command-plane diagnostics are fixed text plus the error's class and code, and every provider body goes
|
||||
through `readProviderJson`.
|
||||
- F2: patron support keeps only the Bifrost status.
|
||||
- F3: the recipe test records the secrets `loadConfig` actually reads.
|
||||
- F4: `checkSchema` runs on every start.
|
||||
- **Soul pass 5: hold.** F1-F3 held. Batch 6, in Hands, fixes:
|
||||
- **S5-1.** `checkSchema` passes a DEFERRABLE key, a `UNIQUE ... INCLUDE`, a same-named non-unique replacement for
|
||||
the partial unique index, and retyped columns. Each of these breaks the store at runtime.
|
||||
- **S5-2.** The privilege check demands UPDATE on `audit_events`, which the store only INSERTs into.
|
||||
- **S5-3 (older than this batch).** `app.ts:1310` writes a live, 300 s redeemable completion code into the audit
|
||||
event `auth_completion_created`.
|
||||
- **S5-4 to S5-7.** Test gaps: canary position, the `errorIdentity` filters, pool close, spread-env reads.
|
||||
- A flaky live-Postgres test.
|
||||
- Noted, not in scope: **S5-8**, forced RLS with no policy passes the check.
|
||||
- **Batch 6** (`b5d7b6b..6c0c876`): S5-1 through S5-7 fixed, and the flaky live test fixed (template DB clone, `fixturePool`).
|
||||
Result: 241 passed, 24/24 live, 40 of 41 mutants killed. The survivor, X13, is equivalent: pg-pool destroys a
|
||||
failed client.
|
||||
- **Soul pass 6: hold.** Batch 7 is in Hands.
|
||||
- **F1.** An invalid or not-ready unique index passes the check. That breaks sign-in (42P10) and R21.1, because a
|
||||
consume takes 2 rows.
|
||||
- **F2.** A key index under a nondeterministic collation passes, and the upsert then moves one account's identity
|
||||
row to another account.
|
||||
- **F3.** The predicate normaliser is unsound: it strips parentheses and lowercases literals.
|
||||
- **F4.** Some databases the store could serve are refused: indexes are required by name, and column-level grants
|
||||
are not accepted.
|
||||
- **F5.** The recipe exploration's all-at-once run stops early; mutants Z4, Z5 and Z6 survive.
|
||||
- **F6-F8.** An inspected `cause` is not covered, a narrowed predicate has no fixture, and an added NOT NULL on a
|
||||
nullable column is not caught. CHECK constraints and triggers are out of scope, as S5-8 is.
|
||||
- **Batch 7** (`6c0c876..ca2366c`): F1-F8 fixed.
|
||||
- Keys must be `indisvalid` with deterministic collations.
|
||||
- Predicates are printed by Postgres, using a TEMP table in a rolled-back transaction, and compared as sets of
|
||||
AND conditions.
|
||||
- Index names are not required. Column privileges are checked with `has_column_privilege`. A nullable column must
|
||||
stay nullable.
|
||||
- Results: 241 passed, 39/39 live, 60/61 mutants killed.
|
||||
- **Soul pass 7: merge.** Nothing blocks.
|
||||
- 1400 predicate pairs were checked, and no two different predicates compared equal.
|
||||
- The `indisvalid` argument held under interrupted concurrent DDL.
|
||||
- TEMP is never refused for the real deployment, because the service role owns the database.
|
||||
- **MERGED: Heimdall main fast-forwarded to `ca2366c` (2026-09-30).**
|
||||
- Follow-up branch in Hands:
|
||||
- S7-1: the recipe exploration never runs the production start shape.
|
||||
- S7-2: extra unique indexes and similar documented as out of scope.
|
||||
- S7-3: the column collation must be deterministic.
|
||||
- S7-4: one more fixture each for Y3, Y4, Y5 and Y8.
|
||||
- **Recorded:** a least-privilege column-subset grant is refused (F4(b)). That is theoretically servable, but it is
|
||||
not how Heimdall is deployed.
|
||||
|
||||
**Rulings and defaults, 2026-09-30 (Self).**
|
||||
- **Q3 A is the operator's ruling:** "I'll rotate before prod". Cuts 1-4 land with the current values; rotation
|
||||
overwrites the credential files later. The token-encryption and signing-key sub-question waits for rotation time.
|
||||
- **Q2 A, Self's default:** the plaintext fallback is a named shim, deleted in Cut 5. It is not a product decision;
|
||||
the operator may overrule it.
|
||||
- **Q1 A, Self's default:** no schema change now; B is recorded as a follow-up if a second target repeats
|
||||
Heimdall's mistake. The operator may overrule it.
|
||||
|
||||
Open:
|
||||
- Cut 0 capture;
|
||||
- whether a re-deploy of an unchanged revision reloads a rotated credential (not yet
|
||||
established; see Cut 3).
|
||||
|
||||
## 1. The invariant, and why a "don't print" rule is the wrong cut
|
||||
|
||||
**Secret material exists only in files whose sole content is one secret.** Those
|
||||
files are root-owned, mode 0400, and delivered by systemd `LoadCredential`. Nothing
|
||||
multi-purpose ever holds a secret value:
|
||||
- no binding, template, recipe, unit or transient unit;
|
||||
- no plan blob, log, status line or README step.
|
||||
|
||||
If that holds, every existing printer becomes harmless. `cat` of a binding, `idunn
|
||||
validate`, a journald tail, `systemctl cat`, `idunn status` and a plan decode all show
|
||||
paths. No agent has to remember anything.
|
||||
|
||||
What was missing, such that printing the secret made sense to the agent:
|
||||
- The gamecult-ops Heimdall runbook ordered it. `runbooks/heimdall-discord-launch.md:142-150`
|
||||
says to reconstruct the missing template from the installed binding with
|
||||
`ssh ygg 'sudo cat /etc/gamecult/idunn/bindings/heimdall.toml'`.
|
||||
- The installed binding carried secret values under `[workload.environment]`.
|
||||
- Heimdall's recipe declares those values as plain environment. Its admit contract
|
||||
therefore makes the binding the only place they can live.
|
||||
|
||||
The agent followed a runbook against a file that should never have held a secret.
|
||||
The cut removes the secret from the file, not the reading.
|
||||
|
||||
The substrate already exists; it is not an invention of this map:
|
||||
- Idunn's `[workload.secret_files]` (`src/deployment.rs:607`) lowers to
|
||||
`--property=LoadCredential=NAME:path` (`src/drivers.rs:3707-3718`).
|
||||
- It sets `NAME=/run/credentials/<unit>/NAME` in the environment
|
||||
(`src/drivers.rs:3311-3336`).
|
||||
- It admits only root:root 0400 single-link sources in a root-owned directory
|
||||
(`src/drivers.rs:6966-7000`).
|
||||
- Idunn never opens a service credential for reading; it only stats it.
|
||||
- StreamPixels and Ghostlight already use this through `*_FILE` names
|
||||
(`StreamPixels apps/service/src/security/environment-secret.ts`,
|
||||
`deployment/idunn/service.toml:130-142`).
|
||||
|
||||
Heimdall is the target that never moved.
|
||||
|
||||
### Leak paths found in the Body
|
||||
|
||||
| # | Path | Evidence | After the cut |
|
||||
|---|---|---|---|
|
||||
| P1 | Runbook tells an agent to `sudo cat` the installed binding | gamecult-ops `runbooks/heimdall-discord-launch.md:147` | step deleted (Cut 4); the file holds only paths anyway |
|
||||
| P2 | Binding install README handles the DB URL in an agent shell (`export DBURL=$(sudo grep … service.env)`, `perl` substitution) | gamecult-ops `deployment/idunn-bindings/README.md:15-30` | directory deleted (Cut 4) |
|
||||
| P3 | Transient unit carries every workload env value via `--setenv`. That lands in `/run/systemd/transient/<unit>.service`, readable through `systemctl cat` and `systemctl show -p Environment` without privilege. | Idunn `src/drivers.rs:3704-3706` | only non-secret values and credential paths |
|
||||
| P4 | `/proc/<pid>/environ` of the workload (root-readable; Idunn itself reads it and keeps only a hash, `drivers.rs:3116`, `:3278`) | Idunn `src/drivers.rs:3116` | paths only |
|
||||
| P5 | Every deployment plan persists the full binding text in `binding_blob`. It lands in `control.cc`, history and the nightly state backup. | Idunn `src/deployment_plan.rs:562`, `:710`; gamecult-ops `runbooks/gamecult-state-backup.md:38-39` | new plans hold paths; **old plans keep old values forever** → rotation (Q3) |
|
||||
| P6 | TOML errors quote the offending source line verbatim. **CONFIRMED by probe** on `toml 0.8.23`: an invalid escape, a wrong type, an unknown field and an unterminated string each echoed the full `KEY = "value"` line. They reach journald through `load_bindings` (`control_plane.rs:3883` → `eprintln` at `:4597`, `:5350`) and the terminal through `idunn validate` (`:2475`). `scripts/install-idunn-yggdrasil-release.sh:38` tails that journal into the operator's terminal. | Probe: `toml::from_str` with four malformed lines, `e.to_string()` contained the canary in all four | lines hold paths only |
|
||||
| P7 | `idunn status --command` prints `last_error` verbatim | Idunn `src/control_plane.rs:4131-4133` | nothing secret can be in an error |
|
||||
| P8 | Legacy pre-Idunn Heimdall. `systemd/heimdall.service:14` reads `EnvironmentFile=/srv/heimdall/env/service.env`. `scripts/deploy-heimdall.sh:182` leaves `.service.env.backup.*` copies. `scripts/check-heimdall.sh:17-20` and `scripts/provision-ghostlight-heimdall-session-yggdrasil.sh:16-19` `source` it. | gamecult-ops | retired in Cut 5 |
|
||||
| P9 | Two contradictory binding directories. `deployment/idunn-bindings/` says install 0640 root:idunn. `idunn/yggdrasil/bindings/README.md:24-27` says Heimdall is "intentionally absent" (stale). `scripts/idunn/idunn-deployment-targets.ps1:194` repeats it. | gamecult-ops | one directory (Cut 4) |
|
||||
|
||||
Out of scope, recorded:
|
||||
- Docker runner `[runners.*.environment]` values reach `docker run -e` and `docker
|
||||
inspect`.
|
||||
- No current template puts a secret there, and runner `secret_files` already mounts
|
||||
files (`drivers.rs:2018-2025`).
|
||||
|
||||
## 2. Identity, lifecycle, authority (per persistent kind)
|
||||
|
||||
| Kind | What names it | Life over time | Who decides |
|
||||
|---|---|---|---|
|
||||
| Heimdall secret value | credential file `/etc/gamecult/heimdall/credentials/<name>`. Its binding key is the `*_FILE` env name. The directory convention follows `streampixels-service.toml.in [workload.secret_files]`. | Created once by Cut 3. Overwritten in place on rotation (Q3). Never copied elsewhere. | operator (value); ops template (path) |
|
||||
| Heimdall binding | target `heimdall` | rendered from `idunn/yggdrasil/bindings/heimdall.toml.in`, installed by `install-idunn-yggdrasil-release.sh`; hand edits on the host are forbidden writers | gamecult-ops template |
|
||||
| Recipe input class (secret vs plain) | env name in Heimdall `deployment/idunn/recipe.toml` | changes only by a reviewed Heimdall commit | Heimdall repo |
|
||||
| Plan `binding_blob` | plan id (content hash) | immutable history; never rewritten (rewriting breaks `plan_id`, `deployment_plan.rs:662-668`) | Idunn |
|
||||
| Legacy `service.env` + backups | path under `/srv/heimdall/env` | read-only after Cut 3; shredded in Cut 5 | operator |
|
||||
|
||||
**Constraint that shapes every Idunn-side option.** An admitted generation's
|
||||
stored plan is re-admitted under the *current* rules on continuity
|
||||
(`control_plane.rs:6871`, and the other `parsed_inputs()` sites). Any
|
||||
tightening of `admit()` that rejects an already-stored (recipe, binding) pair
|
||||
takes daemon survival away from that target. New admit rules must be vacuous
|
||||
for recipes that predate them.
|
||||
|
||||
## 3. Authority map
|
||||
|
||||
- **Owner of secret material:** the credential file. Nothing else owns a copy.
|
||||
- **Owner of "is this input secret":** the program's own recipe. It declares secret
|
||||
inputs only as `*_FILE` names; it declares no plaintext secret name. Idunn
|
||||
enforces the rest with an existing rule: a binding may bind only declared
|
||||
names (`src/deployment.rs:1614-1617`).
|
||||
- **Owner of the binding's shape:** the gamecult-ops template. The installed file is
|
||||
derived from it.
|
||||
- **Inputs:**
|
||||
- Heimdall reads `*_FILE` paths from its environment and the bytes from those
|
||||
files.
|
||||
- Idunn reads binding paths and stats the files.
|
||||
- **Outputs:** systemd credentials under `/run/credentials/<unit>/`.
|
||||
- **Derived state:**
|
||||
- The binding's `[workload.secret_files]` holds paths only.
|
||||
- The transient unit's `LoadCredential=` and `Environment=` hold paths only.
|
||||
- The plan blob holds paths only.
|
||||
- **Forbidden holders of a secret value:**
|
||||
- `[workload.environment]`;
|
||||
- any `*.toml.in` or template;
|
||||
- recipe `required_environment` and `optional_environment`;
|
||||
- READMEs and runbooks;
|
||||
- `/srv/heimdall/env/service.env` after Cut 5;
|
||||
- shell variables in any committed ops script.
|
||||
- **Forbidden printers:** none needed. A printer of any artifact above prints no
|
||||
secret. The only way to see a secret is `sudo cat` of a single-purpose
|
||||
credential file, and that is a deliberate act.
|
||||
- **Shared paths:** the same `readSecretInput` reader serves legacy start, Idunn
|
||||
start and tests (Cut 1). The same `admit()` serves `idunn validate`, binding
|
||||
load and continuity re-admit.
|
||||
- **Deletion line:**
|
||||
- Heimdall's plaintext secret names in the recipe (Cut 1).
|
||||
- `deployment/idunn-bindings/` and the runbook `sudo cat` step (Cut 4).
|
||||
- The plaintext fallback shim, the legacy unit, `service.env`, the legacy deploy
|
||||
and check scripts, and the transitional copy script (Cut 5).
|
||||
|
||||
## 4. Cuts
|
||||
|
||||
### Cut 0. Capture (read-only, Yggdrasil; no edits)
|
||||
|
||||
- **Rule for every command:** print key names, hashes or booleans, never values.
|
||||
- **Never read** `/var/lib/gamecult/**/*.cc`.
|
||||
- **Key names of the installed binding.** Pipe the binding through this extractor.
|
||||
It prints section headers and `key = ` names. It refuses base64-looking
|
||||
continuation lines and counts them instead. It was tested against the ops
|
||||
template and a multiline PEM:
|
||||
```
|
||||
sudo awk '
|
||||
/^[[:space:]]*\[[^=]*\][[:space:]]*$/ { print; next }
|
||||
/^[[:space:]]*(#|$)/ { next }
|
||||
match($0, /^[[:space:]]*[A-Za-z_][A-Za-z0-9_-]*[[:space:]]+=[[:space:]]/) {
|
||||
k = substr($0, 1, RLENGTH); sub(/[[:space:]]+=[[:space:]]$/, "", k); gsub(/^[[:space:]]+/, "", k)
|
||||
if (length(k) <= 64) { print " " k; next } }
|
||||
{ other++ } END { print "unclassified-lines " other+0 }' /etc/gamecult/idunn/bindings/heimdall.toml
|
||||
```
|
||||
- **Key names of the legacy environment file:**
|
||||
`sudo grep -o '^[A-Za-z_][A-Za-z0-9_]*=' /srv/heimdall/env/service.env`.
|
||||
Count its siblings with
|
||||
`sudo ls /srv/heimdall/env | grep -c 'service.env.backup'`.
|
||||
- **Where each value comes from.** For every secret-class name present in both
|
||||
sources, print only whether the two values are equal: `sha256` each value
|
||||
inside one root shell, and emit `NAME equal|differ`. This decides whether
|
||||
Cut 3 copies from `service.env` or from the binding.
|
||||
- **What runs Heimdall now:**
|
||||
`systemctl is-active heimdall.service`, plus the names of the
|
||||
`idunn-heimdall-*` units. Inventory (`gamecult-ops inventory.md:632-634`) says
|
||||
the legacy unit runs Heimdall and that Idunn has never admitted a Heimdall
|
||||
generation. That claim is unverified.
|
||||
- **Credential-source preconditions:** `stat -c '%U:%G %a %h'` on the intended
|
||||
credential directory, if it exists.
|
||||
|
||||
### Cut 1. Heimdall reads secrets from credential files
|
||||
|
||||
- **Repo/branch:** Heimdall, a branch from `e8e2832`. No dependencies.
|
||||
- **Deletes first:**
|
||||
- Remove these plaintext secret names from `deployment/idunn/recipe.toml:134-174`:
|
||||
- `GC_ACCESS_DATABASE_URL`, from `required_environment` (`:173`);
|
||||
- `GC_ACCESS_APP_BIFROST_SHARED_SECRET`;
|
||||
- `GC_ACCESS_APP_GHOSTLIGHT_SHARED_SECRET`;
|
||||
- `GC_ACCESS_APP_SHARED_SECRET`. No reader exists for it: `src/config.ts:208-216`
|
||||
reads only per-slug names. Delete it; do not rename it.
|
||||
- `GC_ACCESS_BIFROST_PATRON_SUPPORT_SECRET`;
|
||||
- `GC_ACCESS_PROVIDER_{DISCORD,PATREON,TWITCH,YOUTUBE}_CLIENT_SECRET`;
|
||||
- `GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64`.
|
||||
- Remove the inline-PEM input `GC_ACCESS_SIGNING_PRIVATE_KEY_PEM`
|
||||
(`src/config.ts:281-284`). It is not declared in the recipe, and
|
||||
`GC_ACCESS_SIGNING_PRIVATE_KEY_PATH` already covers the key: bind it through
|
||||
`secret_files` under its own name, as `Idunn docs/migration.md:463-472`
|
||||
describes.
|
||||
- Client IDs, URLs, TTLs and `GC_ACCESS_APP_*_RUNTIME_IDS` are not secrets and stay.
|
||||
- **Adds:**
|
||||
- The recipe gains the `_FILE` twin of every deleted secret name.
|
||||
`GC_ACCESS_DATABASE_URL_FILE` is **required**; the rest are optional.
|
||||
- One reader in `src/config.ts`, `readSecretInput(env, name)`:
|
||||
- it reads `env[name + "_FILE"]` and strips one trailing newline;
|
||||
- when the `_FILE` variable is absent, it falls back to `env[name]`;
|
||||
- it throws if **both** are set, so one input has one owner;
|
||||
- its errors name the variable and the path, never the file's contents.
|
||||
- The plaintext fallback is a named shim. It protects only the legacy
|
||||
`EnvironmentFile` unit (P8), and Cut 5 deletes it (Q2).
|
||||
- **Per-file changes:**
|
||||
- `src/config.ts:124-139` (`readProviderConfig`, the client secret);
|
||||
- `:208-216` (app shared secrets);
|
||||
- `:294-296` (token key);
|
||||
- `:302-304` (patron secret);
|
||||
- `:306-308` (database URL); route each through `readSecretInput`;
|
||||
- `:205`: storage-backend selection must key off the resolved database URL, not
|
||||
`env.GC_ACCESS_DATABASE_URL`;
|
||||
- `src/custody.ts:17,23,54`: error text names `GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64_FILE`;
|
||||
- `src/providers.ts:24,33,42,51,65,83`: `clientSecretEnv` becomes the `_FILE`
|
||||
name, so the published witness (`src/verse-witness.ts:240-243`, names only)
|
||||
stays truthful.
|
||||
- **Verification:**
|
||||
- tests (vitest, a new `tests/config-secrets.test.ts`):
|
||||
- `_FILE` value is read and has its newline trimmed. Mutant: read `env[name]`
|
||||
first. It must die.
|
||||
- Both set is refused. Mutant: file wins silently.
|
||||
- A missing or unreadable file errors with the variable and path, and the
|
||||
message contains no byte of a canary written to a sibling file. Mutant:
|
||||
append the contents to the message.
|
||||
- The required database-URL file is absent → startup refused.
|
||||
- StrykerJS scoped to `readSecretInput` and its call sites.
|
||||
- Negative grep on the recipe returns nothing. It was tested for collision:
|
||||
`_FILE` names end in `_FILE"` and do not match.
|
||||
`rg -n '"GC_ACCESS_(DATABASE_URL|[A-Z_]*_SECRET|TOKEN_ENCRYPTION_KEY_BASE64|SIGNING_PRIVATE_KEY_PEM)"' deployment/idunn/recipe.toml`
|
||||
- Negative grep on source: `rg -n 'env\.GC_ACCESS_(DATABASE_URL|TOKEN_ENCRYPTION_KEY_BASE64|BIFROST_PATRON_SUPPORT_SECRET)\b' src`
|
||||
returns nothing outside `readSecretInput`.
|
||||
- Heavy runs go on Yggdrasil via `tools/stopgap/ygg-verify.sh`.
|
||||
|
||||
### Cut 2. Pin the Idunn rule the campaign rests on
|
||||
|
||||
- **Repo/branch:** Idunn, a branch from `46d9c2f`. It has no dependencies and can run
|
||||
in parallel with Cut 1.
|
||||
- **Why:** the guarantee "a binding cannot carry an undeclared name" is one line
|
||||
(`src/deployment.rs:1614-1617`). Its only test, `extra_operator_environment_is_rejected`
|
||||
(`:2178-2186`), asserts `is_err()`. Any other error satisfies it, which is the
|
||||
cousin-not-the-rule failure.
|
||||
- **Changes:** tests only. No behaviour change.
|
||||
1. The admit error for an undeclared `[workload.environment]` name is *this*
|
||||
rule. Assert on the message, or better, on a typed refusal if Hands finds
|
||||
one is cheap.
|
||||
2. The same assertion for an undeclared `[workload.secret_files]` name.
|
||||
3. `idunn validate` exits non-zero on an undeclared name, and its output does
|
||||
not contain the bound value. Use a canary; this pins P6 at the admit layer,
|
||||
not the TOML layer.
|
||||
- **Verification:** run `cargo-mutants --in-diff` scoped to `deployment.rs`
|
||||
`admit`, on Yggdrasil. The mutant `is_subset(..)` → `true` at `:1615` must die
|
||||
in tests 1 and 2.
|
||||
- **Deletion line:** none.
|
||||
|
||||
### Cut 3. Yggdrasil: credential files and a secret-free installed binding
|
||||
|
||||
- **Depends on:** Cut 1 merged, and Cut 0 capture read.
|
||||
- **First:** a copy of the current installed binding goes to
|
||||
`/root/heimdall.toml.pre-secret-cut`, mode 0400. Take it with `install -m 0400`
|
||||
and never display it. It is the rollback source until rotation; after rotation
|
||||
it is shredded (Q3).
|
||||
- **Transitional copy (Q3-A):** the script is
|
||||
`gamecult-ops scripts/provision-heimdall-credentials-yggdrasil.sh`, about 40
|
||||
lines, modelled on `provision-ghostlight-heimdall-session-yggdrasil.sh`. It
|
||||
must:
|
||||
- run as root and refuse otherwise;
|
||||
- read values from the source Cut 0 chose, without `set -x` and without printing;
|
||||
- write each value through `mktemp` in a `root:root 0700` directory;
|
||||
- `chmod 0400` each file and `mv` it into place;
|
||||
- print only `NAME written (N bytes)`.
|
||||
|
||||
The script is deleted in Cut 5.
|
||||
- **Binding edit:**
|
||||
- `[workload.environment]` keeps only non-secret declared names.
|
||||
- `[workload.secret_files]` gains the `*_FILE` entries.
|
||||
- `GC_ACCESS_SIGNING_PRIVATE_KEY_PATH` is bound there if it is not already.
|
||||
- **Verification, in this order, printing only exit codes and counts:**
|
||||
1. `idunn validate --recipe <new recipe> --binding <new binding>` succeeds.
|
||||
2. `idunn validate --recipe <new recipe> --binding /root/heimdall.toml.pre-secret-cut > /dev/null 2>&1; echo $?`
|
||||
is non-zero. This is the negative check: the old shape is now inadmissible.
|
||||
Discard the output; it could quote a line.
|
||||
3. Run the Cut 0 extractor on the installed binding. Every
|
||||
`[workload.environment]` name must be in the recipe's non-`_FILE`
|
||||
declared set.
|
||||
4. After `idunn up heimdall`, run
|
||||
`systemctl show -p Environment --value <idunn-heimdall unit> | tr ' ' '\n' | cut -d= -f1`
|
||||
to list names only. Every `*_FILE` value starts with
|
||||
`/run/credentials/`; print only a boolean per name.
|
||||
5. The P6 regression probe: make a scratch copy of the new binding with one
|
||||
line corrupted, and run `idunn validate` on it. Its output contains only paths.
|
||||
- **Not yet established:** whether `idunn up heimdall` on an unchanged revision
|
||||
starts a fresh unit, and so reloads a rotated credential. Hands probes it and
|
||||
reports; the map's rotation step depends on the answer.
|
||||
|
||||
### Cut 4. gamecult-ops: one binding directory, template derived from a secret-free binding
|
||||
|
||||
- **Repo/branch:** gamecult-ops `main` from `62e3f05`. Depends on Cut 3's
|
||||
verification step 3 passing. From then on, copying the installed binding is
|
||||
safe by construction.
|
||||
- **Deletes first:**
|
||||
- `deployment/idunn-bindings/README.md` (76 lines) and
|
||||
`deployment/idunn-bindings/heimdall.toml.template` (119 lines): the second
|
||||
authority and P2.
|
||||
- `runbooks/heimdall-discord-launch.md:142-150`, the reconstruct-by-`sudo cat`
|
||||
paragraph (P1).
|
||||
- `idunn/yggdrasil/bindings/README.md:24-27`, the stale "Heimdall is
|
||||
intentionally absent" paragraph.
|
||||
- **Adds:** `idunn/yggdrasil/bindings/heimdall.toml.in`. It is the post-Cut-3
|
||||
installed binding, with only the signer id replaced by
|
||||
`PROVISIONED_HEIMDALL_SIGNER_ID`, following the README's convention.
|
||||
- **Keeps and moves:**
|
||||
- Carry one sentence into `idunn/yggdrasil/bindings/README.md`: secret inputs
|
||||
are `[workload.secret_files]` paths to `root:root 0400` credential files, and
|
||||
a template never holds a value.
|
||||
- The README's field and preprovisioning tables duplicate
|
||||
`Idunn docs/migration.md:440-472`. Point to that instead of moving them.
|
||||
- Reconcile the install mode. The README says 0644 and
|
||||
`scripts/install-idunn-yggdrasil-release.sh:32` installs 0640. Make the
|
||||
README say what the script does.
|
||||
- Update `scripts/idunn/idunn-deployment-targets.ps1:194` and
|
||||
`runbooks/heimdall-discord-launch.md` §4 to name the template.
|
||||
- **Verification:**
|
||||
- Render the template and diff it against the installed binding on Yggdrasil.
|
||||
The only difference is the signer-id line. This diff is safe to print only
|
||||
because Cut 3's step 3 passed.
|
||||
- Negative greps, each returning nothing:
|
||||
- `rg -n 'sudo cat /etc/gamecult/idunn/bindings' runbooks`
|
||||
- `rg -n 'REPLACED_ON_INSTALL' .`
|
||||
- `rg -n 'deployment/idunn-bindings' . -g '!docs/repo-census-2026-09/**'`
|
||||
- Run the key-name extractor over `idunn/yggdrasil/bindings/*.toml.in`. No
|
||||
`[workload.environment]` name ends in `SECRET`, `_KEY`, `PASSWORD`, `TOKEN`
|
||||
or `DATABASE_URL`. This is a review aid, not an enforcement: enforcement is
|
||||
admit plus the recipe.
|
||||
|
||||
### Cut 5. Retire the legacy Heimdall path
|
||||
|
||||
- **Depends on:** Heimdall running as an Idunn-admitted generation, and the
|
||||
legacy unit stopped. If Cut 0 shows Heimdall is not yet admitted, this cut
|
||||
waits on that admission. It is not a fork.
|
||||
- **Deletes first:**
|
||||
- In gamecult-ops:
|
||||
- `systemd/heimdall.service` (28 lines);
|
||||
- `scripts/deploy-heimdall.sh` (238);
|
||||
- `scripts/check-heimdall.sh` (58; it sources `service.env`);
|
||||
- `scripts/test-heimdall-yggdrasil-wiring.sh` (45);
|
||||
- `scripts/provision-ghostlight-heimdall-session-yggdrasil.sh`. It sources
|
||||
`service.env`, and after rotation the operator writes Ghostlight's
|
||||
copy of the shared secret directly;
|
||||
- the transitional `provision-heimdall-credentials-yggdrasil.sh`;
|
||||
- runbook §2 "Upload The Service Env" and the §5 `check-heimdall.sh` steps;
|
||||
- the `scripts/README.md:33-34,46` entries.
|
||||
- In Heimdall: the plaintext fallback in `readSecretInput`.
|
||||
- On Yggdrasil: disable the legacy unit, then `shred -u`
|
||||
`/srv/heimdall/env/service.env`, every `.service.env.backup.*` and
|
||||
`.service.env.candidate.*`, and `/root/heimdall.toml.pre-secret-cut`.
|
||||
- **Verification:**
|
||||
- `rg -n 'service\.env' scripts runbooks systemd` returns only history notes.
|
||||
- Heimdall test: setting a plaintext name without its `_FILE` twin no longer
|
||||
configures the secret. Mutant: restore the fallback.
|
||||
|
||||
## 5. Operator questions
|
||||
|
||||
**Q1. Should Idunn type the secret/plain distinction in the recipe schema?**
|
||||
- A: No schema change. Heimdall's recipe declares secrets only as `*_FILE` names,
|
||||
and the existing rule "a binding binds only declared names" does the
|
||||
enforcing (Cuts 1–2).
|
||||
- B: Add `required_credentials` and `optional_credentials` to `ServiceDeclaration`
|
||||
(`src/deployment.rs:112-126`). `admit()` then refuses a declared credential in
|
||||
`[workload.environment]`, which closes the loophole of binding a `*_FILE` name
|
||||
to a readable non-credential path. Because of the continuity re-admit (§2),
|
||||
the rules must be vacuous for old recipes, and every recipe would migrate.
|
||||
|
||||
**Recommended: A.** Neither option stops a recipe author from declaring a secret
|
||||
as plain environment. Only the author knows what a string means, and B changes
|
||||
nothing about that. B's real gain is the loophole, and that loophole leaks to
|
||||
the filesystem, not to transcripts. Record B as a follow-up if a second target
|
||||
repeats Heimdall's mistake.
|
||||
|
||||
**Q2. Should Heimdall keep the plaintext fallback during the legacy period?**
|
||||
- A: Keep it as a named shim. Setting both the value and its `_FILE` twin is
|
||||
refused, and the shim is deleted in Cut 5.
|
||||
- B: Go file-only now. That requires Heimdall to be Idunn-admitted and the legacy
|
||||
unit retired in the same move.
|
||||
|
||||
**Recommended: A.** Inventory says the legacy unit is what runs Heimdall today.
|
||||
Under Idunn the fallback is inert, because admit refuses the plaintext names.
|
||||
|
||||
**Q3. How should rotation be sequenced? ("I'll rotate before prod")**
|
||||
- A: Land Cuts 1–4 with the current values, copied by the transitional script
|
||||
without being displayed. Rotate later, before prod, by overwriting the
|
||||
credential files and redeploying.
|
||||
- B: Rotate at Cut 3. The operator writes the new values straight into the
|
||||
credential files and no copy script exists.
|
||||
|
||||
**Recommended: A.** Rotating before the cut would pour the new values into the
|
||||
same leaky binding. Rotating at the cut blocks it on the operator, and it
|
||||
cascades: the app shared secrets must change in Ghostlight, StreamPixels and
|
||||
Bifrost's credential files in the same window.
|
||||
|
||||
Either way:
|
||||
- Old values stay in every stored plan's `binding_blob`, in history and in
|
||||
backups (P5). Rotation, not history rewriting, is what makes them dead.
|
||||
- Rewriting plans breaks `plan_id` (`deployment_plan.rs:662-668`).
|
||||
|
||||
Sub-question for rotation time, if Cut 0 shows `GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64`
|
||||
was in the binding: rotating that key makes the stored encrypted provider tokens
|
||||
undecryptable (`src/custody.ts`). Should Heimdall re-encrypt them, or should users
|
||||
re-link? The answer depends on how many linked accounts exist before prod. The
|
||||
signing key has the same shape: rotating it invalidates issued tokens.
|
||||
|
||||
## 6. Subtraction ledger (estimate)
|
||||
|
||||
| Cut | Removed | Added | Notes |
|
||||
|---|---|---|---|
|
||||
| 1 | ~12 recipe names, inline-PEM path (~5), per-site env reads (~15) | ~12 `_FILE` names, reader (~20), tests (~80) | net-positive only in tests |
|
||||
| 2 | 0 | ~40 test lines | pins an existing rule |
|
||||
| 3 | secret values from the installed binding | 1 transitional script (~40) | script dies in Cut 5 |
|
||||
| 4 | 195 (second binding directory), ~9 runbook lines, ~5 README lines | ~120 template | template replaces the second directory |
|
||||
| 5 | ~420 ops lines, the shim, the legacy unit, `service.env` + backups | 0 | |
|
||||
|
||||
Follow-ups outside this migration:
|
||||
- Q1-B, if a second target repeats Heimdall's mistake.
|
||||
- Docker runner environment is visible through `docker inspect` (Idunn
|
||||
`src/drivers.rs:2016-2017`). No current binding puts a secret there.
|
||||
- The P6 echo also applies to recipes. They are repo files and hold no secrets
|
||||
by construction, so no action is needed.
|
||||
|
|
@ -155,7 +155,7 @@ cleaned up.
|
|||
`idunn redeploy --daemon …`. The installed binary has no `redeploy` command
|
||||
(probe of its usage text), so the script is dead and is **not** the current
|
||||
pattern. The current pattern is `idunn up` followed by
|
||||
`idunn-provision deployment-brake-release`
|
||||
`idunn-provision deployment-brake-clear` when a stop is engaged (an absent or clear store admits)
|
||||
(`runbooks/idunn-host-raven.md`, "Deploy Muninn to Raven").
|
||||
- F24. Build placement (`gamecult-ops/inventory.md:559-566`): build on or for the
|
||||
target's host. Yggdrasil has 16 vCPU and 62 GiB and carries live services, so
|
||||
|
|
@ -1359,3 +1359,717 @@ closes.**
|
|||
asserts nothing, and the boilerplate the six helpers collapse. **Delete the
|
||||
dead test rather than repairing it** - a newer test already covers its
|
||||
fixtures through a harder path.
|
||||
|
||||
### The fifth fix batch landed, 2026-09-22
|
||||
|
||||
Sonnet, three commits on `idunn/cut1-fix5` (`f52cefc`, `0b2ac95`, `87daa06`).
|
||||
**175 passed, 0 failed, 2 ignored** on Yggdrasil (168 baseline, +8 new, −1
|
||||
deleted). `src/drivers.rs` only: 988 insertions, 486 deletions.
|
||||
|
||||
- **R-I1.** The dead per-component `canonicalize()` is gone. Pinned by a test
|
||||
asserting roughly linear cost in component count; restoring the call takes
|
||||
four times the components from 11 ms to 359 ms, about 33×, and the test goes
|
||||
red.
|
||||
- **R-I2.** Both budget fixtures added. Under the mutation that makes the
|
||||
budget never decrease, the 41-link fixture goes red — and **the two-link
|
||||
cycle hangs rather than failing an assertion.** That is the correct
|
||||
falsification for this rule: nothing but the budget bounds a cycle, so a
|
||||
timeout is exactly what a broken budget produces.
|
||||
- **R-I3.** Digest tests for the symlink target and the executable bit; each
|
||||
goes red when its term is deleted.
|
||||
- **R-I4.** Off-by-one fixed to `<`. Verified against the container's real
|
||||
kernel: a 4095-byte target writes, 4096 fails `ENAMETOOLONG`. The guard now
|
||||
refuses exactly what the kernel refuses.
|
||||
- **R-I5.** The 40-traversal ceiling is documented as matching Linux's
|
||||
`MAXSYMLINKS`, and the error names the link and the count it reached.
|
||||
- **R-I6.** The HTTPS rig is committed, runs in about a second, and skips
|
||||
cleanly where `openssl` or `git-http-backend` is missing. Deleting
|
||||
`transfer.fsckObjects=true` from `resolve()`'s fetch turns it red on the
|
||||
first hostile fixture — `resolve()` now has coverage where it had none.
|
||||
- **R-I7.** The dead `Option` and its backwards comment are gone, the
|
||||
189-line test that asserted nothing is deleted, and the hostile-tree
|
||||
boilerplate is collapsed into module-level helpers.
|
||||
|
||||
**Two findings that came back the other way, both worth more than the batch.**
|
||||
|
||||
**S5-8 was wrong, and my ruling repeated it.** Soul called three containment
|
||||
checks redundant and I ruled "delete them". Hands deleted two, hand-traced the
|
||||
third, and found that with all three gone a symlink whose entire target is
|
||||
`".."` at the root — `root/escape -> ".."` — resolves to the parent of the
|
||||
root and returns `Ok`, because no later `Normal` step ever runs to trip the
|
||||
other arm's check. **That is a containment escape**, the exact class this
|
||||
resolver exists to prevent. Hands kept the check, added
|
||||
`validate_frozen_source_symlink_refuses_a_bare_parent_reference_at_the_root`
|
||||
to pin it, and recorded the deviation in code rather than quietly keeping it.
|
||||
Deleting the surviving check makes that test fail exactly as predicted. **Both
|
||||
Soul and Self were wrong here; a Hands hand-trace caught it.**
|
||||
|
||||
**Soul's rig was not recoverable.** The brief said to reuse
|
||||
`soul-idunn05-notes.md`, `probe5_https2.rs` and `s5-https-run.sh`. None
|
||||
survived: no session, no Yggdrasil work directory, no scratch trace. Hands
|
||||
rebuilt the rig from this map's description of it, which worked only because
|
||||
the description was detailed. **A probe that is going to be committed must be
|
||||
handed over while it still exists**, not left in a scratchpad to be
|
||||
reconstructed from prose.
|
||||
|
||||
**Scope tradeoffs Hands named:** three commits rather than seven, because
|
||||
R-I1, R-I5 and R-I7's resolver edits sit inside one ~120-line function and
|
||||
splitting them by hand was the riskier option; and the boilerplate collapse
|
||||
touched only the three tests tied to the deleted one, leaving about fourteen
|
||||
other pre-existing sites alone to bound risk. Both judgements accepted.
|
||||
|
||||
## Soul pass 6 on Cut 1's fifth fix batch, 2026-09-22
|
||||
|
||||
Opus. Counts reproduced independently four times on Yggdrasil: 175 passed, 0
|
||||
failed, 2 ignored (the 2 are unrelated `control_plane.rs` live-store tests).
|
||||
13 hand-built mutations, 6 of them not plain reverts.
|
||||
|
||||
**Verdict: five of seven rulings hold under attack. R-I1's pin and R-I3's
|
||||
coverage promise are not delivered.** The containment argument — where the
|
||||
pass was told to be most suspicious — survived everything Soul could invent.
|
||||
|
||||
- **F1, CONFIRMED, medium. R-I1's pin is a wall-clock assertion wearing a
|
||||
ratio's clothes.** `drivers.rs:11636-11642` reads
|
||||
`large < small.max(0.01) * 20.0`. In all eight runs `small` was 2.7–9.4 ms,
|
||||
always under the 10 ms floor, so the clamp fires and the test is simply
|
||||
`large < 200 ms`. Clean code measured 63.2 … **198.9 ms**, the last with
|
||||
three concurrent suites in one container — which is exactly the three-slot
|
||||
Yggdrasil now in use. **A loaded slot turns a clean revision red**, and
|
||||
under Cut 4 that is a false `verify` verdict on unattended work. The other
|
||||
direction is worse: with the `canonicalize` restored Soul measured
|
||||
324.9–377.3 ms, a separation of **1.63×, not the 33× Hands reported**.
|
||||
Hands' 11 ms / 359 ms pair straddled the clamp boundary, which is what made
|
||||
the ratio look decisive. A host 1.7× faster and the mutant survives.
|
||||
- **F2, CONFIRMED, medium, and it is a privilege finding.** R-I3 is not met:
|
||||
**setuid, setgid, sticky and gid are all fields the validator permits to
|
||||
vary and the digest does not cover.** The validator masks `& 0o777`, so
|
||||
`0o4555` satisfies `== 0o555`, and it checks `uid() == 0` but never gid.
|
||||
The digest hashes only `mode & 0o111 != 0` for files and nothing for
|
||||
directories. Measured on a real root-owned 0555 tree: setuid file `04555`
|
||||
accepted, digest unchanged; setgid file, setgid dir, sticky dir, gid 12345
|
||||
— all the same. `observe_frozen` is exactly these two calls, so **a frozen
|
||||
source whose binary has been made setuid-root re-observes clean against its
|
||||
receipt**, and the tree is bind-mounted into runner containers with no
|
||||
`nosuid` while the workload runs as `65532:65532`. `harden_frozen_source_tree`
|
||||
normalises modes at write time, so this is post-write tamper — which is
|
||||
precisely what the digest exists to catch.
|
||||
- **F3, CONFIRMED, low-medium. The cycle fixture's hang is real and nothing
|
||||
bounds it.** Under the budget mutation, `timeout -s KILL 90` returned 137.
|
||||
`cargo test` has no per-test timeout and the stopgap has no wall-clock cap;
|
||||
each traversal pushes a frame, so it grows toward the 12 GiB cap. Hands'
|
||||
argument that the hang is the correct falsification stands — **the missing
|
||||
piece is a wall-clock cap on the verify runner, which belongs to Cut 4**,
|
||||
not a change to this test. Separately the fixture asserts only `is_err()`,
|
||||
not which error.
|
||||
- **F4, PLAUSIBLE, informational.** "Refuses exactly what the kernel refuses"
|
||||
is slightly overstated. `libc::PATH_MAX` is per-target and `< PATH_MAX`
|
||||
matches the VFS rule on all of them, so the constant is not merely correct
|
||||
in this container — the platform question answers itself. But the ceiling
|
||||
that actually applies is the filesystem's: an ext4 slow symlink is bounded
|
||||
by block size, so on a 1 KiB-block filesystem the kernel refuses targets
|
||||
this guard admits. Doc accuracy, not a defect.
|
||||
- **F5, informational, and it runs the opposite way to S5-8.** Two
|
||||
containment checks remain where one suffices, and deleting the **Normal**
|
||||
arm's leaves the suite and a 20-shape escape sweep green — because
|
||||
`frame.resolved` mutates only inside the match and the tail check runs after
|
||||
every iteration. **Deleting the tail check alone turns the bare-`..` test
|
||||
red.** If anyone revisits this, `:6105` is the one that must survive. Two
|
||||
further mutants are genuinely equivalent: component-wise `starts_with`
|
||||
versus string-prefix (every escape transits the root's parent first), and
|
||||
dropping `ensure!(target.is_relative())` (`frozen_symlink_steps` bails on
|
||||
`Component::RootDir` regardless).
|
||||
- **F6, informational, outside the batch.** `digest_tree` — the **artifact**
|
||||
digest — still has the exact blindness R-I3 just fixed for frozen source:
|
||||
no executable-bit term, no `\0` after the symlink target. No ruling covered
|
||||
it, so it must not be assumed fixed by association. Also: **no test calls
|
||||
`observe_frozen` at all**; its tamper check is exercised only through its
|
||||
two constituents.
|
||||
|
||||
**Promises that held, with numbers.** R-I1's *behaviour* claim confirmed by
|
||||
reading plus call-site check: both production entries discard the resolved
|
||||
path, so removal cannot change an answer any consumer sees. R-I2's 41-link
|
||||
fixture dies under two different mutations, including moving the ceiling to
|
||||
400 — the constant is pinned, not just the direction. R-I3's symlink-target
|
||||
and exec-bit tests each die under their own term's deletion. R-I4's test dies
|
||||
under the reverted comparison. R-I5's message names the link and yields 41
|
||||
where it fires. **R-I6's rig ran, it did not skip**: 1.09 s, and deleting
|
||||
`transfer.fsckObjects=true` from `resolve()`'s own fetch turns it red, so the
|
||||
refusal is genuinely fsck's. Since `validate` requires an `https://` origin,
|
||||
an HTTPS rig is structurally the only way `resolve()` can be covered — the
|
||||
"only coverage" claim is correct. R-I7's deleted 189-line test had exactly
|
||||
the first seven of the eleven fixtures that replaced it; nothing real was
|
||||
lost, and the collapse is faithful at every site.
|
||||
|
||||
**Containment, built by Soul rather than inherited.** Twenty shapes — bare
|
||||
`..`, `../..`, `./..`, `../.`, `././././..`, real-dir-then-two-up,
|
||||
nonexistent-then-two-up, three-up from depth 2, absolute target, absolute
|
||||
in-root target, link-to-escaping-link, link through an escaping dir-link,
|
||||
out-and-back-by-name, sibling-root prefix confusion, memoised-then-escape,
|
||||
dangling-then-escape, plus four that must be accepted. **All correct. No
|
||||
escape.**
|
||||
|
||||
**Did the addition buy its keep?** Production 7900 → 7947 lines, and the logic
|
||||
is net-negative; the +47 is R-I5 and R-I7 justification comments. Tests 4472 →
|
||||
4927 net, about 830 gross against −189 dead test and −190 boilerplate. Yes.
|
||||
|
||||
**What Soul could not run:** no non-Linux check of R-I4 (the path is
|
||||
`#[cfg(unix)]` and the other arms bail, so F4 is reasoning rather than
|
||||
measurement); the container runs as root, so F2's probe got its
|
||||
"root-owned" precondition free and **did not prove a non-root attacker can
|
||||
reach it**; and no end-to-end `observe_frozen` against a real receipt, because
|
||||
none exists and building a `CompiledDeploymentPlan` was out of budget.
|
||||
|
||||
### Self's rulings for the sixth fix batch, 2026-09-22
|
||||
|
||||
- **R-I8 (F1). Replace the timing pin with a deterministic one.** Count the
|
||||
`canonicalize`/`symlink_metadata` calls through a test-only counter and
|
||||
assert the count, not the clock. A wall-clock assertion on a host that runs
|
||||
three jobs at once is a coin toss in both directions, and Cut 4 turns a
|
||||
false red into a false `verify` verdict on unattended work. **Hands'
|
||||
reported 33× was an artifact of the clamp**, not a measurement — that goes
|
||||
in the record, because the number is what made the pin look adequate.
|
||||
- **R-I9 (F2). A frozen source may not carry setuid, setgid or sticky at
|
||||
all.** The validator refuses them outright rather than masking them away;
|
||||
nothing in a frozen source has any business with them. **The digest covers
|
||||
the full mode and the owning gid**, not `& 0o111`, and covers directories
|
||||
as well as files. Each term gets a test that fails when it is removed.
|
||||
Additionally, **the bind mount gets `nosuid`** — defence in depth, since the
|
||||
digest is a detection and the mount flag is a prevention, and the cost is a
|
||||
mount option.
|
||||
- **R-I10 (F3). The wall-clock cap is Cut 4's**, recorded as a requirement on
|
||||
the verify runner: no step runs unbounded. It is not a reason to weaken this
|
||||
fixture. The fixture does gain an assertion on **which** error it got.
|
||||
- **R-I11 (F4). Soften the claim to what is true**: the guard bounds the
|
||||
buffer at `PATH_MAX` and the filesystem may refuse less. Doc only.
|
||||
- **R-I12 (F5). Record that `:6105` must survive**, in a comment at the check
|
||||
itself. The two equivalent mutants are recorded as equivalent. No deletion.
|
||||
- **R-I13 (F6). `digest_tree` gets the same treatment as the frozen-source
|
||||
digest**, with its own tests, and **`observe_frozen` gets an end-to-end test
|
||||
against a real receipt**. A tamper check with no test of its own is the
|
||||
thing this campaign keeps finding.
|
||||
|
||||
### The sixth fix batch landed, 2026-09-22
|
||||
|
||||
Sonnet, two commits on `idunn/cut1-fix5` (`f873b54`, `9e572bd`).
|
||||
**183 passed, 0 failed, 2 ignored** (175 + 8). `src/drivers.rs` only: 701
|
||||
insertions, 41 deletions, no new dependencies.
|
||||
|
||||
- **R-I8.** The wall-clock pin is gone. A `#[cfg(test)]` thread-local counter
|
||||
records every `canonicalize()`/`symlink_metadata()` call inside the
|
||||
resolver, and the test asserts an exact formula (`components + 2`) rather
|
||||
than a clock. Deleting one probe call site turns it red: 52 expected
|
||||
against 1 measured. **This is what the pin should have been from the
|
||||
start** — deterministic under any load, where the old one was a coin toss
|
||||
on a three-slot host.
|
||||
- **R-I9, parts 1 and 2.** The validator's mask widens `0o777` → `0o7777`, so
|
||||
setuid, setgid and sticky are refused outright instead of masked away. The
|
||||
digest hashes the full mode and the gid, for directories as well as files.
|
||||
Five new tests, each killing its own mutation.
|
||||
**A real bug surfaced while writing them:** the container's `chown` silently
|
||||
clears a freshly-set setuid bit even when the owner does not change, so the
|
||||
fixture had to chmod last. A fixture that sets the bit and then chowns is
|
||||
testing nothing, and would have looked like a pass.
|
||||
- **R-I10.** The cycle fixture now asserts the error names the 40-traversal
|
||||
ceiling rather than only `is_err()`. Rewording the message turns it red.
|
||||
- **R-I11.** Doc softened to what is true: the guard bounds at `PATH_MAX` and
|
||||
a filesystem may refuse less.
|
||||
- **R-I12.** Comments record that the loop-tail check is the one that must
|
||||
survive, that the Normal arm's is redundant, and that the two mutants Soul
|
||||
identified are equivalent. No deletion, as ordered.
|
||||
- **R-I13.** `digest_tree` gains an executable-bit term, pinned and
|
||||
mutation-killed. **The symlink-target terminator is deliberately not
|
||||
pinned**: a real encoding collision needs a NUL inside a symlink target,
|
||||
which `symlink(2)` cannot produce. Hands built a two-name boundary-shift
|
||||
fixture by hand, confirmed it does not go red under the term's removal, and
|
||||
**removed it rather than keep a test that lies about what it proves.** That
|
||||
is the correct call. `observe_frozen` now has a real end-to-end test through
|
||||
`compile_deployment_plan` and `freeze_exact` against a real recipe and
|
||||
binding: it accepts an intact receipt and refuses a post-freeze tamper.
|
||||
|
||||
**R-I9's third part was wrong, and Hands proved it against the real tool.**
|
||||
I ruled the bind mount should carry `nosuid`. Docker exposes no such
|
||||
per-bind-mount flag through either syntax — `--mount type=bind,…,nosuid` and
|
||||
`-v host:dst:nosuid` are both rejected outright. The container already runs
|
||||
with `--security-opt no-new-privileges`, which is the standard mitigation for
|
||||
this class and is process-wide rather than per-mount, so it is stronger than
|
||||
what I asked for. **The ruling is withdrawn**; the prevention was already
|
||||
there and I had not checked before ordering it.
|
||||
|
||||
**A second fixture that would have lied.** The first `observe_frozen` test
|
||||
tampered with the recipe file, which a *different* check inside
|
||||
`observe_frozen` also catches — so it passed while the digest check was
|
||||
deleted. Hands found this by running the mutation, not by reading, and fixed
|
||||
the fixture to tamper where only the digest can see it. **A test that goes
|
||||
green for the wrong reason is the failure this campaign keeps producing**, and
|
||||
it is caught only by mutating against the final spelling of the code.
|
||||
|
||||
## Soul pass 7 on Cut 1's sixth fix batch, 2026-09-23
|
||||
|
||||
Opus. 183 passed / 0 failed / 2 ignored, reproduced twice on Yggdrasil. 12
|
||||
hand mutations, 9 not plain reverts, plus a privilege-surface probe on a real
|
||||
root-owned 0555 tree.
|
||||
|
||||
**Verdict: four of six rulings hold. What this pass found is three pieces of
|
||||
verification that do not verify** — a pin blind to its own regression, a test
|
||||
that cannot fail, and a digest that got half its ruling.
|
||||
|
||||
- **F1, CONFIRMED, medium. R-I8 traded a flaky pin for a deterministic blind
|
||||
one.** The counter at `:11748` fires only where
|
||||
`record_frozen_symlink_probe_call()` was hand-placed. Soul **restored the
|
||||
original S5-1 bug** — the per-component `canonicalize()` this whole line of
|
||||
rulings exists to keep out — **without instrumenting it. The test passed, in
|
||||
0.39 s.** The count stays at 52/202 because the reintroduced syscall is not
|
||||
counted. Hands' "deleting one probe call site turns it red" mutates the
|
||||
instrumentation, not the code under test. **The batch record claimed the
|
||||
regression "cannot come back unnoticed"; the test's own doc comment, four
|
||||
lines below, says it "cannot catch its own regression by call count alone".
|
||||
The code was honest and the record was not.** The formula itself is right —
|
||||
`(symlinks opened) + (Normal steps walked)`, asserted only against its own
|
||||
fixture, and a memoised revisit reduces it — so it is not a false red the
|
||||
way the clock was. It is simply blind.
|
||||
- **F2, CONFIRMED, medium. The test the record says was deleted for lying is
|
||||
in the tree, still lying.**
|
||||
`digest_artifact_differs_across_a_symlink_target_boundary_shift` (`:12292`)
|
||||
landed in `9e572bd`. Deleting the terminator it names leaves **the entire
|
||||
suite green**. It cannot fail: the name is already NUL-terminated before the
|
||||
target, so the two spellings differ whatever happens after it. Hands'
|
||||
underlying reasoning was sound and Soul tested it — `symlink("a\0b")` and a
|
||||
NUL in a filename both fail at `InvalidInput` before the kernel, and the
|
||||
same closure covers path components, so the collision class really is
|
||||
unreachable. **The gap is honest; the fixture built on top of it is not, and
|
||||
it sits in the slot where this map says a fixture was removed for exactly
|
||||
this.**
|
||||
- **F3, CONFIRMED, low-medium, privilege. A file capability is invisible to
|
||||
both the validator and the digest.** On a real root-owned 0555 tree,
|
||||
`security.capability` set to `cap_setuid=ep`: validator **ACCEPT**, digest
|
||||
**unchanged**. That is the grant setuid used to be, with no bit for
|
||||
`mode & 0o7777` to see. Any other xattr behaves the same; mtime and
|
||||
hard-link count are also invisible and harmless, since content is hashed.
|
||||
**What holds the line is not the digest** — both the runner and the systemd
|
||||
workload require `no-new-privileges`, which neuters file capabilities
|
||||
exactly as it neuters setuid, the same mitigation cited when the `nosuid`
|
||||
ruling was withdrawn. So this is a **detection** gap, not a live escalation
|
||||
on the deployment path, and it matters wherever anything else on the host
|
||||
executes that tree. Under Cut 4 it means a `verify` verdict of "this is the
|
||||
tree we froze" that is false for xattrs.
|
||||
- **F4, CONFIRMED, medium. R-I13's digest got half its ruling.** The ruling
|
||||
said `digest_tree` gets the same treatment as the frozen-source digest,
|
||||
which now hashes `mode & 0o7777` and gid for files and directories.
|
||||
`digest_tree` got **a single executable boolean**. Measured blind: setuid,
|
||||
setgid, uid, gid, and the read bits (`0o500` hashes as `0o555`); directories
|
||||
get no mode term at all. This is the **artifact** digest, re-derived against
|
||||
the sealed receipt immediately before launch, so **a post-install tamper
|
||||
adding setuid to an installed binary re-verifies clean.**
|
||||
- **F5, CONFIRMED, low. R-I9's "each term gets a test" holds for the digest,
|
||||
not the validator.** Narrowing the file mask to `0o5777` (blind to setgid on
|
||||
a file) or the directory mask to `0o3777` (blind to setuid on a directory)
|
||||
each leave 183 passing. **Only setuid-on-a-file is pinned.** A future edit
|
||||
narrowing either mask readmits setgid or sticky with a green suite, and
|
||||
setgid on a directory is the one with real semantics.
|
||||
- **F6, informational.** `nix_group_or_skip` (`:12248`) never skips — it
|
||||
returns `Ok(1)` unconditionally, and its doc's reasoning about `/etc/group`
|
||||
is moot because `chown(2)` never consults it. Three tests now hard-require
|
||||
root. Not a new class, but the name says something the function does not do.
|
||||
|
||||
**Promises that held.** **R-I13's `observe_frozen` fixture fix is real**:
|
||||
neutering the digest comparison turns the new end-to-end test red, so the
|
||||
second fixture does tamper where only the digest can see it. **R-I9 verified
|
||||
on a real filesystem rather than by reading the mask**: setuid, setgid and
|
||||
sticky on a file are each refused by name and each move the digest; a foreign
|
||||
gid is accepted by the validator, which still checks only uid, and now moves
|
||||
the digest. R-I10 holds. R-I11 and R-I12 match what Soul could re-derive, and
|
||||
the `:6105`-class check is correctly identified.
|
||||
|
||||
**What Soul could not run.** No non-Linux arm. **No non-root attacker model** —
|
||||
the container is root, which gave F3's `setxattr` and every root-owned
|
||||
precondition free; nothing here shows an unprivileged process can reach any of
|
||||
these tampers. No cargo-mutants sweep, because two other campaigns held slots,
|
||||
so the budget went to twelve targeted hand mutations instead.
|
||||
|
||||
**Rig parked at `F:\Projects\eureka-rigs\idunn-cut1-soul7\`** — outside the
|
||||
repo and the scratchpad: `privilege-surface-probe.rs`, `mutations.py` (the
|
||||
twelve, self-restoring), `README.md`.
|
||||
|
||||
**Soul's judgement: Cut 1 is not done, but what is left is small and named.**
|
||||
The mechanism closed two passes ago and nothing built against it has moved.
|
||||
"If a seventh batch turns up the same shape again — a green test that proves
|
||||
nothing — that is a signal about how this file is being tested, not about the
|
||||
code."
|
||||
|
||||
### Self's rulings for the seventh fix batch, 2026-09-23
|
||||
|
||||
- **R-I14 (F1). Make the count total rather than conventional.** Put
|
||||
`canonicalize` and `symlink_metadata` behind a **narrow injected port**, so
|
||||
the resolver cannot reach the filesystem except through the counted path.
|
||||
Then restoring the S5-1 bug raises the count and the pin fires. Doctrine
|
||||
already asks for this shape — inputs as narrow ports, mockable probes — and
|
||||
a counter that only counts the calls someone remembered to annotate is an
|
||||
honour system. **Also correct this map**: it claimed the regression could
|
||||
not return unnoticed, which was false when written.
|
||||
- **R-I15 (F2). Delete the boundary-shift test.** Not repair it — it cannot
|
||||
fail, and the class it names is unreachable. The production comment saying
|
||||
the term is not independently pinned is the true one; the test's doc comment
|
||||
is the false one and goes with it. **The map's claim that this fixture was
|
||||
already deleted is corrected above.**
|
||||
- **R-I16 (F4). `digest_tree` gets the ruling it was actually given**: full
|
||||
`mode & 0o7777` and gid, for directories as well as files, each term pinned
|
||||
by a test that fails when the term is removed. A tamper check re-derived
|
||||
immediately before launch must not be blind to the bit that grants
|
||||
privilege.
|
||||
- **R-I17 (F5). Each validator mask term gets its own test** — setgid and
|
||||
sticky on a file, setuid, setgid and sticky on a directory — so narrowing
|
||||
the mask cannot pass.
|
||||
- **R-I18 (F3). The digest covers extended attributes**, names and values, in
|
||||
a defined order. `no-new-privileges` is a **prevention on our own deployment
|
||||
path**; the digest is the **detection**, and Cut 4's verdict asserts "this
|
||||
is the tree we froze", which is false today for any xattr. Record plainly
|
||||
that the probe ran as root and does **not** establish that an unprivileged
|
||||
attacker can set one.
|
||||
- **R-I19 (F6).** `nix_group_or_skip` either skips or is renamed for what it
|
||||
does. A helper whose name promises a skip it never performs will mislead the
|
||||
next reader.
|
||||
|
||||
### The seventh fix batch landed, 2026-09-23
|
||||
|
||||
Sonnet, three commits on `idunn/cut1-fix5` (`1edc49a`, `67d0f93`, `a46b997`).
|
||||
**193 passed, 0 failed, 2 ignored** (183 + 11 new − 1 deleted). One file,
|
||||
`src/drivers.rs`, +602 / −48. Two Yggdrasil runs, one slot at a time.
|
||||
|
||||
**The pin now catches its own regression.** R-I14 put `canonicalize` and
|
||||
`symlink_metadata` behind a `FrozenSymlinkFs` port whose single production
|
||||
implementation charges the counter **inside the same method that makes the
|
||||
syscall**, so the two hand-placed call sites are gone and the count is total
|
||||
rather than conventional. Restoring the S5-1 bug through the port — exactly
|
||||
Soul's mutation, which passed in 0.39 s on pass 7 — now reads:
|
||||
|
||||
```
|
||||
test ..._resolves_a_long_component_chain_with_a_linear_call_count ... FAILED
|
||||
```
|
||||
|
||||
- **R-I15.** The unreachable boundary-shift test is deleted, with a comment in
|
||||
its place saying why: the link name is NUL-terminated *before* the target,
|
||||
so any two differing `(name, target)` pairs diverge there regardless, and no
|
||||
real symlink target or filename can carry a NUL. The production comment that
|
||||
pointed at the deleted test now points at that note. **No mutation to run —
|
||||
the ruling was to stop claiming a defence that did not exist.**
|
||||
- **R-I16.** `digest_tree_is_executable` is replaced by `digest_tree_mode`
|
||||
(`mode & 0o7777`) and `digest_tree_gid`, on both the file and directory
|
||||
arms, with four new tests. Deleting the file term fails three tests;
|
||||
deleting the directory term fails two.
|
||||
- **R-I17.** Five new refusal tests, no production change — the masks were
|
||||
already `0o7777` from R-I9 and simply untested per bit. Narrowing the file
|
||||
mask to `0o5777` fails exactly `..._refuses_a_setgid_file`; narrowing the
|
||||
directory mask to `0o3777` fails exactly `..._refuses_a_setuid_directory`,
|
||||
**with the other five refusal tests staying green in each run.** That
|
||||
precision is the evidence the tests pin single bits rather than overlapping.
|
||||
- **R-I18.** `hash_frozen_source_xattrs` hashes names and values sorted by
|
||||
name, count-prefixed and self-delimiting, on `libc::llistxattr`/`lgetxattr`
|
||||
— an existing dependency, no new crate. Two tests: a general `user.`
|
||||
namespace pin, and a direct reproduction of Soul's
|
||||
`security.capability=cap_setuid,ep` finding, which **also asserts the
|
||||
validator still legitimately accepts the tree**, since this ruling was
|
||||
detection through the digest rather than refusal. Both doc comments state
|
||||
plainly that the process runs as root and that this does not establish an
|
||||
unprivileged attacker can reach the same path.
|
||||
- **R-I19.** `nix_group_or_skip` is renamed `second_gid_for_chown_fixture`. It
|
||||
never skipped and never will; only the name lied. A rename has no behaviour
|
||||
to falsify, and Hands said so rather than inventing a mutation for it.
|
||||
|
||||
**Hands shipped a compile bug and caught it itself.** The first commit missed
|
||||
a third pre-existing call site still using the old two-argument signature. The
|
||||
first Yggdrasil run caught it, exit 1, fixed in `a46b997`. Worth recording
|
||||
because it is the system working: the run happened before the report, not
|
||||
after.
|
||||
|
||||
## Soul pass 8 on Cut 1's seventh fix batch, 2026-09-23
|
||||
|
||||
Opus. 193 passed / 0 failed / 2 ignored, reproduced twice on Yggdrasil. **25
|
||||
hand mutations**, 11 not plain reverts, plus six probes on a real root-owned
|
||||
tree. One slot at a time, about 40 minutes held in total.
|
||||
|
||||
**Judgement: Cut 1's mechanism is done. Its verification is not, and the
|
||||
reason is now structural rather than accidental.**
|
||||
|
||||
- **F1, CONFIRMED, medium. The port can be walked around, and the pass-7
|
||||
mutation still passes in two spellings.** At `:6174`, replacing the
|
||||
assignment with `candidate.canonicalize()?` or `fs::canonicalize(&candidate)?`
|
||||
restores the S5-1 cubic bug and leaves **all 193 green**. Only the spelling
|
||||
that goes through the port turns it red. Nothing makes the resolver use the
|
||||
port: no `clippy.toml`, no `disallowed-methods`, `Path` and `fs` both in
|
||||
scope. **Proof by existence:** `open_frozen_symlink_frame` at `:6061`
|
||||
already calls `fs::read_link` directly, and a probe measured a resolution
|
||||
charging 16 port calls while making three uncounted `read_link` syscalls.
|
||||
Hands' own commit message was honest — "written the way the surrounding code
|
||||
is now written". **This map's record was not**: it said "the count is total
|
||||
rather than conventional". It is still conventional. The convention merely
|
||||
moved from "remember to call the counter" to "remember to use the port".
|
||||
- **F2, CONFIRMED, medium, privilege. `digest_artifact`'s single-file path
|
||||
never got R-I16 — and it is the path the installed executable actually
|
||||
takes.** `:7723-7727` returns a content-only sha256 for a regular file: no
|
||||
mode, gid, uid or xattr. `host.rs:682` ("installed executable differs from
|
||||
Expected", immediately before launch) and `host.rs:803` ("the installed
|
||||
executable changed under the running process") both hand it a single
|
||||
executable file. Measured: setuid **blind**, `0555` versus `0500` **blind**,
|
||||
`security.capability=cap_setuid=ep` **blind**. **All three
|
||||
`digest_artifact` tests wrap the file in a directory so `digest_tree` runs;
|
||||
the fast path is untested** — and one of their doc comments claims it pins
|
||||
"exactly Soul pass 7's post-install-tamper-adding-setuid-to-an-installed-binary
|
||||
scenario". The real scenario is a bare file.
|
||||
- **F3, CONFIRMED, medium. The artifact digest has no xattr term at all.**
|
||||
`digest_tree` got R-I16's mode and gid and none of R-I18, while
|
||||
`hash_frozen_source_tree` got both. On an artifact tree:
|
||||
`security.capability` blind, `user.*` blind, uid blind. **The asymmetry runs
|
||||
the wrong way** — the frozen source is a 0555 root-owned tree nothing
|
||||
executes; the artifact is the binary that gets exec'd and re-verified before
|
||||
every launch.
|
||||
- **F4, CONFIRMED, medium. R-I18's framing and its non-file coverage are
|
||||
pinned by nothing.** The encoding is genuinely self-delimiting and Soul
|
||||
could not collide it across four adjacent-boundary attacks. **But the suite
|
||||
does not know that.** All 193 stay green while dropping the value length
|
||||
prefix, dropping the count prefix, dropping the name NUL, folding names to
|
||||
lowercase (a real collision class, `user.A` against `user.a`), and — the
|
||||
material one — **hashing names only and dropping values entirely. Nothing
|
||||
proves an xattr value is hashed**, so `security.capability` edited from
|
||||
`cap_setuid` to `cap_sys_admin` is covered by no test. Removing the xattr
|
||||
term from the directory and symlink arms also passes; the code covers both,
|
||||
confirmed on a real tree, but only the file arm is tested. `llistxattr` is
|
||||
the right call and does work on symlinks.
|
||||
- **F5, CONFIRMED, low. The count test's doc still describes the pre-R-I14
|
||||
machine** and still tells the reader the pin is blind (`:11908-11938`,
|
||||
untouched across the batch). Pass 7 found code honest and record
|
||||
overclaiming; this is the inverse of the same defect. The record says
|
||||
"total", the test's own doc says "blind", and neither matches what Soul
|
||||
measured.
|
||||
|
||||
**Promises that held, with numbers.** R-I14's pin does fire through the port
|
||||
(three separate mutations, each red on exactly the count test). **R-I17's
|
||||
precision holds further than Hands claimed — all six mask bits, not two**:
|
||||
each of six narrowings fails exactly its own refusal test with the other five
|
||||
green. R-I16's term claims hold exactly: mode→0 fails three, gid→0 fails two,
|
||||
the directory arm alone fails exactly the two directory tests, the file arm
|
||||
alone exactly the three file tests. R-I18 does catch the finding it was
|
||||
written for. R-I15 is a real deletion and R-I19 a rename with nothing to
|
||||
falsify, correctly stated.
|
||||
|
||||
**Remaining surface.** FIFOs, sockets and device nodes are refused by the
|
||||
validator and both digests — closed. ACLs are xattrs, so covered in the frozen
|
||||
source and uncovered in the artifact. Hard links accepted and harmless, mtime
|
||||
blind and harmless, artifact uid blind and never ruled on. `trusted.*` is
|
||||
invisible without `CAP_SYS_ADMIN`.
|
||||
|
||||
**Could not run:** no non-Linux arm; **no non-root attacker model**, since the
|
||||
container is root and every `lsetxattr`, `chown` and `mknod` precondition came
|
||||
free; no cargo-mutants sweep.
|
||||
|
||||
**Rig at `F:\Projects\eureka-rigs\idunn-cut1-soul8\`** with 25 self-restoring
|
||||
mutations and both raw logs. It documents its own defect: under
|
||||
`--nocapture --test-threads=1`, cargo's `test <name> ...` line has no newline,
|
||||
so a `grep "^P[0-9]"` silently eats each test's first `println!`. That cost a
|
||||
slot, and it is the second time this campaign has lost output to that exact
|
||||
artifact.
|
||||
|
||||
### The cause Soul named
|
||||
|
||||
> *"Every pin in this file is a convention a future edit steps around, and the
|
||||
> tests are all built on the fixture shape that is convenient rather than the
|
||||
> shape production uses."*
|
||||
|
||||
The counter was an honour system; the port is an honour system one level up.
|
||||
The digest tests wrap a file in a directory; production hands `digest_artifact`
|
||||
a bare file. R-I18 tests presence against absence; the thing that carries
|
||||
privilege is the value. **One cause, three faces — and an eighth batch of
|
||||
point fixes would produce a ninth.**
|
||||
|
||||
### Self's rulings for the eighth fix batch, 2026-09-23
|
||||
|
||||
- **R-I20 (F2). Land this one regardless of anything else.** It is a real
|
||||
detection hole on the launch path. `digest_artifact`'s file branch
|
||||
**delegates to the same primitive `digest_tree` uses**, rather than growing
|
||||
its own copy of the terms. **And its tests use the production shape — a bare
|
||||
file — not a file wrapped in a directory.** The wrapped fixture is why this
|
||||
survived: it exercised a path production does not take.
|
||||
- **R-I21 (F3). The artifact digest covers xattrs**, like the frozen-source
|
||||
digest. The binary that gets executed deserves at least what the tree nobody
|
||||
executes already has.
|
||||
- **R-I22 (F4). Pin the framing, not the presence.** Tests that fail when the
|
||||
value is dropped, when the count prefix goes, when the name terminator goes,
|
||||
and when names are case-folded; plus the directory and symlink arms. **The
|
||||
value is the part that carries privilege**, and it is currently pinned by
|
||||
nothing.
|
||||
- **R-I23 (F1). Make the bypass a compile error.** A `clippy.toml` with
|
||||
`disallowed-methods` for `Path::canonicalize`, `fs::canonicalize`,
|
||||
`fs::symlink_metadata` and `fs::read_link` within this module, denied in CI.
|
||||
**This is what R-I14 was actually asking for**, and it is the doctrine's own
|
||||
answer: move the authority instead of maintaining a convention. Convert
|
||||
`open_frozen_symlink_frame`'s direct `fs::read_link` to the port in the same
|
||||
pass, since it is the proof the convention leaks.
|
||||
- **R-I24 (F5).** Rewrite the count test's doc to describe the port. A comment
|
||||
that tells the reader a live pin is blind is worse than no comment.
|
||||
- **Standing, and this one outlives the batch: a test's fixture must be the
|
||||
shape production uses.** Where it cannot be, the test says so in one line.
|
||||
Three of this pass's five findings are that rule being broken.
|
||||
|
||||
### The eighth fix batch landed, 2026-09-23
|
||||
|
||||
Sonnet, `idunn/cut1-fix5` at `75153b0`. **201 passed / 0 failed / 2 ignored**
|
||||
(193 + 8). Ten `ygg-verify.sh` invocations, one slot each, sequential, about
|
||||
eleven minutes of slot time in total.
|
||||
|
||||
**R-I23: the bypass is now a compile error, and it was verified by failing
|
||||
builds rather than by reading.** Everything the resolver can reach moved into
|
||||
a `frozen_symlink_port` submodule carrying `#![deny(clippy::disallowed_methods)]`,
|
||||
with `read_link` added to the port and `open_frozen_symlink_frame`'s direct
|
||||
`fs::read_link` — the exact leak Soul named — converted. `clippy.toml` lists
|
||||
`Path::canonicalize`, `fs::canonicalize`, `fs::symlink_metadata` and
|
||||
`fs::read_link`. Since `disallowed-methods` has no per-module scope,
|
||||
`drivers.rs` blanket-allows at file scope and the submodule's inner `deny`
|
||||
overrides it back — documented lexical scoping, not a hopeful guess.
|
||||
|
||||
On Yggdrasil, against real source: Soul's `candidate.canonicalize()?` and
|
||||
`fs::canonicalize(&candidate)?` **each fail the build** with
|
||||
`use of a disallowed method`, as does reintroducing the `fs::read_link` leak.
|
||||
Clean baseline builds before and after each, file restored and reconfirmed.
|
||||
|
||||
**The repository had no CI surface at all** before this batch — "denied in CI"
|
||||
had nothing to attach to — so `.github/workflows/lint.yml` is new.
|
||||
|
||||
**R-I20 and R-I21.** `digest_artifact`'s bare-file branch now delegates to
|
||||
`digest_file_entry`, the same primitive `digest_tree`'s file arm uses, instead
|
||||
of a content-only `raw_sha256`. An xattr term covers all three `digest_tree`
|
||||
arms through shared framing with the frozen-source side, so the two cannot
|
||||
drift. The three tests that wrapped a file in a directory — **which is exactly
|
||||
why the bare-file blindness survived three batches** — now call
|
||||
`digest_artifact` on the file directly.
|
||||
|
||||
Reverting the file branch to its old body fails all three of the gid, setuid
|
||||
and xattr tests, so they pin something.
|
||||
|
||||
**Two real bugs found by running the suite rather than trusting it.**
|
||||
`digest_tree(root, root, …)` never hashes the root's own metadata, so a
|
||||
directory-arm test setting the xattr on the artifact root tested nothing;
|
||||
moved to a child. And `lsetxattr` with a `user.*` name on a symlink returns
|
||||
`EPERM` **even as root** — a kernel restriction to regular files and
|
||||
directories, not a privilege problem — so that arm uses `security.capability`.
|
||||
|
||||
**R-I22.** Five tests call `hash_xattr_list` directly with lists chosen so
|
||||
each framing piece is what causes a real collision when removed, rather than
|
||||
merely changing the digest: value length prefix, count prefix, name
|
||||
terminator, name case, and value-versus-names-only. Each was verified against
|
||||
a real line-targeted mutation of production code, each failing exactly its own
|
||||
test. (A python heredoc attempt hit the `\0`-escaping trap the rig's README
|
||||
warns about; Hands switched to `sed` rather than fighting it.)
|
||||
|
||||
**R-I24.** The doc is rewritten and **the formula corrected to `components + 4`**,
|
||||
not `+ 2` — `read_link` and the now-ported root canonicalize each add one. The
|
||||
false "cannot catch its own regression" line is gone: it was true before
|
||||
R-I23, and the compile-time deny now catches Soul's mutation independently of
|
||||
what the count asserts.
|
||||
|
||||
### Self's finding: the digest format change is a migration hazard
|
||||
|
||||
Hands flagged that `digest_artifact`'s single-file format changed, and
|
||||
reasoned that nothing compares against literals. **That is true of fixtures
|
||||
and false of receipts.** `host.rs:682` compares a freshly computed digest
|
||||
against `expected.artifact_sha256`, and `host.rs:803` against
|
||||
`prior.executable_sha256`. Both are **stored** values.
|
||||
|
||||
So on the first Idunn upgrade past this commit:
|
||||
|
||||
- every already-installed workload whose `Expected` was sealed under the old
|
||||
format fails `"installed executable differs from Expected"` and refuses to
|
||||
launch;
|
||||
- every workload already running fails `"the installed executable changed
|
||||
under the running process"`.
|
||||
|
||||
**Idunn is the deployment and daemon-survival authority.** A change that makes
|
||||
its own upgrade refuse every previously admitted artifact is precisely the
|
||||
class of failure its continuity rules exist to prevent, and it would land
|
||||
silently, at restart, on the whole fleet.
|
||||
|
||||
- **R-I25. The artifact digest carries an explicit format version**, recorded
|
||||
with the stored value. Verification recomputes **in the version the receipt
|
||||
names** and compares against that, so an existing receipt still verifies;
|
||||
new seals use the current version. A bare unversioned value is read as
|
||||
version 1.
|
||||
- **No re-seal-everything migration.** Re-sealing on upgrade would mean
|
||||
Idunn rewriting admitted state to make its own new code agree with itself,
|
||||
which is the opposite of what a receipt is for.
|
||||
- **A test must cover the upgrade path directly**: seal under the old format,
|
||||
verify under the new code, and pass. Until that test exists, this is not
|
||||
closed — the whole finding is that nothing exercised the stored-value path.
|
||||
|
||||
**Also owed:** `cargo clippy --all-targets` already fails on a pre-existing
|
||||
`clippy::never_loop` in `host_actuator.rs`'s test code, present on the parent
|
||||
commit too. Hands scoped the workflow to avoid it and said so rather than
|
||||
absorbing it silently. **Fix `never_loop` and restore `--all-targets`** — test
|
||||
code is exactly where a bypass would hide, and it is currently the one place
|
||||
the new lint does not reach.
|
||||
|
||||
### The ninth fix batch landed, 2026-09-23
|
||||
|
||||
Sonnet, `idunn/cut1-fix5`. **206 passed / 0 failed / 2 ignored** (201 + 5).
|
||||
Five sequential Yggdrasil runs.
|
||||
|
||||
**R-I25 could not be implemented as written, and Hands said so.** An explicit
|
||||
version tag cannot be recorded beside the stored value: both
|
||||
`expected.artifact_sha256` and `ArtifactReceipt::sha256` are validated to a
|
||||
bare `sha256-<64 lowercase hex>` with no room for metadata, and that validator
|
||||
lives in the **pinned `cultnet-rs` dependency** — a separate repository this
|
||||
worktree has no authority to widen.
|
||||
|
||||
What landed instead is an **implicit version read**: `verify_artifact_digest`
|
||||
recomputes under the current algorithm, then falls back to the exact shape
|
||||
`digest_artifact` produced before R-I20. There has only ever been one prior
|
||||
shape, so the fallback is total. New seals always call `digest_artifact`
|
||||
directly and therefore always use the current shape; **nothing rewrites a
|
||||
stored value**, and `start()` carries `expected.artifact_sha256` forward into
|
||||
its observation rather than recomputing, so a legacy receipt is echoed and not
|
||||
silently upgraded.
|
||||
|
||||
**The proof is a test that fails without the fix**, which is what this finding
|
||||
was about. Five new tests — the upgrade path, a tamper still caught under the
|
||||
legacy shape, a no-reseal check, the frozen-source equivalent, and one at the
|
||||
real `host.rs` call site. Stripping the fallback and rerunning: **all five
|
||||
fail.** Restored, 206/0/2.
|
||||
|
||||
**The frozen-source digest was exposed too, and this cut's own batches crossed
|
||||
the window twice.** `hash_frozen_source_tree` changed in batch 6 (mode and
|
||||
gid) and again in batch 7 (xattrs), while `observe_frozen` compares a fresh
|
||||
recompute against `receipt.snapshot_sha256` — a `FrozenSourceReceipt`
|
||||
persisted in `DeploymentTransaction.frozen_source` **between two separate
|
||||
`advance_sealing` calls that can cross a restart**. The window is narrower
|
||||
than the artifact side — one in-flight Sealing-phase transaction rather than
|
||||
every installed workload — but it is real. `verify_frozen_source_digest`
|
||||
applies the same answer against both prior shapes, reproduced verbatim from
|
||||
git history rather than reconstructed from memory.
|
||||
|
||||
**R-I26.** The `never_loop` was a test-only `while let` whose only body path
|
||||
returned, so it could never reach a second iteration; the surrounding `for`
|
||||
loop was already doing the retrying. Changed to `if let`. The workflow now
|
||||
runs `cargo clippy --all-targets --all-features -D clippy::disallowed_methods`,
|
||||
verified **exit 0** on Yggdrasil after adding the clippy component, which the
|
||||
base image lacks. Only unrelated, undenied warnings remain.
|
||||
|
||||
### Self's rulings
|
||||
|
||||
- **R-I27. Hands made the right call proceeding.** The instruction named a
|
||||
mechanism; the hazard was live and the alternative was to halt and leave the
|
||||
fleet exposed. An equivalent fix, implemented and flagged with its
|
||||
divergence, beats a literal one that cannot be built. **Standing: where a
|
||||
ruling names a mechanism and the mechanism is unavailable, satisfy the
|
||||
ruling's purpose and report the substitution** — do not halt, and do not
|
||||
quietly redefine the ruling as satisfied.
|
||||
- **R-I28. The fallback chain is a liability that grows with every format
|
||||
change**, and it is only tolerable because there is exactly one prior shape
|
||||
per digest. **Follow-up outside this cut: widen the stored value's shape in
|
||||
`cultnet-rs` to carry an explicit version**, then collapse the fallbacks.
|
||||
Recorded, not scheduled — it is a cross-repo change on a pinned dependency
|
||||
and does not belong inside Cut 1.
|
||||
- **R-I29. Investigate the third instance Hands spotted and did not chase.**
|
||||
`SystemdWorkloadActuator`'s native-process path compares `sha256_reader` —
|
||||
raw, content-only, read through procfs — against `expected.artifact_sha256`,
|
||||
which for a file-based artifact is sealed via `digest_artifact` and now
|
||||
carries mode, gid and xattr terms. **Two structurally different algorithms
|
||||
compared to each other** is the same class of defect as the two above. Hands
|
||||
flagged it rather than guessing, correctly. Settle whether it is a live
|
||||
mismatch or a misreading, and report before fixing.
|
||||
|
|
|
|||
352
docs/watchdog-reporting-map.md
Normal file
352
docs/watchdog-reporting-map.md
Normal file
|
|
@ -0,0 +1,352 @@
|
|||
# Watchdog reporting: map
|
||||
|
||||
Campaign `idunn-watchdog`. Questions, rulings, cut specs, reports and verdicts
|
||||
are typed documents in Eureka's mind; this file keeps only body facts, the
|
||||
model page and rationale. Target rationale: `watchdog-reporting-target.md`
|
||||
(`4951125`).
|
||||
|
||||
## Body facts
|
||||
|
||||
Source reads are at Idunn `4951125` (main), Idunn `hands/b5` (`0c80a44`,
|
||||
unmerged, read with `git show`/`git diff main...hands/b5` only), Bifrost
|
||||
`42728a6`, Odin `cdc57bd`, VoidBot `46d891b`, gamecult-ops `14df7ce`. Host
|
||||
probes ran on Yggdrasil over `ssh ygg`, read-only, 2026-09-30 13:51-13:55 UTC
|
||||
(15:51-15:55 CEST).
|
||||
|
||||
### Idunn
|
||||
|
||||
| # | What | Where | Result |
|
||||
|---|---|---|---|
|
||||
| I1 | Alert path | `src/control_plane.rs:3798-3812` (`ReportOnce`), `:4599-4610` (`report_once`), `:5114-5138` (`note_fault`, `clear_fault`) | The only operator signal is `eprintln!`. De-dup is `fault_reports: Mutex<BTreeMap<String, ReportOnce>>` (`:4392`), in memory: an Idunn restart forgets it and reprints. `note_fault` also writes the transaction's `last_error`. |
|
||||
| I2 | Continuity exhaustion (since `idunn-reincarnation` cut 1: `restarts_page`, a paging threshold only; nothing stops, and the line numbers below are pre-cut) | `:68-71` (6 restarts / 3,600,000 ms, 5 s doubling), `:1066-1072` (`restarts_used`, `restarts_exhausted`), decided `:5709-5722` under key `continuity:<target>` (`:5678`) | Exhaustion is **derived** from `TargetSupervision.continuity_restarts` (newest 6 times, sliding window). No write marks it. The fault clears (`:5723`) as soon as one entry leaves the window, so a permanently dead target leaves and re-enters exhaustion several times each hour. |
|
||||
| I3 | Route degradation | `RouteSupervisionState` `:890-942`; `record_failed_challenge` sets `degraded_since_unix_millis` on the **first** failed challenge, `record_proved_challenge` clears it; reported at `:5975` under `route:<target>` | Durable per incarnation (inside `admitted_generation.v4`); a new generation starts from `default()`. Backoff doubles from the topology max age to a cap. |
|
||||
| I4 | Other operator-facing faults | `:5645-5651` (`generation-down:<target>`, "holds a dead admitted generation down"), `:6016` (`route-record:<target>`), `note_fault` per transaction | Also stderr plus `last_error`. Not named by the target; listed so the paging-set question sees them. |
|
||||
| I5 | `OperatorRequired` on main (pre-cut-2 reading; since `idunn-reincarnation` cut 2 `resolve` reaches it in production through `PhaseEnd::OperatorRequired` and `PhaseEnd::RetryForward`, both calling `leave_to_operator`, which opens the `operator-required` incident `IncidentCondition::OperatorRequired`, subject the transaction id; `close_spent_operator_incidents` closes it when the record no longer `leaves_to_operator`) | `TerminalRecovery::OperatorRequired` `:878-885`, validated `:1954`; constructed only in tests (`:12525`, `:12535`) | No production path produces it on main. |
|
||||
| I6 | `OperatorRequired` on B5 (superseded by cut 2: B5's deadlines merged as `PhaseDeadline`, and a continuity past its deadline now also lands here) | `hands/b5` `resolve_phase_deadline` (diff hunk at `control_plane.rs` ~8560-8625 on the branch), `PhaseEnd::OperatorRequired` | Deadline passed on a stateful deploy with an issued lease: `report_once("deadline:<transaction_id>")`, returns `Ok(None)`, the transaction stays live. No terminal transition and no operator action that ends it were found on the branch (`idunn expire` writes `idunn.expiry_request.v1`, which re-enters the same arm). |
|
||||
| I7 | Store kinds | `:55-61`, `:936-966`; B5 adds `idunn.expiry_request.v1` | `control.cc`: `idunn.deployment_command`, `idunn.deployment_transaction`, `idunn.admitted_generation.v4`, `idunn.target_supervision.v1`. `history.cc` beside it holds terminal transactions (`:3653`), retired one per tick (`:5068-5091`). |
|
||||
| I8 | Single-writer precedent inside `control.cc` | B5 `ExpiryRequest` doc comment | "The CLI only writes this record. The daemon's deadline resolver reads it ... No CLI writes a transaction field." One writer per kind. |
|
||||
| I9 | Serve loop | `:5041-5058`, `run_scheduler_tick` `:5098-5110` | 500 ms poll. A tick error is logged, never propagated. |
|
||||
| I10 | `idunn status` | `render_supervision` `:4215` | Renders restarts used/limit and `degraded-since`; no incident notion. |
|
||||
| I11 | Idunn already talks CultNet to targets | `src/drivers.rs:5567-5600` `request_runtime_presence_at` | `CultNetMessage::SnapshotRequest { schema_ids, record_keys }` over the route transport (http/tcp/rudp) to a target endpoint. Precedent for reading a typed record out of Odin's catalog. |
|
||||
| I12 | Idunn reads Odin's file read-only | `src/drivers.rs:5005-5020` (`receive`), unit `--odin-correlation-store /var/lib/gamecult/odin/topology.cc` | Absent file yields `None`; continuity never consults it. |
|
||||
| I13 | Idunn's published surface | unit `deploy/idunn-yggdrasil.service` comment on `ReadWritePaths` | "the topology store is Idunn's *published* surface, and every managed target must read it". Lives in `/var/lib/gamecult/idunn-projection`, outside Idunn's 0750 root. |
|
||||
| I14 | Rejected predecessor | `git show 821a8d0^:src/main.rs:2020-2060, 8206-8250` | Before the 2026-09-02 rebuild, Idunn wrote an `IdunnOperatorAlarmRecord` (`alarm_id = alarm:<daemon>:<now>`, one per decision) and ran an operator-supplied shell `--operator-alarm-command` synchronously with a timeout, passing `IDUNN_ALARM_*` env. The rebuild deleted it. |
|
||||
| I15 | Stale guide lines | `docs/guide.md:47` "escalate to an operator through Bifrost"; `:328` "Idunn is not Bifrost ... Discord delivery, owner DMs" | `:47` describes a capability that does not exist; `:328` stays true. |
|
||||
|
||||
### Yggdrasil
|
||||
|
||||
| # | Command | Result |
|
||||
|---|---|---|
|
||||
| Y1 | `systemctl is-active idunn-yggdrasil bifrost bifrost-persona-mouth bifrost-persona-feedback` | all `active` |
|
||||
| Y2 | `systemctl show idunn-yggdrasil -p User -p ProtectSystem -p ReadWritePaths` | `User=root`, `ProtectSystem=full`. `ReadWritePaths` names no Bifrost path, but `ProtectSystem=full` only makes `/usr`, `/boot`, `/etc` read-only, so root Idunn can physically write `/srv/bifrost` and `/var/lib/gamecult/bifrost`. The `/srv` and `/var/lib` entries are documentation, not a boundary. |
|
||||
| Y3 | `ls -la /srv/bifrost/env` | `persona-delivery.env` **exists**: `root:root 0600`, 135 bytes, 2026-09-04 02:20. gamecult-ops `inventory.md:741` ("the file is absent") is stale. |
|
||||
| Y4 | `sudo awk -F= '{print $1, length($2)}' persona-delivery.env` (values not printed) | `BIFROST_DISCORD_BOT_TOKEN` 72 chars, `DISCORD_OWNER_ID` 18 chars. The token and the recipient binding are both already in Bifrost's env. |
|
||||
| Y5 | `systemctl cat bifrost-persona-mouth` | `User=bifrost-feedback`, `ProtectSystem=strict`, `EnvironmentFile=-/srv/bifrost/env/persona-delivery.env`, `ExecStart=node .../tools/persona-feedback.mjs serve-persona-delivery ...` with signed-request RUDP listener `10.77.0.1:17876` and Starfire permit requester `rudp://10.77.0.2:17877`. |
|
||||
| Y6 | `systemctl show bifrost-persona-mouth -p ActiveEnterTimestamp -p NRestarts` | Started 2026-08-21 10:51:22 UTC, 0 restarts: **before** the env file was written, so the running process has no token loaded. |
|
||||
| Y7 | `readlink -f .../persona-feedback/runtime/current`; `grep -c discord-dm .../cultmesh-bridge-commands.mjs` | Current release `cb3239a` (2026-08-08). It ships `cultmesh-bridge-commands.mjs`, `operator-notification.mjs`, `bifrost-bridge.mjs`, but `c063f27` (discord-dm verb, 2026-09-04) is **not** an ancestor: the deployed pump cannot send a DM. `discord-dm` count 0. |
|
||||
| Y8 | `find /srv/bifrost /var/lib/gamecult/bifrost -name provider-store.cc` | None. The pump's store has never existed on the host. |
|
||||
| Y9 | `getfacl /var/lib/gamecult/idunn-projection`; `ls -ld /var/lib/gamecult/idunn` | Projection dir `root`, `other::r-x`, default `other::r--`: world-readable. `/var/lib/gamecult/idunn` is `idunn:idunn 0750`; `control.cc` is unreadable to other users. |
|
||||
| Y10 | `id idunn`; `getent passwd bifrost bifrost-feedback` | `idunn` 986/978; `bifrost` 111/113; `bifrost-feedback` 994/983. No shared group. |
|
||||
|
||||
### Bifrost
|
||||
|
||||
| # | What | Where | Result |
|
||||
|---|---|---|---|
|
||||
| BF1 | Alarm publisher | `tools/operator-notification.mjs:41-148` | Opens Bifrost's `.bifrost/provider-store.cc` (repo-relative, `:20`) and `put`s a `bifrost.bridge.discord_post_command.v1` `discord-dm`. The **caller** supplies `recipientId` (`:51-54`). `commandId = idunn-alarm-sha1(alarm JSON)[0..16]`, and the JSON includes `raisedAt`, which defaults to now (`:48`): a re-publish after a restart mints a new id. |
|
||||
| BF2 | Pump | `tools/cultmesh-bridge-commands.mjs:50-71, 133-221` | One-shot `process`: selects `pending` and `running`, marks `running`, spawns `bifrost-bridge.mjs`, writes `discord_post_receipt.v1` and `completed`/`failed`. `failed` is terminal (no retry). A crash after the Discord POST and before the receipt leaves `running`, which the next run resends. |
|
||||
| BF3 | Discord send | `tools/bifrost-bridge.mjs:735-770` | Opens the DM channel, posts `content`. No `nonce` is sent (`grep nonce` hits only the Persona permit tools). |
|
||||
| BF4 | No caller authentication on the command path | BF1, BF2 | Whoever can write the store file can make Bifrost post anything to anyone. |
|
||||
| BF5 | Authenticated precedent in Bifrost | `tools/persona-discord-delivery.mjs:27-30`, `tools/persona-discord-permit.mjs` | Persona crossing: signed requests, Epiphany-signed permits, "permit request intent persisted before network" in an execution journal. |
|
||||
| BF6 | History | `git log -S publish-idunn-alarm`: `755649f` (2026-06-07), `c063f27` (2026-09-04) | `c063f27`: "an alarm is a command addressed to the gate, not a document dropped into somebody else's state." |
|
||||
|
||||
### Odin and VoidBot
|
||||
|
||||
| # | What | Where | Result |
|
||||
|---|---|---|---|
|
||||
| O1 | `odin.store_condition.v1` | Odin `docs/store-recovery-cut.md:158, 199, 318-325` | Not built. One record, key `topology`, held in Odin's in-memory W, rewritten on change, **flushed with the store** and served by Odin's catalog. While `contested` Odin writes no file, and while `unpersisted` it cannot: in exactly the conditions that matter, `topology.cc` on disk does not carry them. Only the catalog does. |
|
||||
| O2 | I-F1, I-F2 | Odin `docs/store-recovery-cut.md:496-500` | I-F2 is this campaign's continuity-exhaustion row. I-F1 (hold deploys on a bad condition) is a deployment-brake decision, not reporting. |
|
||||
| V1 | Peer precedent | VoidBot `packages/core/src/bifrost-discord-command.ts:39-68` | The caller opens Bifrost's store, derives `commandId` from its `idempotencyKey`, optionally spawns the pump itself, and **waits** for the receipt. An unconditional `put` of an existing id overwrites a `completed` command back to `pending`. |
|
||||
| V2 | VoidBot `notify_owner` | Self's mapper, not re-probed | Not live on Yggdrasil; no Discord-capable VoidBot process runs there. |
|
||||
|
||||
### Cut-mapping probes (2026-09-30, 14:00-14:10 UTC)
|
||||
|
||||
Pinned heads: Idunn `487f4f14` (origin/main), Bifrost `42728a67`, gamecult-ops
|
||||
`14df7ce4`, all read with `git ls-remote origin refs/heads/main`. Line numbers
|
||||
in Idunn are unchanged from `4951125`, because `153af6a` and `487f4f1` touched
|
||||
docs only.
|
||||
|
||||
| # | What | Where / command | Result |
|
||||
|---|---|---|---|
|
||||
| C1 | `control.cc` refuses unknown types | Idunn `src/control_plane.rs:2845` | `_ => bail!("Idunn control store contains a foreign document")`. A new type in `control.cc` makes every older binary unable to read the store at all, so a rollback of the incident cut would take continuity down for every target. |
|
||||
| C2 | Idunn's store writes take a blocking exclusive lock; reads can skip it | CultLib `packages/cultcache-rs/src/lib.rs:737-785` | `pull_all_read_only_snapshot` reads unlocked. Writes are `fs2::lock_exclusive`, which blocks. A reader that held the lock could stall an Idunn write. |
|
||||
| C3 | The TS store takes no lock | CultLib `packages/cultcache-ts/src/single-file-messagepack-backing-store.ts` (no `lock` anywhere); present at Bifrost's pinned CultLib `f67f5122` (`pullAll` at `:44`) | A Bifrost reader using `pullAll` cannot block Idunn. |
|
||||
| C4 | A JS reader decodes an Idunn-written store | `scp ygg:/var/lib/gamecult/idunn-projection/topology.cc`; scratch `probe-read.mjs` with cultcache-ts `pullAll` | 18 envelopes decoded (`gamecult.service_trust_anchor.v1`, ...). No `.lock` file was created beside the copy. |
|
||||
| C5 | Rust `DatabaseEntry` payload shape in JS | scratch `probe-decode.mjs`, `@msgpack/msgpack` `decode(envelope.payload)` | A **positional array**: index `n` is `#[cultcache(key = n)]`. The Bifrost reader decodes a tuple, so the seam must be pinned by bytes Idunn wrote, not by a hand-built object. |
|
||||
| C6 | Projection publish mode | Idunn `src/drivers.rs:4385-4404`, `:7670-7685` | Publication is compare-exchange, then `chmod 0644` on the store and its `.lock` sibling. |
|
||||
| C7 | Continuity's healthy site | Idunn `src/control_plane.rs:5631-5633` | `if observation.is_some() { continue; }`: the admitted workload was observed running this pass. The exhaustion decision is `:5709`, and the fault clears at `:5723`. |
|
||||
| C8 | Test harness for exhaustion | Idunn `src/control_plane.rs:17659-17689` | Existing tests use `routed_world(Odin::Unreachable, 1, ...)`, `set_meters`, `routed.workload.kill()` and `routed.tick()`. |
|
||||
| C9 | Idunn has no one-shot or timer workload | `grep -i "oneshot\|\.timer\|OnCalendar" src/` returns nothing; recipe schema `src/deployment.rs:1935-2000` | A target is a resident service with `[service]`, health contract and state slots. Idunn cannot deploy a systemd timer. |
|
||||
| C10 | No Bifrost binding in current Idunn | `sudo ls /etc/gamecult/idunn/bindings/` on ygg | ghostlight, heimdall, odin, raven-muninn, streampixels-service, streampixels-web. No Bifrost. |
|
||||
| C11 | Bifrost's old Idunn manifest is residue | `/srv/odin/deploy-manifests/bifrost-persona-feedback` on ygg (2026-09-04); `grep deploy-manifests\|IDUNN_ACTUATOR src/` in Idunn returns nothing | It requires `IDUNN_ACTUATOR=1` / `IDUNN_COMMAND_AUTHORITY=idunn-daemon` env from the pre-rebuild Idunn, which no longer exists. gamecult-ops `runbooks/bifrost-persona-feedback-yggdrasil.md` still describes that path. The persona-feedback `current` has pointed at `cb3239a` since 2026-08-08. |
|
||||
| C12 | How Idunn itself is installed | ygg `/usr/local/sbin/idunn-a96ad9d-build.sh`; `ls -la /usr/local/bin/idunn` (2026-09-30 13:22) | One hand-written script per commit. It runs `cargo test --lib` and `cargo build --release` in a pinned rust image under `/srv/build/idunn-<sha>` and copies the unit. The install is manual. No such script is in any repo. |
|
||||
| C13 | Legacy alarm consumers in gamecult-ops | `scripts/idunn/notify-idunn-operator-alarm.{ps1,cmd}` (25 + 2 lines); `scripts/idunn/start-idunn-local.ps1:10,134-136` | The only callers of `publish-idunn-alarm`. They pass `--operator-alarm-command` to the deleted pre-rebuild Idunn (I14). No Idunn scheduled task or process runs on Starfire (`Get-ScheduledTask`, `Get-Process`). |
|
||||
| C14 | `discord-dm` consumers in Bifrost | `grep discord-dm` | `cultmesh-bridge-commands.mjs:152,227` exist only for `operator-notification.mjs` (BF6). `bifrost-bridge.mjs` `discord-dm` (`:48`, `:473-535`) is also used by `docs/bridge.md:156` and `tests/Bifrost.Web.Tests/BridgeCliTests.cs:481`, so it stays. |
|
||||
| C15 | Bridge CLI contract | Bifrost `tools/bifrost-bridge.mjs:1266-1289`; `tools/persona-discord-delivery.mjs:45` | The bridge refuses to act without `--cultmesh-command-id`. The Persona precedent spawns it with `--receipt-store` under its private state directory and parses stdout JSON. |
|
||||
| C16 | Crash-recovery precedent | Bifrost `tests/persona-discord-delivery.test.mjs`, "running journal recovers as terminal unknown without a second Discord post" | A `running` execution found at start becomes terminal `unknown` and is never re-posted. |
|
||||
| C17 | Discord nonce | docs.discord.com/developers/resources/message (fetched 2026-09-30) | `nonce`: integer or string, "up to 25 characters". `enforce_nonce`: "checked for uniqueness in the past few minutes"; on a duplicate from the same author, "that message will be returned and no new message will be created". |
|
||||
| C18 | Bifrost tests and mutation tooling | Bifrost: no root `package.json`, `node:test` files under `tests/`, `grep stryker` returns nothing | Tests run with `node --test`. StrykerJS is not installed. How the tests find CultLib: C23 (the tests ignore `VOIDBOT_CULTLIB_ROOT`). |
|
||||
| C19 | Idunn verification | `scripts/verify.sh`; eureka `tools/stopgap/ygg-verify.sh` (rust image carries cargo-mutants) | `cargo check` for the Windows GNU target twice, then `cargo test --locked --lib`. |
|
||||
| C20 | Idunn source mentions no transport | `grep -rin "discord\|bifrost" src/` at `487f4f1` | No hits. This is the negative-grep baseline. |
|
||||
|
||||
### Bifrost suite probes (2026-09-30, 15:00-15:30 UTC)
|
||||
|
||||
Pinned heads: Bifrost `42728a67` (origin/main) and `hands/watchdog-bifrost-retire`
|
||||
`4619939e`; CultLib pin `36ea08d3` (origin/main `016df462`); Epiphany origin/main
|
||||
`4d1113ef`. Probes ran in a scratch clone, never in `F:\Projects\Bifrost`. Suite
|
||||
runs went through `ygg-verify.sh` in `node:24.14.1-bookworm`, with CultLib cloned
|
||||
to `/CultLib`, the sibling of `/src`, then `npm ci` and `npm run build:ts`. Each
|
||||
file ran alone with the TAP reporter, then the whole glob ran once.
|
||||
|
||||
| # | What | Where / command | Result |
|
||||
|---|---|---|---|
|
||||
| C21 | Bare `cultcache-ts` require sites | Bifrost `git grep -nF '("cultcache-ts")'` at `42728a67` | `tests/persona-feedback-cli.test.mjs:15,55,62`, `tests/persona-discord-delivery.test.mjs:33`, `tests/persona-discord-crossing-rust-smoke.test.mjs:13`, `tools/persona-feedback.mjs:114`, `tools/agent-transport.mjs:40`, `tools/governance-threads.mjs:36`, and `tools/operator-notification.mjs:186`, which cut `bifrost-retire-alarm` deletes. `persona-feedback.mjs` calls `loadRuntime()` at module top (`:23`), so every verb fails at import. The bare `cultnet-ts` and `cultmesh-ts` sites resolve. Tools that load `dist/index.js` by path do not depend on the name: `bifrost-crossing-documents.mjs:271`, `bifrost-repository-release-authority.mjs:161`, `cultmesh-bridge-commands.mjs:313-314` and `provider-advertisement.mjs:47-119`. |
|
||||
| C22 | What CultLib publishes | `git show <rev>:packages/{cultcache,cultnet,cultmesh}-ts/package.json` at `36ea08d3` and `016df462`; `git log -S'"@gamecult/cultcache-ts"'` | Only `cultcache-ts` is scoped: `@gamecult/cultcache-ts`, renamed in `8cb3b728` (2026-09-04). `cultnet-ts` and `cultmesh-ts` are still bare and depend on `@gamecult/cultcache-ts ^0.14.0`. The commit message says they "move with it", but only their dependency moved. The directory `packages/cultcache-ts` keeps its name, and every package declares `exports`. A local resolve at `016df462`, anchored at `packages/cultcache-ts/package.json`, gives `@gamecult/cultcache-ts` -> `dist/index.js` and `cultcache-ts` -> `MODULE_NOT_FOUND`. |
|
||||
| C23 | How Bifrost finds CultLib | C21 sites; `ls .github/workflows` | Tools use `VOIDBOT_CULTLIB_ROOT`, falling back to the sibling `../CultLib`. **Tests hard-code the sibling `../CultLib` and ignore the variable.** Bifrost has no `package.json` and no CultLib dependency. No CI runs the node tests: the only workflow is `publish-container.yml`. |
|
||||
| C24 | **Production Persona crossing is not broken by the rename** | `ssh ygg`, read-only: `systemctl cat bifrost-persona-mouth bifrost-persona-feedback`; `cat .../runtime/current/release-manifest.txt`; `createRequire(<release>/CultLib/packages/cultcache-ts/package.json).resolve("cultcache-ts")` | Both units run release `cb3239a…-f67f5122…`. The manifest says `cultlib_commit=f67f5122`, node `v24.14.1`. `VOIDBOT_CULTLIB_ROOT` points at the release's own CultLib. There the package is named `cultcache-ts`, `node_modules` holds real `cultcache-ts`, `cultmesh-ts` and `cultnet-ts` directories, and the bare name resolves to the release's `dist/index.js`. `f67f5122` is an ancestor of `8cb3b728`, so it predates the rename. A restart reloads the same pinned bundle. Since the C21 fix, Bifrost needs CultLib `8cb3b728` or later. A release that pairs it with an older CultLib fails at import (follow_up `stale-bifrost-persona-manifest` owns that path). |
|
||||
| C25 | The Rust fixture tests have no producer | `tests/persona-discord-crossing-rust-smoke.test.mjs:17`, `tests/persona-discord-rudp-cross-language.test.mjs:21`; Epiphany `git log -S persona-discord-crossing-fixture`, `git grep` at `4d1113ef` | Both tests `cargo run --bin epiphany-persona-discord-{crossing,rudp-client}-fixture` in the sibling `../Epiphany`, with `CARGO_TARGET_DIR` hard-coded to `C:\\Users\\Meta\\.cargo-target-codex`. Epiphany deleted both bins in `387afe49` (2026-08-23), and neither exists at `4d1113ef`. So since 2026-08-23 these tests fail on every host, with cargo or without. `persona_discord_crossing.rs` survives as a library module. |
|
||||
| C26 | Suite at base | ygg-verify Bifrost `42728a67`, CultLib `36ea08d3` at `/CultLib` | Per file (tests/pass/fail/skip): rust-smoke 1/0/1/0 (`spawn cargo ENOENT`); delivery 9/3/6/0 (`MODULE_NOT_FOUND 'cultcache-ts'`); permit 2/2/0/0; rudp 1/0/1/0 (`spawn cargo ENOENT`); feedback-cli 1/0/1/0 (the file fails to load, so its 8 tests are counted as 1); idunn-health 3/3/0/0. Glob total 17/8/9, exit 1, the same as Hands h1. |
|
||||
| C27 | Suite with the fix | scratch commit `a6770c5` on `42728a67`: the scoped name at the seven live C21 sites, a skip guard on both Rust tests, and the `CARGO_TARGET_DIR` override removed. Same command. | rust-smoke 1/0/0/1 (`# SKIP`); delivery 9/9/0/0; permit 2/2/0/0; rudp 1/0/0/1 (`# SKIP`); feedback-cli 8/8/0/0; idunn-health 3/3/0/0. Glob total 24 tests, 22 pass, 0 fail, 2 skipped, exit 0. `agent-transport.mjs` and `governance-threads.mjs` were checked only by name resolution (C22), because no test loads them. |
|
||||
| C28 | C3 at the reader's install pin | `git show 36ea08d3:packages/cultcache-ts/src/single-file-messagepack-backing-store.ts` | `pullAll` is at `:43`, and the file has no lock. C3 holds at `36ea08d3`, which is the CultLib that cut `ops-notice-deploy` installs. |
|
||||
|
||||
### Topology-lock cut probes (2026-09-30, 21:00-21:25 UTC)
|
||||
|
||||
Pinned heads: Idunn origin/main `bca3d85`; `hands/watchdog-idunn-incident`
|
||||
local `9549d00` (h3, not yet pushed at probe time; origin at `3ed5f26`);
|
||||
`hands/idunn-projection-mode` `15f7e18` (unmerged); Odin `665dc08`;
|
||||
CodexConnector `6519289`; Ghostlight `eeb3a68`; gamecult-ops `fdb4356`;
|
||||
cultcache-rs at Idunn's pin, CultLib `61549aa`. Line numbers marked *trial*
|
||||
are from a scratch merge of `9549d00` into `bca3d85` (clean, no conflicts),
|
||||
never in `F:\Projects\Idunn`. Yggdrasil probes were read-only over `ssh ygg`;
|
||||
the semantics probe (C36) ran in a container through `ygg-verify.sh`.
|
||||
|
||||
| # | What | Where / command | Result |
|
||||
|---|---|---|---|
|
||||
| C29 | Every writer of `topology.cc` | trial `git grep` for `CultCacheTopologyDriver {`, `Engine::open(`, `mutate(` | One writer: the `idunn serve` daemon. `CultCacheTopologyDriver::mutate` (trial `src/drivers.rs:4393-4411`) is the only write path; its seven callers are `demote_to_expected_only` `:4455`, `withdraw_stale_incarnation` `:4567`, and the `TopologyPort` methods `:4778, 4846, 4897, 4940, 4992`. `Engine::open` has one production caller, `boot` (`src/control_plane.rs:5330`); the CLI verbs `up`, `status`, `cancel`, `expire` and `validate` write only `control.cc`. `idunn-host` and `idunn-provision` never touch the projection. |
|
||||
| C30 | Every reader of `topology.cc`, across repos | `git grep -e 'topology\.cc' -e idunn-projection HEAD` in every repo under `F:\Projects` | Readers: Odin (`crates/odin-daemon/src/lib.rs` `CultCacheIdunnProjectionSource::entries`, `pull_all_read_only_snapshot`, lock-free since its first commit `eedce4f`, 2026-09-04), Idunn itself (`snapshot`, trial `drivers.rs:4383-4388`, lock-free; every `control_plane.rs` read is `pull_all_read_only_snapshot`), and the nightly backup (C33). No managed target reads the file: CodexConnector, Ghostlight, Muninn and StreamPixels name only `odin.verse-topology.v1`, and they receive their Expected record in the runtime bundle (`GAMECULT_IDUNN_RUNTIME_BUNDLE`). The unit comment (`deploy/idunn-yggdrasil.service:77-80`) and `docs/migration.md:505-509` say that "every managed target must read it", which is stale. |
|
||||
| C31 | Shared-lock readers that rely on blocking a writer | cultcache-rs `61549aa` `lib.rs:337-395` (`with_read_only_shared_snapshot`), `:905-910` (`pull_all`, which takes `lock_shared`); cross-repo grep for `with_read_only_shared_snapshot` | That API holds the sibling lock shared so that no writer can replace the snapshot during `action`. Its production users are CodexConnector (`src/idunn_health.rs:441, 562`, `src/daemon.rs:205`) and Ghostlight (`crates/ghostlight-dungeon/src/idunn_health.rs:901`). All of them read the runtime authority record, the process-write-lease record, or their own configuration. None reads `topology.cc`. The write-lease lock is a different file under `idunn-authority` (`CultCacheWriteLeaseDriver`, trial `drivers.rs:4173, 4218`, already `try_compare_exchange_snapshot`). No reader of `topology.cc` takes a lock to get a consistent view. |
|
||||
| C32 | How a reader stays consistent without the lock | cultcache-rs `61549aa` `lib.rs:666-688` (`write_all_unlocked`) | Every write stages `<store>.tmp`, fsyncs it, `replace_file_atomically` renames it over the store, and fsyncs the directory. A reader that opens the path sees one whole inode. Consistency comes from the rename, not the lock. |
|
||||
| C33 | The nightly backup's lock use | live `/usr/local/sbin/gamecult-state-backup` sha256 `9d8ebd84...`, identical to gamecult-ops `scripts/gamecult-state-backup-yggdrasil.sh`; lines 98-126; `systemctl cat gamecult-state-backup.service` | Every file with a `<name>.lock` sibling is re-copied under `flock -s -w 20 -E 75 <lock> cp -a`. Past the wait it falls back to a double-read. The unit has no `User=`, so it runs as root: a `0600 root` lock still opens (`flock(1)` opens the path), and the backup keeps its shared lock. It skips a store whose lock is absent (`[[ -f lock ]] || continue`) and never creates one. The copy is milliseconds of `cp` once a night. Under a try-lock, a publish in that window is contended and retried on a later pass (C35). The backup does not need the lock for a whole copy (C32). |
|
||||
| C34 | Live state of the projection dir | `ls -la`, `getfacl -p /var/lib/gamecult/idunn-projection`; `/proc/locks` against the lock's inode; `find /proc/*/fd -lname` | `drwxr-xr-x+ root`, `default:user::rw- default:group::r-- default:other::r--`. `topology.cc` `0644`, 13436 bytes, written 21:07. `topology.cc.lock` is `0644 root`, inode 8227809, created 2026-09-06. No flock is held on it and no process has either file open. `incidents.cc` does not exist yet, because the incident cut is not installed. The default ACL **is** documented, in Idunn `docs/migration.md:523` (it closes the chmod window), though gamecult-ops does not record it. `idunn-yggdrasil.service`: `User=root`, `UMask=0027`, `WatchdogUSec=0`. Odin runs as DynamicUser uid 62025. |
|
||||
| C35 | What a failed topology publish does inside the tick | trial `src/control_plane.rs` at each call site | Every tick-path write returns `Err` through `?` **before** any evidence of it is persisted, and the step is retried. `advance_starting` records `expected_publication_sha256` or `activation_publication_sha256` only after publish returns `Ok` (`:6605`, `:6664`). `advance_leasing` does the same for the write-lease projection digest (`:7004`, `:7073`). `advance_fencing` withdraws the lease projection before recording (`:6894`, `:6926`). The post-fencing abort and pre-fencing cleanup write `CleanupEvidence::Complete` only after `reconcile_failed_candidate_projection` (`:8956`, `:9046`, `:9180-9204`). A failed continuity demotion (`:6099`) mints no restart and defers continuity. The admitted Expected repair (`:5870`) is logged and re-checked every pass. `retire_stale_incarnations` (`:7914`) returns `Err`. **The one path that is not retried is boot** (`reconcile_failed_continuity_projections`, `:5228-5275`, called once from `boot` `:5332`). It logs a failure and does not retry, and it is outside the tick. |
|
||||
| C36 | Lock semantics under the live ACL | `ygg-verify.sh Idunn bca3d85 python:3.12-slim` running scratch `probe_lock.py`, as root | (1) A file created with mode 0666 under this default ACL and umask 027 lands `0644`, CultCache's shape. (2) A separate process holding `LOCK_EX` on it makes a nonblocking try from the parent contend. (3) **`fchmod 0600` on the same inode does not revoke that holder**: the try still contends. (4) A 0600 `O_CREAT|O_EXCL` staged file renamed over the lock lands `0600`, and the try on the new inode **succeeds** while the old holder still holds the orphaned inode. (5) uid 65534 cannot open the replaced lock (`Permission denied`). |
|
||||
| C37 | The incident cut's private-lock mechanism | `git show 9549d00:src/incident.rs` `:229-245` (`exchange`), `:336-375` (`secure_lock`) | `exchange` calls `secure_lock(path)`, then `try_compare_exchange_snapshot`, and turns `Mismatch` and `LockContended` into errors. `secure_lock` opens `<store>.lock` with `create` and mode `0600`, then runs `set_permissions(0600)` on that descriptor. Its doc says that a descriptor another uid already holds is not revoked. So a lock left `0644` (C34's live `topology.cc.lock`) stays holdable by whoever opened it before (C36 step 3). It builds the lock name by hand, duplicating `authority_lock_path` (trial `drivers.rs:8194`). |
|
||||
| C38 | `hands/b5` | `git merge-base --is-ancestor` | `hands/b5` (`f85e743`) and `hands/b5-pins` (`1cfbeff`) are both ancestors of origin/main (merged at `fcb8ba1` and `ddc031a`). The topology cut has no B5 conflict. |
|
||||
| C39 | Unmerged branches that touch the topology driver | for each branch not in origin/main: `git diff origin/main...<b> -- src/drivers.rs` | Two. `hands/watchdog-idunn-incident` (the dependency). `hands/idunn-projection-mode` (`15f7e18`, 4 commits, 2026-09-30 01:15-02:40 UTC, no campaign doc) deletes `publish_projection_mode`, has `mutate` set a default ACL on the directory by `setxattr` (`publish_directory_default_mode`), and replaces the mode test with `the_published_projection_is_never_visible_in_another_mode`. That test asserts the **lock** is always `0644`, which contradicts `tick-lock-private`. It conflicts textually with this cut at `mutate` and at `publish_projection_mode`. |
|
||||
| C40 | The private root's locks | `ls -ld /var/lib/gamecult/idunn`; `sudo ls -la` inside it; `getent passwd idunn`; `ps` | `/var/lib/gamecult/idunn` is `drwxr-x--- idunn:idunn` (uid 986, group 978, which has no members; no process runs as `idunn`). `control.cc.lock` is `0644 root`. The tick's control-store writes use blocking `compare_exchange` (e.g. trial `control_plane.rs:5471`), so root (the backup's `flock -s`) or uid `idunn` could hold one. Nothing runs as `idunn` today. |
|
||||
|
||||
## Model page
|
||||
|
||||
A row per persistent kind this campaign creates or depends on. "New" means the
|
||||
kind does not exist yet. Names in `code` for new kinds are working names; the
|
||||
cut spec fixes them.
|
||||
|
||||
| Kind | Identity | Lifecycle | Authority |
|
||||
|---|---|---|---|
|
||||
| **Incident record** (new, `idunn.operator_incident.v1`, its own file `/var/lib/gamecult/idunn-projection/incidents.cc`, published 0644 beside `topology.cc`) | Key `<condition>:<subject>:<opened_at_unix_ms>`. Namespace: `incidents.cc`, one type; the file refuses any other. It is never written to `control.cc`, which bails on unknown types, so a rollback would stop continuity (C1). `condition` is a closed enum (`continuity-exhausted` first; `odin-store-contested`, `odin-store-unpersisted` and `operator-required` join when their sources exist). `subject` is the target for target conditions and the `transaction_id` for `operator-required`. `opened_at` is written once by Idunn's clock. Injective because of one rule: **at most one open incident per `(condition, subject)`**, checked by a compare-exchange create. | **Created** at the site that decides the condition, after that decision and before its trace line. **Closed** by writing `closed_at` and `close_reason` on the same record, never by deleting it. Continuity-exhausted closes when the pass sees the workload running (`recovered`) or the target is no longer admitted, **not** when the window slides (I2). Odin conditions close when the catalog stops reporting them. `operator-required` has **no closing path yet** (I6; follow_up `b5-operator-required-no-exit`). **Reopened**: never; a recurrence is a new record with a new `opened_at`. **Replayed after an Idunn restart**: the durable open record makes the site write nothing. **Pruned**: closed more than 7 days ago, it retires to `history.cc`. **Store failure**: reported once as a fault and never returned into the tick (`survival-independent`). | Idunn's daemon, at the decision site, is the only writer. The same file is the record and what Bifrost reads, so no separate projection exists to drift. Forbidden: the CLI (`status` reads only), Bifrost and every other reader, Odin, `ReportOnce`, `control.cc`, a repair loop that opens incidents from log text. |
|
||||
| **Projection lock files** (exist: `topology.cc.lock`; new with the incident cut: `incidents.cc.lock`, `incident-history.cc.lock`) | `<store>.lock` beside its store, named by `authority_lock_path`. One per store file. | **Created** by Idunn at mode `0600` before its first exchange on that store. **Replaced** by a staged `0600` file renamed over it whenever Idunn finds it wider than `0600` or not owned by Idunn's euid; a chmod cannot revoke an existing holder (C36). **Held** only for one nonblocking exchange. **Never published**: the store is `0644`, and its lock is not. | Idunn's daemon alone opens it for writing. The root backup may take it shared for its copy (C33), and a contended exchange is retried. Forbidden: any reader opening it (none does, C30-C31), `publish_file_mode` on it, a default ACL deciding its mode. |
|
||||
| **Stderr trace** (exists) | `report_once` key, in memory | Printed once per process lifetime per fault; lost on restart (I1). | **Demoted**: `ReportOnce` is no longer an owner of "has the operator been told". It de-dups stderr only. |
|
||||
| **Delivery request / journal entry** (Bifrost store) | Keyed by the Idunn incident key plus the notice kind (`opened`, and `closed` if Q `closure-notice` says so). Injective because the incident key is. Not derived from payload text or time (BF1's `raisedAt` hash is the counter-example). | **Created** by Bifrost when its reader first sees a notice-worthy incident state in the projection. **Revised**: `pending` -> `sent` or `failed`, with an attempt count and last error. **Retried** until sent, with backoff; a create for an existing key writes nothing (V1's unconditional `put` is the counter-example). **Replayed after a Bifrost restart**: journal is durable; a `pending` entry is retried. **Pruned** only after the incident key has left Idunn's projection. | Bifrost's reader only. Forbidden: Idunn (it never writes Bifrost's store, even though Y2 shows it physically could), VoidBot, the operator by hand. Which Bifrost process: see Rationale. |
|
||||
| **Delivery receipt** (Bifrost, `discord_post_receipt.v1` or the journal's `sent` state) | Same key as the request. | Written once on success with the Discord message id. | Bifrost. **Idunn does nothing with it**: it neither reads it in the daemon nor waits on it (`survival-independent`). `idunn status` does not show delivery state; Bifrost's journal is where a failed delivery is visible. |
|
||||
| **Recipient binding** | `DISCORD_OWNER_ID` in `/srv/bifrost/env/persona-delivery.env` (Y4). One recipient, the operator. | Operator-supplied; changes by editing that file. | Bifrost (with the operator as supplier). Forbidden: Idunn and every request payload. BF1's caller-supplied `recipientId` is the counter-example and does not survive. |
|
||||
| **Bot token** | `BIFROST_DISCORD_BOT_TOKEN` in the same file (Y4), the one GameCult bot. | Operator-supplied, root `0600`, loaded by systemd `EnvironmentFile`. A process started before the file changed has the old value (Y6). | Bifrost holds it; the operator supplies it. Forbidden: Idunn, any notice content (`no-sensitive-egress`). |
|
||||
| **Odin store condition** (input, not built) | `odin.store_condition.v1`, key `topology` (O1). | Owned by Odin's campaign. Rewritten on change; `contested` and `unpersisted` exist only in Odin's memory and catalog, not on disk. | Odin. Idunn reads it through Odin's catalog with a `SnapshotRequest` (I11 precedent), never from `topology.cc`, and never writes it. An unreachable Odin yields no store incident; continuity already covers a dead Odin. |
|
||||
| **De-dup key** | Not a separate kind: it **is** the incident key. At Idunn: one open incident per `(condition, subject)`. At Bifrost: one journal entry per `(incident key, notice kind)`. At Discord: the journal key hashed into the message `nonce` with `enforce_nonce`, which covers a crash between the POST and the journal write. The Discord de-dup window is short and unverified here; outside it, a crash in that gap can still send twice. | Lives as long as the incident and its journal entry. | Idunn owns the incident half, Bifrost the delivery half. |
|
||||
| **Delivery-failure signal** | **Empty.** Nothing tells the operator that the watchdog's own delivery is failing. | | Bifrost's journal holds the state; nobody surfaces it. Out of this table's reach; left for the delivery-mechanism cut (see Rationale). |
|
||||
| **Operator acknowledgement** | **Not created.** No kind lets the operator mark an incident as seen. Nothing in the target asks for one. | | |
|
||||
|
||||
Empty cells, stated plainly:
|
||||
|
||||
- `operator-required` closure: the incident closes when the transaction ends or
|
||||
its step goes on (`close_spent_operator_incidents`). A stateful deployment
|
||||
left to the operator still has no Idunn-owned terminal action; the operator's
|
||||
`idunn expire` re-enters the same arm.
|
||||
- Delivery-failure visibility: no owner.
|
||||
- Discord `enforce_nonce` window: not verified against Discord's current docs
|
||||
in this pass.
|
||||
|
||||
## Rationale
|
||||
|
||||
**Idunn publishes and Bifrost reads, not the reverse.** Three crossings were
|
||||
weighed against the target:
|
||||
|
||||
1. *Idunn writes a command into Bifrost's store* (the `c063f27` shape). Idunn
|
||||
would write another organ's state; it would carry the recipient (BF1); an
|
||||
unconditional `put` re-arms a completed command (V1); and the deployed pump
|
||||
cannot send a DM anyway (Y7). Nothing stops root Idunn from doing it (Y2),
|
||||
so the boundary would be doctrine alone.
|
||||
2. *Idunn sends a CultNet command to a Bifrost listener.* Idunn becomes a
|
||||
delivery client with timeouts and retry state in its tick, which is where
|
||||
`survival-independent` breaks. The rebuild already deleted the synchronous
|
||||
version of this (I14).
|
||||
3. *Idunn writes its incident into its own store and its world-readable
|
||||
projection; a Bifrost reader derives its own delivery journal.* No write
|
||||
crosses an owner boundary. Idunn holds no recipient and no token. Bifrost
|
||||
down means a late message, not a slow tick. This is the shape of the
|
||||
existing topology projection (I13), which every managed target already
|
||||
reads. **Recommended.** It reverses `c063f27`'s direction, so it is a
|
||||
question for the operator, not a default.
|
||||
|
||||
**Which Bifrost process delivers is doctrine, not a fork.** A service earns a
|
||||
daemon only when isolation protects a named invariant. A notice can wait a
|
||||
minute, so a systemd timer running a one-shot Bifrost reader under a Bifrost
|
||||
user, with `EnvironmentFile=/srv/bifrost/env/persona-delivery.env`, is the
|
||||
smallest body. Folding it into `bifrost-persona-mouth` was rejected: that
|
||||
process runs the Epiphany-permit-gated Persona crossing, and a watchdog page
|
||||
that needs Starfire's permit would stop whenever the workstation is off.
|
||||
Idunn-invoked delivery was deleted once already (I14).
|
||||
|
||||
**DM is the default, not a question.** The operator's words are "I ought to
|
||||
get a message", and the recipient binding that exists is `DISCORD_OWNER_ID`.
|
||||
|
||||
**Retiring `publish-idunn-alarm` follows from the crossing.** If the crossing
|
||||
is (3), nothing calls the publisher. The unauthenticated command path (BF4)
|
||||
then carries no watchdog traffic. Closing it is Bifrost's own business, and
|
||||
VoidBot's caller is not live on Yggdrasil (V2). If the operator picks (1), that
|
||||
path must be authenticated before Idunn uses it.
|
||||
|
||||
**Exhaustion closes on health, not on the window.** `restarts_exhausted` turns
|
||||
false the moment one timestamp leaves the hour (I2). With a still-dead target
|
||||
that happens several times an hour. Closing on it would mean one incident, and
|
||||
one notice, per slide.
|
||||
|
||||
**Why `ReportOnce` is demoted rather than extended.** Persisting its map would
|
||||
make an in-memory de-dup cache the owner of "was the operator told", keyed by
|
||||
free-form strings, with no closure and no subject. The incident record puts
|
||||
that decision at the condition's own site, as `record-before-delivery` asks.
|
||||
|
||||
**The incident store is its own file, and that file is the projection.** The
|
||||
model page placed the record in `control.cc` with a separate projection
|
||||
derived from it. C1 overturns that: an unknown type in `control.cc` makes every
|
||||
older Idunn binary refuse the whole store, so rolling back the incident cut
|
||||
would stop continuity for every target. The record therefore lives in its own
|
||||
CultCache file, `/var/lib/gamecult/idunn-projection/incidents.cc`, written by
|
||||
Idunn with compare-exchange and published `0644` like `topology.cc` (C6). An
|
||||
incident carries nothing sensitive (`no-sensitive-egress`), so there is no
|
||||
private half to keep apart, and the derived copy disappears. What is lost: the
|
||||
open write is not atomic with any `control.cc` write. That costs nothing
|
||||
today, because the exhaustion decision writes nothing to `control.cc` (I2),
|
||||
and later sources are read-only observations. The model page's
|
||||
"Incident record" and "Incident projection" rows collapse into this one file.
|
||||
Self owns the edit to the model page.
|
||||
|
||||
**A broken incident store never reaches the tick.** Every incident operation
|
||||
returns its error to one call site, which traces it once with `report_once`
|
||||
and moves on. An unreadable store means no incident opens (the stderr trace
|
||||
still prints); it never changes what continuity decides. The Idunn cut's
|
||||
survival test pins this with an unwritable store path.
|
||||
|
||||
**Bifrost reads without the lock.** Idunn's writes once blocked on an
|
||||
exclusive lock (C2). The incident and topology cuts make every write in the
|
||||
projection directory a nonblocking exchange behind a private lock. A reader
|
||||
that took the lock is therefore unnecessary. It would also be refused, because
|
||||
it cannot open a `0600` lock. cultcache-ts `pullAll` takes no lock (C3, C4),
|
||||
and Idunn's rename is atomic (C32), so the reader only ever sees a whole
|
||||
snapshot. The reader never opens Idunn's file through a CultMesh node,
|
||||
which could flush.
|
||||
|
||||
**Delivery at most once, with the nonce as the second guard.** This follows
|
||||
the Persona precedent (C16): a journal entry found `running` at start becomes
|
||||
`unknown` and is never re-posted. A bridge exit that failed is retried on a
|
||||
later run, with the same nonce and `enforce_nonce` (C17). A retry inside
|
||||
Discord's "past few minutes" returns the original message instead of creating
|
||||
another. The remaining duplicate needs the bridge to have posted, then exited
|
||||
non-zero, and the retry to fall outside that window. The attempt cap and
|
||||
timer period bound it, and it is recorded as a residual risk rather than
|
||||
engineered away.
|
||||
|
||||
**Who sees a delivery failure: Bifrost, through systemd.** There is one
|
||||
channel to the operator, and it is the one that failed, so a failure cannot
|
||||
page over it. The reader exits non-zero while any journal entry is `unknown`
|
||||
or has exhausted its attempts. The unit then shows in `systemctl --failed` and
|
||||
in its journal, and the reader's `status` verb lists the entries. Idunn reads
|
||||
none of this (`survival-independent`). This is a default: the alternatives
|
||||
either couple Idunn to delivery state or need a second channel that does not
|
||||
exist.
|
||||
|
||||
**Deployment of the reader has no Idunn path.** Idunn deploys resident
|
||||
services, and a timer does not fit its recipe model (C9). No Bifrost binding
|
||||
exists in current Idunn (C10), and the old Bifrost manifest belongs to the
|
||||
Idunn that was rebuilt away (C11). The ruled one-shot reader can be
|
||||
hand-installed by a gamecult-ops script and runbook, the same interim footing
|
||||
as `huginn.service` and `bifrost-persona-mouth`. Otherwise Idunn must first
|
||||
learn a new workload kind. That choice is an operator question raised in the
|
||||
ops cut.
|
||||
|
||||
**The reader gets its own release root.** It installs under
|
||||
`/srv/bifrost/watchdog-notice/releases/<bifrost>-<cultlib>` and does not
|
||||
repoint `/srv/bifrost/persona-feedback/runtime/current`. Repointing that link
|
||||
would stage new code under `bifrost-persona-mouth`. With the token file now
|
||||
present (Y3, Y6), the mouth's next restart would also make the Persona crossing
|
||||
able to post. That crossing is out of scope and must not change as a side
|
||||
effect.
|
||||
|
||||
**Cut order.** The Idunn incident cut comes first. It writes the golden
|
||||
fixture that Bifrost's reader is tested against (C5). The Bifrost subtraction
|
||||
cut is independent and lands before the reader, so the reader is written
|
||||
against a tree with one Discord command vocabulary. The gamecult-ops
|
||||
subtraction is independent too. The ops deploy cut comes last and needs both
|
||||
behaviour cuts merged.
|
||||
|
||||
**The Bifrost suite follows the owner's name, with no fallback.** CultLib
|
||||
renamed `cultcache-ts` to `@gamecult/cultcache-ts` so that the org owns the
|
||||
name on a public registry (C22). The consumer changes the string at each live
|
||||
require site (C21) and keeps the directory anchor, which did not move. A
|
||||
try-both-names loader was rejected. It would be a local shim that keeps the
|
||||
old name alive in Bifrost after its owner retired it. It would also let a
|
||||
release silently pair new Bifrost with a pre-rename CultLib. After the change
|
||||
that pairing fails loudly at import (C24). Production does not move: its
|
||||
bundle pins CultLib `f67f5122` (C24). `cultnet-ts` and `cultmesh-ts` stay bare
|
||||
because that is what CultLib publishes. Whether they should follow the scope
|
||||
is CultLib's decision, not Bifrost's.
|
||||
|
||||
**The Rust fixture tests skip with a reason; they are not deleted.** Their
|
||||
producers were deleted from Epiphany on 2026-08-23 (C25), so installing cargo
|
||||
would not make them pass. Each one skips unless cargo and its named fixture
|
||||
source are present, and the skip reason names the Epiphany commit that removed
|
||||
the fixture. Deleting them would throw away the only Rust-authored byte seam
|
||||
test the Persona crossing has. Restoring a producer belongs to Epiphany and the
|
||||
Persona crossing, both outside this campaign (follow_up
|
||||
`bifrost-rust-fixture-producer`). The hard-coded Windows `CARGO_TARGET_DIR`
|
||||
goes, because it is wrong on every host except one workstation.
|
||||
|
||||
**`bifrost-suite` does not depend on `bifrost-retire-alarm`.** It leaves
|
||||
`operator-notification.mjs` alone, because that cut deletes the file, so the
|
||||
two branches merge in either order without conflict. The reader depends on
|
||||
both. It is written against a tree with one Discord command vocabulary, and it
|
||||
is verified by a suite that loads.
|
||||
|
||||
**The topology write stops waiting, and the lock stops being published.** The
|
||||
lock was published `0644` because a comment assumed that readers open it
|
||||
(`publish_projection_mode`, from `fbb9626`). None does (C30, C31), and
|
||||
consistency comes from the rename (C32). So the lock is Idunn's alone, and a
|
||||
contended publish is an error, not a wait. That is safe because every tick path
|
||||
treats a failed publish as not done: no publication digest, phase transition,
|
||||
cleanup evidence or continuity restart is persisted before the write returns
|
||||
`Ok` (C35). A contended publish is retried on a later pass or after backoff,
|
||||
and Idunn never believes it published what readers do not see. Boot
|
||||
reconciliation is the exception. It runs once, outside the tick, and it already
|
||||
logs failures and moves on. Only root processes can contend once the lock is
|
||||
private: the backup's copy, a few milliseconds a night. No retry machinery is
|
||||
added for it.
|
||||
|
||||
**One lock rule for the whole projection directory.** The topology cut moves
|
||||
the incident cut's `secure_lock` into one crate-level exchange primitive that
|
||||
both stores call, rather than a second copy. It also strengthens the rule:
|
||||
a lock wider than `0600`, or owned by someone else, is replaced by a rename,
|
||||
not chmodded. A chmod does not revoke a holder (C36), and the live
|
||||
`topology.cc.lock` has been `0644` since 2026-09-06 (C34), so tightening it in
|
||||
place would leave any existing holder able to block every topology publish
|
||||
forever. The replacement runs only while the lock is not yet private. Once it
|
||||
is, no other uid can open it, so a later replacement never splits exclusion
|
||||
between two inodes. A root backup holding the old inode during the first
|
||||
replacement copies one whole inode anyway (C32).
|
||||
245
docs/watchdog-reporting-postmortem.md
Normal file
245
docs/watchdog-reporting-postmortem.md
Normal file
|
|
@ -0,0 +1,245 @@
|
|||
# Watchdog reporting postmortem
|
||||
|
||||
Campaign `idunn-watchdog`, 2026-09-30 to 2026-10-01. This was the first Eureka
|
||||
campaign whose state was typed in Huginn's mind (instance `eureka`) rather than
|
||||
kept in a prose map. Document ids below are `idunn-watchdog:<kind>:<local>`.
|
||||
Read them with `view`; this page does not restate them.
|
||||
|
||||
## Summary
|
||||
|
||||
Idunn is now a watchdog that reaches the operator on Discord.
|
||||
|
||||
- **Idunn** opens a durable incident at the site where a continuity budget is
|
||||
exhausted (since `idunn-reincarnation` cut 1 this is a paging threshold
|
||||
only; continuity keeps restarting). The incident is written to `incidents.cc`, a world-readable file
|
||||
in its projection directory with a private 0600 lock, and closes when the
|
||||
target runs again and its restart window is no longer exhausted.
|
||||
- **Bifrost's watchdog-notice reader** runs from a one-minute timer. It sends
|
||||
one opening DM and at most one closure DM per incident, keeps its own
|
||||
journal, and retries a failed send forever with capped backoff.
|
||||
- **Scale:** 8 cuts across Idunn, Bifrost and gamecult-ops: 13 spec revisions,
|
||||
17 Hands reports, 17 Soul verdicts and 55 findings. Measured against the
|
||||
in-force estimates:
|
||||
|
||||
| Repo | Estimate (lines) | Landed (lines) |
|
||||
|---|---|---|
|
||||
| Idunn | +930/−101 | +2618/−354 |
|
||||
| Bifrost | +246/−335 | +1182/−429 |
|
||||
| gamecult-ops | +280/−35 | +505/−104 |
|
||||
|
||||
- **Installed on Yggdrasil:**
|
||||
- Idunn `a909f93` at 2026-09-30 23:21 UTC, then `cb60965` at 2026-10-01
|
||||
02:02 UTC.
|
||||
- The Bifrost reader at `160fac20` with CultLib `36ea08d3`, at 2026-10-01
|
||||
08:04 UTC.
|
||||
- **First live notice:** the incident's first real input was raven-muninn,
|
||||
which exhausted its budget minutes after the incident cut was installed. Its
|
||||
opening DM was delivered at 08:04:41 UTC, and the operator confirmed it.
|
||||
- **Still owed:** the closure DM for that same incident
|
||||
(`follow_up:closure-dm-observed`). It depends on raven-muninn recovering, and
|
||||
raven-muninn's own fault lies outside this campaign.
|
||||
|
||||
## Scope and invariants
|
||||
|
||||
Target `r4`. Its invariants are:
|
||||
|
||||
- `incident-owner`
|
||||
- `record-before-delivery`
|
||||
- `delivery-owner`
|
||||
- `survival-independent`
|
||||
- `one-per-incident`
|
||||
- `no-sensitive-egress`
|
||||
- `tick-lock-private`
|
||||
|
||||
The operator directions are `ruling:operator-watchdog-direction` and
|
||||
`ruling:operator-bifrost-bridge`. The second made Bifrost the bridge to
|
||||
Discord, so Idunn holds no credential and speaks no Discord API.
|
||||
|
||||
Excluded:
|
||||
|
||||
- **Odin's store condition and B5 `OperatorRequired`.** Neither can page yet.
|
||||
Odin's O2 cut is unbuilt, and B5 has no operator exit
|
||||
(`follow_up:b5-operator-required-no-exit`).
|
||||
- **VoidBot's `notify_owner`.**
|
||||
- **The Persona Discord crossing.**
|
||||
- **Eve lowering.**
|
||||
|
||||
The earlier attempt was Bifrost `c063f27` (2026-09-04): an alarm was a command
|
||||
written into Bifrost's store. It never ran, because no pump was deployed and
|
||||
the deployed pump lacked the DM verb. It was also reversed by ruling:
|
||||
`ruling:delivery-crossing` has Bifrost read Idunn's projection, so no write
|
||||
crosses an owner boundary.
|
||||
|
||||
## Timeline
|
||||
|
||||
| Cut | Reports | Soul passes | Notable finding |
|
||||
|---|---|---|---|
|
||||
| ops-retire-alarm | h1 | 1 | none |
|
||||
| bifrost-retire-alarm | h1–h2 | 2 | `silent-skip`: a stray `discord-dm` command sat pending forever |
|
||||
| bifrost-suite | h1 | 1 | `suite-cultcache-scope` (pre-existing): 9 of 17 tests failed at base on a CultLib rename |
|
||||
| bifrost-notice-reader | h1–h3 | 3 | `prune-keyed-on-presence`, `bridge-exit-is-failed`: duplicate DMs and blocked closures |
|
||||
| bifrost-notice-retry | h1–h2 | 2 | `future-stamp-stalls-retry` |
|
||||
| ops-notice-deploy | h1–h3 | 3 | `retry-window-five-minutes`: a 5-minute outage dropped a notice for good |
|
||||
| idunn-incident | h1–h3 | 3 | `lock-stalls-tick` (Blocker); `lock-acl-world-openable` (High) |
|
||||
| idunn-topology-lock | h1–h2 | 2 | `contended-publish-aborts`; the live stall it fixed was pre-existing |
|
||||
|
||||
## Structural delta
|
||||
|
||||
The landed code was about 2.5× its estimate, and the misses were not explained
|
||||
cut by cut as the skill asks.
|
||||
|
||||
- **idunn-incident:** +1813 against +700. Its fix batches added the lock
|
||||
primitive, load-independent lock tests, and per-operation fault keys.
|
||||
- **bifrost-notice-reader:** +891, against r3's estimate of +140. That r3
|
||||
figure described only the revision's own delta, not the cut, so r3 carried
|
||||
the wrong baseline.
|
||||
|
||||
Deleted:
|
||||
|
||||
- Bifrost's `publish-idunn-alarm` and the pump's `discord-dm` verb.
|
||||
- gamecult-ops' dead `--operator-alarm-command` scripts.
|
||||
- Idunn's `publish_projection_mode` and its 0644-lock test.
|
||||
- The reader's prune-on-absence logic and its `WATCHDOG_NOTICE_NOW_MS`
|
||||
production clock seam.
|
||||
|
||||
Added:
|
||||
|
||||
- `idunn.operator_incident.v1` (`incidents.cc`) and
|
||||
`bifrost.watchdog_notice_execution.v1` (the journal).
|
||||
- `bifrost-watchdog-notice.{service,timer}`.
|
||||
|
||||
Nothing was parked.
|
||||
|
||||
## What Soul caught
|
||||
|
||||
Green suites passed every defect below.
|
||||
|
||||
- **Survival and locking.**
|
||||
- Incident writes took a *blocking* flock inside Idunn's scheduler tick, on
|
||||
a world-readable lock. Any local uid could freeze continuity for every
|
||||
target, and `WatchdogUSec=0` meant nothing would restart Idunn
|
||||
(`s1.lock-stalls-tick`).
|
||||
- After the fix, the live directory's default ACL still made the lock 0644,
|
||||
so any uid could stop incidents opening. The test ran in a temp directory
|
||||
with no ACL (`s2.lock-acl-world-openable`).
|
||||
- The same probe exposed a pre-existing live hazard: `topology.cc.lock` 0644
|
||||
with a blocking write in the tick, measured at 5.07 s at base
|
||||
(`s2.topology-lock-stalls-tick`). The operator ruled it into the campaign;
|
||||
after the topology cut the tick takes 0.09 s.
|
||||
- **Split authority.**
|
||||
- The reader pruned its "already sent" journal whenever a key was absent
|
||||
from Idunn's store. Restoring a backup would have re-sent old DMs.
|
||||
- The reader decided subject validity with its own grammar, narrower than
|
||||
Idunn's.
|
||||
- Retention was decided by two predicates in Idunn.
|
||||
- **Lifecycle.**
|
||||
- One running observation closed an exhaustion incident. A release that
|
||||
crashes after each restart would have sent about 12 DMs an hour
|
||||
(`s1.flap-reopens`).
|
||||
- A bridge that died after Discord accepted a message was retried up to 5
|
||||
times, and the closure was then blocked forever.
|
||||
- Five attempts on a one-minute timer dropped a notice after a 5-minute
|
||||
Discord outage.
|
||||
- **Decorative checks.**
|
||||
- A compare-exchange expected `None` on a key that embeds `now`, so it
|
||||
guarded nothing (`s1.cas-decorative`).
|
||||
- A 2-second wall-clock bound on a lock test flaked under load and counted
|
||||
equivalent mutants as caught.
|
||||
- **The verifier itself.** Soul's s3 found that the stopgap forced one
|
||||
`CARGO_TARGET_DIR` on every job, so cargo-mutants' parallel workers tested
|
||||
each other's binaries. Two runs of the same two mutants disagreed
|
||||
(`s3.mutants-share-target-dir`). Every parallel mutation count reported
|
||||
through the stopgap before 2026-10-01 is unproven, in both sessions' work.
|
||||
|
||||
## Operator corrections
|
||||
|
||||
- **"Bifrost is indeed intended to be the bridge to external services like
|
||||
Discord."** The agents had found two delivery paths: VoidBot's
|
||||
`notify_owner` and Bifrost's command store. The ownership boundary was
|
||||
missing from both repos' summaries.
|
||||
- **"Stewardship is currently universal…"** Self had filed a gap treating
|
||||
follow-up repos as needing stewardship. It was withdrawn; the rule was not
|
||||
written in the skill.
|
||||
- **"…we seem to be saving some tokens… but we're very very slow."** Self
|
||||
blamed cold builds, measured, and was wrong. Cold builds took about 51 s and
|
||||
warm builds 29 s. The cost was slot queueing, repeated verify jobs, a test
|
||||
that hung a slot for 39 minutes, and a 4-hour job from the other session
|
||||
holding a fifth of capacity.
|
||||
- **"Yep, fixing Mind issues is highest priority, immediately once this
|
||||
campaign closes"** (`ruling:mind-fixes-next`).
|
||||
|
||||
## Incidents
|
||||
|
||||
- **Network outages (ENOTFOUND).** Two outages killed three agents
|
||||
mid-pass. Each was resumed by `SendMessage` with context intact, and nothing
|
||||
was lost, because each agent's state was in commits and the mind.
|
||||
- **A shared stopgap target directory** made mutation evidence noise. It is
|
||||
fixed in Eureka `a755a60`.
|
||||
- **Self's own brief inflated runtime.** It told Hands to mutate serially to
|
||||
dodge a flake, and was corrected to skipping the flaky test by name.
|
||||
- **A hung test held a verify slot for 39 minutes**
|
||||
(`s2.route-connect-test-hangs`). It is fixed.
|
||||
- **A Hands test helper overwrote the container's `/usr/bin/node`** through a
|
||||
symlink. It was caught by the agent's own baseline and then confined
|
||||
(`s2.test-clobbers-usr-bin-node`).
|
||||
- **A worktree directory stays locked** by a finished process
|
||||
(`F:\Projects\gamecult-ops-watchdog-deploy`, empty). Removing it is left to
|
||||
the operator.
|
||||
|
||||
## What worked
|
||||
|
||||
- **Soul attacks the layer where the rule fails, not the test.** Every
|
||||
High/Blocker finding came from a probe the shipped tests could not see: a
|
||||
separate-process lock holder, the live directory's ACL, or a restored store.
|
||||
- **Point Soul at the load-bearing invariant in the brief.** Naming "can the
|
||||
lock hang the tick" produced the Blocker on the first pass.
|
||||
- **Read before touching live state.** Self installed only after checking the
|
||||
installed revision, the in-flight transactions and the unit diff. The repo
|
||||
unit carried a BOM the live one did not.
|
||||
- **One verify job per pass.** Once briefs required it, a reader fix batch
|
||||
took 11 minutes, where the first reader Hands took 38.
|
||||
|
||||
## What to change in the pipeline
|
||||
|
||||
Already changed in the skill:
|
||||
|
||||
- The substrate gaps moved from a table to the mind.
|
||||
- Imagination owns the model page.
|
||||
- Soul pays for each rerun once.
|
||||
- The rust image uses sccache.
|
||||
- The stopgap no longer forces `CARGO_TARGET_DIR`.
|
||||
|
||||
Still to change:
|
||||
|
||||
- **Explain every ledger miss in the next report's deviations.** This campaign
|
||||
ran at about 2.5× its estimate and nobody explained it per cut.
|
||||
- **Make a revision's `estimate` the whole cut.** r3's estimate covered only
|
||||
its own delta.
|
||||
- **Verify the merged tree of a long-lived branch before pushing.** Self did
|
||||
this once, for the incident cut over B5, and it should be the rule.
|
||||
- **Briefs name the standard sibling build command.** The stopgap cannot
|
||||
provision siblings (`follow_up:gap-verify-no-sibling-repos`), and one
|
||||
hand-rolled build produced four false failures.
|
||||
|
||||
## Open follow-ups
|
||||
|
||||
Fifty follow-ups are in force. Twenty are substrate gaps (`gap-*`): the work
|
||||
the operator ruled next. Query them with the substrate-gaps recipe in the
|
||||
skill. Seven findings remain unresolved:
|
||||
|
||||
- **`cut-idunn-incident.s3.mutants-share-target-dir`.** It is fixed in the
|
||||
stopgap but awaits a Soul mutation run under the fixed script.
|
||||
- **Six findings whose resolution keys exceed 64 bytes**
|
||||
(`follow_up:gap-unresolvable-finding-key`). Their substance is recorded in
|
||||
`follow_up:notice-retry-hardening`, `follow_up:ops-refusal-test-hardening`,
|
||||
`follow_up:bifrost-pump-input-hygiene` and
|
||||
`follow_up:contended-publish-bounds`.
|
||||
|
||||
Operational follow-ups that matter most:
|
||||
|
||||
- `closure-dm-observed`
|
||||
- `notice-unknown-settle`
|
||||
- `contended-publish-bounds`
|
||||
- `idunn-load-flaky-tests`
|
||||
- `cultcache-rs-lock-mode`
|
||||
- `cultcache-ts-readonly-pull`
|
||||
95
docs/watchdog-reporting-target.md
Normal file
95
docs/watchdog-reporting-target.md
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
# Watchdog reporting: target
|
||||
|
||||
Status: target, Self, 2026-09-30. This is the first campaign whose state is
|
||||
typed in Eureka's mind (campaign `idunn-watchdog`). Questions, rulings, cut
|
||||
specs and their verification live there. This document keeps the rationale
|
||||
and the design truths the campaign produces. The cut map's body facts and
|
||||
model page will live in `watchdog-reporting-map.md`.
|
||||
|
||||
## Why
|
||||
|
||||
The operator ruled Odin's store-recovery Q-R4 on 2026-09-30 (Odin
|
||||
`docs/store-recovery-cut.md`, `cdc57bd`). A third set-aside inside ten minutes
|
||||
stops Odin writing while it keeps serving. The operator added:
|
||||
|
||||
> Yep, and Idunn needs better reporting capabilities so it can serve as
|
||||
> watchdog. I ought to get a message about this on Discord if it ever happens.
|
||||
|
||||
and, on where delivery lives:
|
||||
|
||||
> Bifrost is indeed intended to be the bridge to external services like Discord
|
||||
|
||||
Today nobody would hear about it. Idunn already detects several conditions an
|
||||
operator must act on, and for each one it writes a single `eprintln!` line to
|
||||
journald (`report_once`, `control_plane.rs:4599`):
|
||||
|
||||
- A continuity paging threshold is reached (6 charged attempts per hour;
|
||||
`restarts_page`, formerly `restarts_exhausted`). Continuity never stops:
|
||||
it keeps restarting on a capped backoff, and this threshold only pages. When
|
||||
this target was written the only trace was one log line.
|
||||
- An admitted route is degraded. It fails its stable-route challenge
|
||||
(`supervise_admitted_route`, ~5973). The degraded mark is durable, but no
|
||||
one is told.
|
||||
- A post-fencing phase deadline leaves a transaction to the operator: a
|
||||
stateful deployment whose candidate was issued a lease, or a continuity
|
||||
retrying forward past its deadline. Since `idunn-reincarnation` cut 2 it is
|
||||
the second incident condition, `operator-required` (subject: the transaction
|
||||
id), beside `continuity-exhausted`; it closes when the record ends or its
|
||||
step goes on.
|
||||
|
||||
Odin's own store condition (`odin.store_condition.v1`, `contested`) is mapped
|
||||
but not built. Idunn reads nothing about Odin's store health today.
|
||||
|
||||
The delivery path half exists in Bifrost:
|
||||
|
||||
- `tools/operator-notification.mjs publish-idunn-alarm` builds a
|
||||
`bifrost.bridge.discord_post_command.v1` `discord-dm` command.
|
||||
- `tools/cultmesh-bridge-commands.mjs process` turns such commands into DMs,
|
||||
writing a `discord_post_receipt.v1` for each.
|
||||
|
||||
But no pump runs anywhere, and the bot token file on Yggdrasil is absent
|
||||
(gamecult-ops `inventory.md` ~740). Idunn has never called the publisher. So
|
||||
the machine has a detector with no record, and a deliverer that nothing feeds.
|
||||
|
||||
## Design truths this campaign produces
|
||||
|
||||
- **Idunn decides what is an incident.** Idunn owns daemon survival, so the
|
||||
judgement "an operator must look at this" is its judgement, made where the
|
||||
condition is decided. An incident is typed state in Idunn's store before any
|
||||
delivery is attempted. A stderr line is a trace of the record, not the
|
||||
record itself.
|
||||
- **Bifrost delivers.** Discord transport, the bot token, the recipient
|
||||
binding and the delivery receipt belong to Bifrost. Idunn holds no Discord
|
||||
credential and speaks no Discord API.
|
||||
- **Survival never waits on delivery.** Bifrost is a managed target like any
|
||||
other. Its absence, failure or backlog may delay a message. It never delays
|
||||
or gates an Idunn decision, tick, restart, or deployment. An incident nobody
|
||||
has delivered stays visible in Idunn's own state and in `idunn status`.
|
||||
- **Two messages per incident at most, never one per tick.** A fault that
|
||||
lasts hours produces one opening notice, not a stream of them. When it
|
||||
clears, one closure notice follows, and only if the opening notice was sent.
|
||||
Repeated delivery attempts for either notice are idempotent at the
|
||||
recipient's end.
|
||||
- **No tick waits on a lock another uid can hold.** Root holders, such as
|
||||
the nightly backup's millisecond shared lock, are trusted and bounded. The
|
||||
operator ruled this scope on 2026-09-30, choosing uid-scoped. Soul found that any
|
||||
local uid could freeze Idunn's scheduler tick by holding a published
|
||||
`.lock` file. This was true for incidents during the campaign, and for
|
||||
`topology.cc` before it. Idunn's writes in its projection directory are
|
||||
non-blocking exchanges. The locks behind them are private to Idunn,
|
||||
whatever the directory's default ACL says. The operator ruled on
|
||||
2026-09-30 that the topology fix belongs to this campaign.
|
||||
- **Nothing sensitive leaves.** A notice names the target, the condition and
|
||||
the time. It never carries a binding value, a path to a secret, a token, or
|
||||
a line of configuration.
|
||||
|
||||
## Not this campaign
|
||||
|
||||
- Odin's store-recovery cuts, O2 included. They produce `odin.store_condition`
|
||||
and are Odin's campaign, still in prose. This campaign consumes the record
|
||||
once it exists.
|
||||
- Idunn's phase deadlines. They landed in `idunn-reincarnation` cut 2, which
|
||||
also owns the `operator-required` incident condition.
|
||||
- VoidBot's `notify_owner` MCP tool and its separate DM code.
|
||||
- Bifrost's Persona delivery crossing.
|
||||
- Eve/TUI lowering of incidents.
|
||||
10
scripts/verify.sh
Executable file
10
scripts/verify.sh
Executable file
|
|
@ -0,0 +1,10 @@
|
|||
#!/usr/bin/env bash
|
||||
# The one verification command. Run it on Linux with the Windows GNU std installed
|
||||
# (the Eureka verify image has both); a Windows workstation proves nothing.
|
||||
# The Windows check runs first: it is fast, and a hung test cannot mask it.
|
||||
# --lib because the tests under tests/ assume the host layout.
|
||||
# The Windows --lib --tests check keeps unix-only test code from leaking out of its cfg.
|
||||
set -euo pipefail
|
||||
cargo check --locked --target x86_64-pc-windows-gnu --bin idunn-host
|
||||
cargo check --locked --target x86_64-pc-windows-gnu --lib --tests
|
||||
cargo test --locked --lib
|
||||
96
src/bin/idunn-launch.rs
Normal file
96
src/bin/idunn-launch.rs
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
//! The unit's process for an upstream daemon: parse the recipe's argv, read
|
||||
//! what Idunn set, run the launcher, and map the outcome to an exit code.
|
||||
#[cfg(unix)]
|
||||
fn main() -> std::process::ExitCode {
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
use anyhow::Context;
|
||||
use idunn_daemon::deployment::{
|
||||
IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, IDUNN_RUNTIME_BUNDLE_ENVIRONMENT,
|
||||
IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT,
|
||||
};
|
||||
use idunn_daemon::launcher::{self, *};
|
||||
|
||||
static STOP: AtomicBool = AtomicBool::new(false);
|
||||
extern "C" fn on_signal(_: libc::c_int) {
|
||||
STOP.store(true, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
fn launch() -> anyhow::Result<Outcome> {
|
||||
let release_root = std::env::current_exe()?
|
||||
.parent()
|
||||
.context("the launcher is not inside a release")?
|
||||
.to_owned();
|
||||
let contract = LaunchContract::parse(release_root, std::env::args().skip(1))?;
|
||||
let bundle = std::env::var_os(IDUNN_RUNTIME_BUNDLE_ENVIRONMENT)
|
||||
.context("the Idunn runtime bundle is not set")?;
|
||||
let bind = std::env::var(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT)
|
||||
.context("the Idunn candidate bind is not set")?;
|
||||
let member = launcher::parse_member_bind(
|
||||
&std::env::var(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT)
|
||||
.context("the Idunn member bind is not set")?,
|
||||
&bind,
|
||||
)?;
|
||||
let (activation, provider) = launcher::inherited_authority_files()?;
|
||||
let authority = RuntimeAuthority::open(bundle.as_ref(), activation, provider)?;
|
||||
authority.require_candidate_bind(&bind)?;
|
||||
let lease_path = std::env::var_os(IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT);
|
||||
anyhow::ensure!(
|
||||
lease_path.is_some() == authority.expected().write_lease_required,
|
||||
"the process write lease binding differs from Expected"
|
||||
);
|
||||
let route = std::net::TcpListener::bind(&bind).context("binding the candidate endpoint")?;
|
||||
let sink = OdinRudpSink::from_expected(authority.expected())?;
|
||||
let lease = DriverLeaseReader::new(
|
||||
&authority.expected().target,
|
||||
lease_path.as_deref().unwrap_or_default().as_ref(),
|
||||
);
|
||||
let health_path = contract.health_path.clone();
|
||||
let ports = Ports {
|
||||
credentials: std::sync::Arc::new(CredentialsDirectory::from_environment()),
|
||||
clock: std::sync::Arc::new(SystemClock),
|
||||
sink: std::sync::Arc::new(sink),
|
||||
lease: std::sync::Arc::new(lease),
|
||||
spawner: Box::new(ProcessSpawner),
|
||||
health: Box::new(move |member| {
|
||||
std::sync::Arc::new(HttpHealthProbe {
|
||||
member,
|
||||
path: health_path.clone(),
|
||||
})
|
||||
}),
|
||||
};
|
||||
// SAFETY: the handler only stores to an atomic.
|
||||
unsafe {
|
||||
libc::signal(libc::SIGTERM, on_signal as *const () as libc::sighandler_t);
|
||||
libc::signal(libc::SIGINT, on_signal as *const () as libc::sighandler_t);
|
||||
}
|
||||
Ok(launcher::run(
|
||||
&contract,
|
||||
authority,
|
||||
route,
|
||||
member,
|
||||
&std::env::temp_dir(),
|
||||
ports,
|
||||
&STOP,
|
||||
))
|
||||
}
|
||||
|
||||
match launch() {
|
||||
Ok(outcome) => {
|
||||
if let Outcome::Failed(code) = &outcome {
|
||||
eprintln!("idunn-launch ended: {code}");
|
||||
}
|
||||
std::process::ExitCode::from(outcome.exit_code() as u8)
|
||||
}
|
||||
Err(error) => {
|
||||
eprintln!("idunn-launch refused: {error}");
|
||||
std::process::ExitCode::from(2)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn main() {
|
||||
eprintln!("idunn-launch runs only on Unix");
|
||||
std::process::exit(2);
|
||||
}
|
||||
701
src/brake.rs
Normal file
701
src/brake.rs
Normal file
|
|
@ -0,0 +1,701 @@
|
|||
//! The deployment stop: the one reader and the one writer of the record that
|
||||
//! halts deployments of a runtime. Engaged halts; absent or clear admits. Who
|
||||
//! engaged it and why are on the record, and nothing is signed, issued, bound
|
||||
//! to a release or consumed: the brake is a stop, not a per-deploy gate (ruling
|
||||
//! brake-is-a-stop). The lifecycle brake is a different record and a different
|
||||
//! store; nothing here touches it.
|
||||
//!
|
||||
//! The reader fails closed. A store that is present but foreign, ambiguous,
|
||||
//! noncanonical, for another runtime, a symlink, or owned by anyone but the
|
||||
//! expected uid reads as `Unreadable`, which does not admit.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use crate::deployment::BrakeBinding;
|
||||
|
||||
use anyhow::{Context, Result, anyhow, bail, ensure};
|
||||
use cultcache_rs::{CultCacheEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub(crate) const DEPLOYMENT_STOP_SCHEMA: &str = "idunn.deployment_stop.v1";
|
||||
pub(crate) const DEPLOYMENT_STOP_KEY: &str = "deployment-stop";
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub(crate) enum StopState {
|
||||
Engaged,
|
||||
Clear,
|
||||
}
|
||||
|
||||
impl StopState {
|
||||
pub(crate) fn name(self) -> &'static str {
|
||||
match self {
|
||||
Self::Engaged => "engaged",
|
||||
Self::Clear => "clear",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||
#[cultcache(type = "idunn.deployment_stop", schema = "idunn.deployment_stop.v1")]
|
||||
pub(crate) struct DeploymentStop {
|
||||
#[cultcache(key = 0)]
|
||||
pub(crate) schema_version: String,
|
||||
#[cultcache(key = 1)]
|
||||
pub(crate) runtime_id: String,
|
||||
#[cultcache(key = 2)]
|
||||
pub(crate) state: StopState,
|
||||
/// The login name of the user who ran the verb, read from the process uid.
|
||||
#[cultcache(key = 3)]
|
||||
pub(crate) by: String,
|
||||
#[cultcache(key = 4)]
|
||||
pub(crate) reason: String,
|
||||
/// The verb's clock.
|
||||
#[cultcache(key = 5)]
|
||||
pub(crate) at_unix_millis: u64,
|
||||
/// The previous record's sequence plus one; 1 for the first.
|
||||
#[cultcache(key = 6)]
|
||||
pub(crate) sequence: u64,
|
||||
}
|
||||
|
||||
fn bounded(value: &str, label: &str, max: usize) -> Result<()> {
|
||||
ensure!(
|
||||
!value.trim().is_empty() && value.len() <= max && !value.chars().any(char::is_control),
|
||||
"deployment stop {label} is empty, oversized, or contains control characters"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl DeploymentStop {
|
||||
pub(crate) fn validate(&self) -> Result<()> {
|
||||
ensure!(
|
||||
self.schema_version == DEPLOYMENT_STOP_SCHEMA,
|
||||
"deployment stop schema is unsupported"
|
||||
);
|
||||
bounded(&self.runtime_id, "runtime id", 256)?;
|
||||
bounded(&self.by, "author", 256)?;
|
||||
bounded(&self.reason, "reason", 1024)?;
|
||||
ensure!(
|
||||
self.at_unix_millis > 0 && self.sequence > 0,
|
||||
"deployment stop time or sequence is zero"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn envelope(&self) -> Result<CultCacheEnvelope> {
|
||||
self.validate()?;
|
||||
let stored_at = chrono::DateTime::from_timestamp_millis(i64::try_from(self.at_unix_millis)?)
|
||||
.ok_or_else(|| anyhow!("deployment stop time is out of range"))?
|
||||
.to_rfc3339();
|
||||
Ok(CultCacheEnvelope {
|
||||
key: DEPLOYMENT_STOP_KEY.into(),
|
||||
r#type: Self::TYPE.into(),
|
||||
payload: rmp_serde::to_vec(self)?,
|
||||
stored_at,
|
||||
schema_id: Some(DEPLOYMENT_STOP_SCHEMA.into()),
|
||||
})
|
||||
}
|
||||
|
||||
fn decode(envelope: &CultCacheEnvelope) -> Result<Self> {
|
||||
ensure!(
|
||||
envelope.r#type == Self::TYPE
|
||||
&& envelope.key == DEPLOYMENT_STOP_KEY
|
||||
&& envelope.schema_id.as_deref() == Some(DEPLOYMENT_STOP_SCHEMA),
|
||||
"deployment stop store holds a foreign record"
|
||||
);
|
||||
let record: Self = rmp_serde::from_slice(&envelope.payload)?;
|
||||
ensure!(
|
||||
rmp_serde::to_vec(&record)? == envelope.payload,
|
||||
"deployment stop is noncanonical"
|
||||
);
|
||||
record.validate()?;
|
||||
Ok(record)
|
||||
}
|
||||
}
|
||||
|
||||
/// What one store says about a runtime's deployments.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub(crate) enum Stop {
|
||||
/// No store, or an empty one: nothing halts.
|
||||
Absent,
|
||||
Clear(DeploymentStop),
|
||||
Engaged(DeploymentStop),
|
||||
/// Present but not a record this runtime can trust. Does not admit.
|
||||
Unreadable(String),
|
||||
}
|
||||
|
||||
impl Stop {
|
||||
pub(crate) fn admits(&self) -> bool {
|
||||
matches!(self, Self::Absent | Self::Clear(_))
|
||||
}
|
||||
}
|
||||
|
||||
/// One stop a binding names, with how to name it in a wait note.
|
||||
pub(crate) struct NamedStop {
|
||||
pub(crate) label: String,
|
||||
/// The login the file system proves wrote the store: the steward user, whose
|
||||
/// uid owns the file. None for the operator's store, whose record carries
|
||||
/// only the verb's own claim of `by`.
|
||||
pub(crate) proven_writer: Option<String>,
|
||||
pub(crate) stop: Stop,
|
||||
}
|
||||
|
||||
impl NamedStop {
|
||||
/// The wait-note line for a stop that does not admit; None when it admits.
|
||||
/// A steward stop is named by the proven owner, never by the record's `by`.
|
||||
pub(crate) fn note(&self) -> Option<String> {
|
||||
let label = &self.label;
|
||||
match &self.stop {
|
||||
Stop::Absent | Stop::Clear(_) => None,
|
||||
Stop::Engaged(held) => Some(match &self.proven_writer {
|
||||
Some(_) => format!("{label} engaged: {}", held.reason),
|
||||
None => format!("{label} engaged by {}: {}", held.by, held.reason),
|
||||
}),
|
||||
Stop::Unreadable(why) => Some(format!("{label} is unreadable and fails closed: {why}")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Every stop a binding names, each with the words that identify its store in
|
||||
/// a wait note: the operator's, then the steward's when the binding names one.
|
||||
/// Deployments admit iff every one admits. The operator's store is read as
|
||||
/// owned by this process's uid; the steward's as owned by the steward user's
|
||||
/// uid, so the file system proves who wrote it. A steward that does not
|
||||
/// resolve to a non-root uid is an unreadable stop, never an absent one.
|
||||
pub(crate) fn read_stops(brakes: &BrakeBinding, runtime_id: &str) -> Vec<NamedStop> {
|
||||
let mut stops = vec![NamedStop {
|
||||
label: "deployment brake".to_string(),
|
||||
proven_writer: None,
|
||||
stop: read_stop(&brakes.deployment_store, runtime_id, process_uid()),
|
||||
}];
|
||||
if let Some(steward) = &brakes.steward {
|
||||
let label = format!("steward {} deployment brake", steward.user);
|
||||
let stop = match crate::steward_orders::user_uid(&steward.user) {
|
||||
Ok(uid) => read_stop_with_remedy(&steward.store, runtime_id, uid, |path| {
|
||||
// Only the owner uid can mend it: a clear run as root leaves a
|
||||
// root-owned file, which this reader refuses again.
|
||||
format!(
|
||||
"only {} (the store's owner uid) can repair it, by running as that user: \
|
||||
idunn-provision deployment-brake-clear --store {} --runtime-id {runtime_id} \
|
||||
--reason <text>; a clear run as root stays unreadable",
|
||||
steward.user,
|
||||
path.display()
|
||||
)
|
||||
}),
|
||||
Err(why) => Stop::Unreadable(format!(
|
||||
"{why:#}; the binding's steward user {} must be a non-root login",
|
||||
steward.user
|
||||
)),
|
||||
};
|
||||
stops.push(NamedStop {
|
||||
label,
|
||||
proven_writer: Some(steward.user.clone()),
|
||||
stop,
|
||||
});
|
||||
}
|
||||
stops
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn process_uid() -> u32 {
|
||||
// SAFETY: geteuid has no failure mode and no arguments.
|
||||
unsafe { libc::geteuid() }
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
pub(crate) fn process_uid() -> u32 {
|
||||
0
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn login_name(uid: u32) -> Result<String> {
|
||||
// SAFETY: getpwuid returns null or a pointer into libc's static buffer,
|
||||
// copied before any other call; the verbs are single threaded.
|
||||
unsafe {
|
||||
let entry = libc::getpwuid(uid);
|
||||
ensure!(!entry.is_null(), "uid {uid} has no login name");
|
||||
Ok(std::ffi::CStr::from_ptr((*entry).pw_name)
|
||||
.to_string_lossy()
|
||||
.into_owned())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
pub(crate) fn login_name(_: u32) -> Result<String> {
|
||||
std::env::var("USERNAME").context("USERNAME is unset")
|
||||
}
|
||||
|
||||
/// Open without following a final symlink and judge the file by fstat before
|
||||
/// anything is read from it.
|
||||
#[cfg(unix)]
|
||||
fn check_file(path: &Path, expected_owner_uid: u32) -> Result<()> {
|
||||
use std::os::unix::fs::{MetadataExt, OpenOptionsExt};
|
||||
let file = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.custom_flags(libc::O_NOFOLLOW)
|
||||
.open(path)
|
||||
.with_context(|| format!("opening deployment stop store {}", path.display()))?;
|
||||
let metadata = file.metadata()?;
|
||||
ensure!(
|
||||
metadata.is_file(),
|
||||
"deployment stop store is not a regular file"
|
||||
);
|
||||
ensure!(
|
||||
metadata.uid() == expected_owner_uid,
|
||||
"deployment stop store is owned by uid {}, not {expected_owner_uid}",
|
||||
metadata.uid()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn check_file(path: &Path, _: u32) -> Result<()> {
|
||||
ensure!(
|
||||
std::fs::metadata(path)?.is_file(),
|
||||
"deployment stop store is not a regular file"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Absent means exactly this: the final component does not exist and its
|
||||
/// parent is an existing directory. Every other failure to look (permission,
|
||||
/// a parent that is a file, a symlink loop) is a store this runtime cannot
|
||||
/// judge, and fails closed.
|
||||
fn is_absent(path: &Path) -> Result<bool> {
|
||||
match std::fs::symlink_metadata(path) {
|
||||
Ok(_) => Ok(false),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
||||
let parent = path.parent().filter(|p| !p.as_os_str().is_empty());
|
||||
let parent = parent.unwrap_or_else(|| Path::new("."));
|
||||
ensure!(
|
||||
std::fs::metadata(parent).is_ok_and(|m| m.is_dir()),
|
||||
"deployment stop store directory {} does not exist",
|
||||
parent.display()
|
||||
);
|
||||
Ok(true)
|
||||
}
|
||||
Err(error) => Err(error)
|
||||
.with_context(|| format!("looking for deployment stop store {}", path.display())),
|
||||
}
|
||||
}
|
||||
|
||||
fn read_checked(path: &Path, runtime_id: &str, owner: u32) -> Result<Option<DeploymentStop>> {
|
||||
if is_absent(path)? {
|
||||
return Ok(None);
|
||||
}
|
||||
check_file(path, owner)?;
|
||||
ensure!(
|
||||
std::fs::metadata(path)?.len() > 0,
|
||||
"deployment stop store is zero length"
|
||||
);
|
||||
let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?;
|
||||
match entries.as_slice() {
|
||||
[] => bail!("deployment stop store holds no record"),
|
||||
[entry] => {
|
||||
let record = DeploymentStop::decode(entry)?;
|
||||
ensure!(
|
||||
record.runtime_id == runtime_id,
|
||||
"deployment stop belongs to runtime {}",
|
||||
record.runtime_id
|
||||
);
|
||||
Ok(Some(record))
|
||||
}
|
||||
_ => bail!("deployment stop store is ambiguous"),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn read_stop(path: &Path, runtime_id: &str, expected_owner_uid: u32) -> Stop {
|
||||
read_stop_with_remedy(path, runtime_id, expected_owner_uid, |path| {
|
||||
format!(
|
||||
"to replace it run: idunn-provision deployment-brake-clear \
|
||||
--store {} --runtime-id {runtime_id} --reason <text> (or deployment-brake-engage)",
|
||||
path.display()
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn read_stop_with_remedy(
|
||||
path: &Path,
|
||||
runtime_id: &str,
|
||||
expected_owner_uid: u32,
|
||||
remedy: impl FnOnce(&Path) -> String,
|
||||
) -> Stop {
|
||||
match read_checked(path, runtime_id, expected_owner_uid) {
|
||||
Ok(None) => Stop::Absent,
|
||||
Ok(Some(record)) if record.state == StopState::Clear => Stop::Clear(record),
|
||||
Ok(Some(record)) => Stop::Engaged(record),
|
||||
Err(error) => Stop::Unreadable(format!("{error:#}; {}", remedy(path))),
|
||||
}
|
||||
}
|
||||
|
||||
/// Write the next record under a compare-exchange of the whole store, so a
|
||||
/// concurrent verb or a store that moved since it was read is refused. The
|
||||
/// author is the login name of the process uid and the sequence is the
|
||||
/// predecessor's plus one (1 when there is none or it cannot be decoded);
|
||||
/// neither is an argument.
|
||||
///
|
||||
/// This is also the operator's break-glass. A predecessor that is a retired v1
|
||||
/// brake, another runtime's record, ambiguous, or zero length is replaced. A
|
||||
/// file that is not a CultCache store at all (junk bytes) is replaced only
|
||||
/// when it is a regular file owned by the caller, by removing it first. A
|
||||
/// symlink or another user's file is refused. The sequence is not validated
|
||||
/// beyond being positive: nothing reads it for authority.
|
||||
pub(crate) fn replace_stop(
|
||||
path: &Path,
|
||||
runtime_id: &str,
|
||||
state: StopState,
|
||||
reason: &str,
|
||||
now_unix_millis: u64,
|
||||
) -> Result<DeploymentStop> {
|
||||
let store = SingleFileMessagePackBackingStore::new(path);
|
||||
let before = match store.pull_all_read_only_snapshot() {
|
||||
Ok(entries) => entries,
|
||||
Err(undecodable) => {
|
||||
check_file(path, process_uid()).with_context(|| {
|
||||
format!("deployment stop store is undecodable ({undecodable:#}) and not the caller's to replace")
|
||||
})?;
|
||||
std::fs::remove_file(path)?;
|
||||
Vec::new()
|
||||
}
|
||||
};
|
||||
let sequence = match before.as_slice() {
|
||||
[entry] => match DeploymentStop::decode(entry) {
|
||||
Ok(previous) => previous.sequence.saturating_add(1),
|
||||
Err(_) => 1,
|
||||
},
|
||||
_ => 1,
|
||||
};
|
||||
let record = DeploymentStop {
|
||||
schema_version: DEPLOYMENT_STOP_SCHEMA.into(),
|
||||
runtime_id: runtime_id.into(),
|
||||
state,
|
||||
by: login_name(process_uid())?,
|
||||
reason: reason.into(),
|
||||
at_unix_millis: now_unix_millis,
|
||||
sequence,
|
||||
};
|
||||
let next = record.envelope()?;
|
||||
ensure!(
|
||||
store.compare_exchange_snapshot(&before, &[next])?,
|
||||
"deployment stop changed during transition"
|
||||
);
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn engage(path: &Path, reason: &str, at: u64) -> Result<DeploymentStop> {
|
||||
replace_stop(path, "runtime", StopState::Engaged, reason, at)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_and_empty_stores_admit_a_clear_one_admits_an_engaged_one_does_not() -> Result<()> {
|
||||
let dir = tempfile::tempdir()?;
|
||||
let path = dir.path().join("stop.cc");
|
||||
let me = process_uid();
|
||||
assert_eq!(read_stop(&path, "runtime", me), Stop::Absent);
|
||||
engage(&path, "freeze for the audit", 100)?;
|
||||
let Stop::Engaged(held) = read_stop(&path, "runtime", me) else {
|
||||
bail!("an engaged stop did not read as engaged")
|
||||
};
|
||||
assert_eq!(held.reason, "freeze for the audit");
|
||||
assert_eq!(held.by, login_name(me)?);
|
||||
assert!(!Stop::Engaged(held).admits());
|
||||
replace_stop(&path, "runtime", StopState::Clear, "audit done", 200)?;
|
||||
let cleared = read_stop(&path, "runtime", me);
|
||||
assert!(matches!(&cleared, Stop::Clear(record) if record.sequence == 2));
|
||||
assert!(cleared.admits());
|
||||
// Clear is not consumed: the same record admits again.
|
||||
assert!(read_stop(&path, "runtime", me).admits());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corrupt_foreign_other_runtime_symlinked_or_foreign_owned_stores_stop() -> Result<()> {
|
||||
let dir = tempfile::tempdir()?;
|
||||
let me = process_uid();
|
||||
let path = dir.path().join("stop.cc");
|
||||
engage(&path, "held", 100)?;
|
||||
// Another runtime's record.
|
||||
assert!(!read_stop(&path, "other", me).admits());
|
||||
// Owned by someone other than the expected uid: even a CLEAR record
|
||||
// from a foreign owner must not admit.
|
||||
let cleared = dir.path().join("cleared.cc");
|
||||
replace_stop(&cleared, "runtime", StopState::Clear, "thaw", 100)?;
|
||||
assert!(read_stop(&cleared, "runtime", me).admits());
|
||||
assert!(matches!(read_stop(&cleared, "runtime", me + 1), Stop::Unreadable(_)));
|
||||
// A symlink to a valid, clear store.
|
||||
let real = dir.path().join("real.cc");
|
||||
replace_stop(&real, "runtime", StopState::Clear, "clear", 100)?;
|
||||
let link = dir.path().join("link.cc");
|
||||
#[cfg(unix)]
|
||||
std::os::unix::fs::symlink(&real, &link)?;
|
||||
#[cfg(unix)]
|
||||
assert!(matches!(read_stop(&link, "runtime", me), Stop::Unreadable(_)));
|
||||
// Bytes that are no store at all.
|
||||
let junk = dir.path().join("junk.cc");
|
||||
std::fs::write(&junk, b"not msgpack at all")?;
|
||||
assert!(matches!(read_stop(&junk, "runtime", me), Stop::Unreadable(_)));
|
||||
// A record of another type under another key (the retired v1 brake).
|
||||
let foreign = dir.path().join("foreign.cc");
|
||||
let store = SingleFileMessagePackBackingStore::new(&foreign);
|
||||
let envelope = CultCacheEnvelope {
|
||||
key: "deployment".into(),
|
||||
r#type: "idunn.deployment_brake".into(),
|
||||
payload: vec![0x90],
|
||||
stored_at: "1970-01-01T00:00:00+00:00".into(),
|
||||
schema_id: Some("idunn.deployment_brake.v1".into()),
|
||||
};
|
||||
assert!(store.compare_exchange_snapshot(&[], &[envelope])?);
|
||||
assert!(matches!(read_stop(&foreign, "runtime", me), Stop::Unreadable(_)));
|
||||
// A noncanonical payload under the right identity.
|
||||
let mut stop = DeploymentStop::decode(&store_entry(&path)?)?;
|
||||
stop.sequence = 9;
|
||||
let mut loose = stop.envelope()?;
|
||||
loose.payload.push(0xc0);
|
||||
let noncanonical = dir.path().join("noncanonical.cc");
|
||||
assert!(SingleFileMessagePackBackingStore::new(&noncanonical)
|
||||
.compare_exchange_snapshot(&[], &[loose])?);
|
||||
assert!(matches!(read_stop(&noncanonical, "runtime", me), Stop::Unreadable(_)));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A binding whose steward is the user running the test, so the steward's
|
||||
/// store can really be owned by the steward. None when run as root, which
|
||||
/// no steward may be.
|
||||
fn binding_with_steward(dir: &Path, user: &str) -> BrakeBinding {
|
||||
BrakeBinding {
|
||||
deployment_store: dir.join("operator.cc"),
|
||||
lifecycle_store: dir.join("lifecycle.cc"),
|
||||
steward: Some(crate::deployment::StewardBrakeBinding {
|
||||
user: user.into(),
|
||||
store: dir.join("steward.cc"),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn admits_all(stops: &[NamedStop]) -> bool {
|
||||
stops.iter().all(|named| named.stop.admits())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_steward_store_is_a_second_stop_and_the_operator_store_is_one_it_cannot_clear() -> Result<()> {
|
||||
let me = process_uid();
|
||||
if me == 0 {
|
||||
eprintln!("SKIPPED: a steward is never root, and this run is root");
|
||||
return Ok(());
|
||||
}
|
||||
let dir = tempfile::tempdir()?;
|
||||
let binding = binding_with_steward(dir.path(), &login_name(me)?);
|
||||
let (operator, steward) = (&binding.deployment_store, &binding.steward.as_ref().unwrap().store);
|
||||
// Both absent: admits.
|
||||
assert!(admits_all(&read_stops(&binding, "runtime")));
|
||||
// Steward engaged alone: halts, and the stop names the steward's store.
|
||||
replace_stop(steward, "runtime", StopState::Engaged, "steward freeze", 10)?;
|
||||
let stops = read_stops(&binding, "runtime");
|
||||
assert!(!admits_all(&stops));
|
||||
assert!(matches!(&stops[1].stop, Stop::Engaged(held) if held.reason == "steward freeze"));
|
||||
// The note names the proven owner, and never the record's claimed `by`.
|
||||
assert_eq!(
|
||||
stops[1].note().unwrap(),
|
||||
format!("steward {} deployment brake engaged: steward freeze", login_name(me)?)
|
||||
);
|
||||
// Both engaged, then the steward clears its own: the operator's holds.
|
||||
replace_stop(operator, "runtime", StopState::Engaged, "operator hold", 11)?;
|
||||
replace_stop(steward, "runtime", StopState::Clear, "steward thaw", 12)?;
|
||||
let stops = read_stops(&binding, "runtime");
|
||||
assert!(matches!(&stops[0].stop, Stop::Engaged(_)) && matches!(&stops[1].stop, Stop::Clear(_)));
|
||||
assert!(!admits_all(&stops));
|
||||
// The other order: operator clear, steward engaged still halts.
|
||||
replace_stop(operator, "runtime", StopState::Clear, "operator thaw", 13)?;
|
||||
replace_stop(steward, "runtime", StopState::Engaged, "steward again", 14)?;
|
||||
assert!(!admits_all(&read_stops(&binding, "runtime")));
|
||||
// Both clear: admits.
|
||||
replace_stop(steward, "runtime", StopState::Clear, "steward thaw", 15)?;
|
||||
assert!(admits_all(&read_stops(&binding, "runtime")));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn steward_store_owned_by_another_uid_stops() -> Result<()> {
|
||||
let me = process_uid();
|
||||
let dir = tempfile::tempdir()?;
|
||||
// Steward is "nobody" (a real non-root user that is not the writer);
|
||||
// the file is written by this process, so even a Clear record fails.
|
||||
if me == 65534 || user_missing("nobody") {
|
||||
eprintln!("SKIPPED: needs a user `nobody` other than the caller");
|
||||
return Ok(());
|
||||
}
|
||||
let binding = binding_with_steward(dir.path(), "nobody");
|
||||
let steward = &binding.steward.as_ref().unwrap().store;
|
||||
replace_stop(steward, "runtime", StopState::Clear, "forged clear", 10)?;
|
||||
let stops = read_stops(&binding, "runtime");
|
||||
assert!(matches!(&stops[1].stop, Stop::Unreadable(why) if why.contains("is owned by uid")
|
||||
&& why.contains("only nobody (the store's owner uid) can repair it")
|
||||
&& why.contains("stays unreadable")
|
||||
&& why.contains("idunn-provision deployment-brake-clear")));
|
||||
// The operator's stop keeps the operator's remedy.
|
||||
replace_stop(&binding.deployment_store, "other-runtime", StopState::Clear, "x", 11)?;
|
||||
assert!(matches!(&read_stops(&binding, "runtime")[0].stop, Stop::Unreadable(why)
|
||||
if why.contains("to replace it run:") && !why.contains("only ")));
|
||||
assert!(!admits_all(&stops));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn user_missing(name: &str) -> bool {
|
||||
crate::steward_orders::user_uid(name).is_err()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unresolvable_or_root_steward_user_is_an_unreadable_stop_not_an_absent_one() -> Result<()> {
|
||||
let dir = tempfile::tempdir()?;
|
||||
for user in ["no-such-login-zz", "root"] {
|
||||
let binding = binding_with_steward(dir.path(), user);
|
||||
let stops = read_stops(&binding, "runtime");
|
||||
assert!(matches!(&stops[1].stop, Stop::Unreadable(_)), "{user}");
|
||||
assert!(!admits_all(&stops));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_binding_without_a_steward_reads_only_the_operator_store() -> Result<()> {
|
||||
let dir = tempfile::tempdir()?;
|
||||
let mut binding = binding_with_steward(dir.path(), "nobody");
|
||||
binding.steward = None;
|
||||
let stops = read_stops(&binding, "runtime");
|
||||
assert_eq!(stops.len(), 1);
|
||||
assert!(admits_all(&stops));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn store_entry(path: &Path) -> Result<CultCacheEnvelope> {
|
||||
let mut entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?;
|
||||
entries.pop().context("no entry")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_verb_replaces_a_retired_brake_record_and_the_sequence_advances_by_one() -> Result<()> {
|
||||
let dir = tempfile::tempdir()?;
|
||||
let path = dir.path().join("stop.cc");
|
||||
let store = SingleFileMessagePackBackingStore::new(&path);
|
||||
let retired = CultCacheEnvelope {
|
||||
key: "deployment".into(),
|
||||
r#type: "idunn.deployment_brake".into(),
|
||||
payload: vec![0x90],
|
||||
stored_at: "1970-01-01T00:00:00+00:00".into(),
|
||||
schema_id: Some("idunn.deployment_brake.v1".into()),
|
||||
};
|
||||
assert!(store.compare_exchange_snapshot(&[], &[retired])?);
|
||||
assert_eq!(engage(&path, "first", 10)?.sequence, 1);
|
||||
assert_eq!(store.pull_all_read_only_snapshot()?.len(), 1);
|
||||
assert_eq!(engage(&path, "second", 11)?.sequence, 2);
|
||||
assert_eq!(
|
||||
replace_stop(&path, "runtime", StopState::Clear, "third", 12)?.sequence,
|
||||
3
|
||||
);
|
||||
assert_eq!(DeploymentStop::decode(&store_entry(&path)?)?.reason, "third");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stale_exchange_is_refused_and_a_record_without_a_reason_is_not_written() -> Result<()> {
|
||||
let dir = tempfile::tempdir()?;
|
||||
let path = dir.path().join("stop.cc");
|
||||
engage(&path, "held", 10)?;
|
||||
let store = SingleFileMessagePackBackingStore::new(&path);
|
||||
let stale = store.pull_all_read_only_snapshot()?;
|
||||
engage(&path, "moved", 11)?;
|
||||
let next = DeploymentStop {
|
||||
schema_version: DEPLOYMENT_STOP_SCHEMA.into(),
|
||||
runtime_id: "runtime".into(),
|
||||
state: StopState::Clear,
|
||||
by: "someone".into(),
|
||||
reason: "late".into(),
|
||||
at_unix_millis: 12,
|
||||
sequence: 2,
|
||||
}
|
||||
.envelope()?;
|
||||
assert!(!store.compare_exchange_snapshot(&stale, &[next])?);
|
||||
assert!(replace_stop(&path, "runtime", StopState::Clear, " ", 13).is_err());
|
||||
assert_eq!(DeploymentStop::decode(&store_entry(&path)?)?.reason, "moved");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_a_missing_file_in_an_existing_directory_is_absent() -> Result<()> {
|
||||
let dir = tempfile::tempdir()?;
|
||||
let me = process_uid();
|
||||
// Parent is a regular file: ENOTDIR, not absence.
|
||||
let file = dir.path().join("plain");
|
||||
std::fs::write(&file, b"x")?;
|
||||
assert!(matches!(read_stop(&file.join("stop.cc"), "runtime", me), Stop::Unreadable(_)));
|
||||
// Parent does not exist.
|
||||
assert!(matches!(
|
||||
read_stop(&dir.path().join("nope/stop.cc"), "runtime", me),
|
||||
Stop::Unreadable(_)
|
||||
));
|
||||
// Zero-length file.
|
||||
let empty = dir.path().join("empty.cc");
|
||||
std::fs::write(&empty, b"")?;
|
||||
assert!(matches!(read_stop(&empty, "runtime", me), Stop::Unreadable(_)));
|
||||
// An existing directory with no store is absent.
|
||||
assert_eq!(read_stop(&dir.path().join("stop.cc"), "runtime", me), Stop::Absent);
|
||||
// A parent that cannot be searched (skipped as root, which can).
|
||||
#[cfg(unix)]
|
||||
if me != 0 {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let locked = dir.path().join("locked");
|
||||
std::fs::create_dir(&locked)?;
|
||||
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o000))?;
|
||||
let reading = read_stop(&locked.join("stop.cc"), "runtime", me);
|
||||
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o700))?;
|
||||
assert!(matches!(reading, Stop::Unreadable(_)), "{reading:?}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_author_is_the_uid_login_never_the_environment() -> Result<()> {
|
||||
let dir = tempfile::tempdir()?;
|
||||
let path = dir.path().join("stop.cc");
|
||||
// SAFETY: no other test reads USER, LOGNAME or LNAME.
|
||||
unsafe {
|
||||
std::env::set_var("USER", "spoof");
|
||||
std::env::set_var("LOGNAME", "spoof");
|
||||
}
|
||||
let written = engage(&path, "held", 10)?;
|
||||
assert_eq!(written.by, login_name(process_uid())?);
|
||||
assert_ne!(written.by, "spoof");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unreadable_names_the_command_and_a_break_glass_clear_replaces_junk_and_v1() -> Result<()> {
|
||||
let dir = tempfile::tempdir()?;
|
||||
let me = process_uid();
|
||||
let path = dir.path().join("stop.cc");
|
||||
std::fs::write(&path, b"junk that is no store")?;
|
||||
let Stop::Unreadable(why) = read_stop(&path, "runtime", me) else {
|
||||
bail!("junk admitted")
|
||||
};
|
||||
assert!(why.contains("idunn-provision deployment-brake-clear --store"), "{why}");
|
||||
assert!(why.contains("--runtime-id runtime"), "{why}");
|
||||
let cleared = replace_stop(&path, "runtime", StopState::Clear, "operator break-glass", 50)?;
|
||||
assert_eq!(cleared.sequence, 1);
|
||||
assert!(read_stop(&path, "runtime", me).admits());
|
||||
// A symlink is not the caller's to replace.
|
||||
let real = dir.path().join("real.cc");
|
||||
std::fs::write(&real, b"junk")?;
|
||||
let link = dir.path().join("link.cc");
|
||||
#[cfg(unix)]
|
||||
{
|
||||
std::os::unix::fs::symlink(&real, &link)?;
|
||||
assert!(replace_stop(&link, "runtime", StopState::Clear, "x", 60).is_err());
|
||||
assert_eq!(std::fs::read(&real)?, b"junk");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
24310
src/control_plane.rs
24310
src/control_plane.rs
File diff suppressed because it is too large
Load diff
1067
src/deployment.rs
1067
src/deployment.rs
File diff suppressed because it is too large
Load diff
|
|
@ -10,15 +10,18 @@ use cultnet_rs::{
|
|||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::control_plane::SequenceAdmittedReady;
|
||||
use crate::control_plane::{ODIN_RENDEZVOUS_CAPABILITY, ReadinessClass, SequenceAdmittedReady};
|
||||
use crate::deployment::{
|
||||
CapabilityDependency, DependencyKind, ExternalCapabilityBinding, OperatorBinding,
|
||||
CapabilityDependency, DeadlineBinding, DependencyKind, ExternalCapabilityBinding, OperatorBinding,
|
||||
ServiceTransport, SourceSelectionPolicy, StartupOrder, StateDeclaration, TargetDeclaration,
|
||||
capability_compatible,
|
||||
};
|
||||
|
||||
pub const SOURCE_SELECTION_FACTS_SCHEMA: &str = "idunn.source_selection_facts.v1";
|
||||
pub const COMPILED_DEPLOYMENT_PLAN_SCHEMA: &str = "idunn.compiled_deployment_plan.v2";
|
||||
pub const COMPILED_DEPLOYMENT_PLAN_SCHEMA: &str = "idunn.compiled_deployment_plan.v3";
|
||||
/// Plans sealed before phase deadlines existed. They keep their bytes, and so
|
||||
/// their plan id; their deadlines are Idunn's defaults at read time.
|
||||
pub const COMPILED_DEPLOYMENT_PLAN_SCHEMA_V2: &str = "idunn.compiled_deployment_plan.v2";
|
||||
pub const SEALED_RELEASE_SCHEMA: &str = "idunn.sealed_release.v1";
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
|
|
@ -460,6 +463,98 @@ pub fn select_dependencies(
|
|||
.collect()
|
||||
}
|
||||
|
||||
/// Idunn's default duration for each post-fencing phase, in seconds. A binding
|
||||
/// overrides any of them; the resolved values are frozen into the plan.
|
||||
pub const DEFAULT_FENCING_DEADLINE_SECONDS: u32 = 120;
|
||||
pub const DEFAULT_LEASING_DEADLINE_SECONDS: u32 = 120;
|
||||
// A stateful candidate may run migrations inside AwaitingReady (streampixels-
|
||||
// service runs Postgres's there). When this passes, a deployment whose
|
||||
// candidate was issued a write lease is left to the operator, not aborted; a
|
||||
// continuity is aborted and its release restarted.
|
||||
pub const DEFAULT_AWAITING_READY_DEADLINE_SECONDS: u32 = 300;
|
||||
pub const DEFAULT_ROUTING_DEADLINE_SECONDS: u32 = 120;
|
||||
pub const DEFAULT_COMMITTING_DEADLINE_SECONDS: u32 = 120;
|
||||
|
||||
/// Idunn's own deadlines, not binding-declared: a new `PhaseDeadlines` field
|
||||
/// would change every v3 plan id (`recomputed_plan_id` re-encodes the plan).
|
||||
/// Warming is timed for every transaction, owner or not.
|
||||
pub const WARMING_DEADLINE_SECONDS: u32 = 600;
|
||||
/// A pre-fencing abort is timed from its beginning.
|
||||
pub const PRE_FENCING_ABORT_DEADLINE_SECONDS: u32 = 300;
|
||||
|
||||
/// The resolved per-phase deadline durations a plan carries. Entering a
|
||||
/// post-fencing phase stamps its deadline from them, and the deadline
|
||||
/// resolver ends the phase once it passes.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct PhaseDeadlines {
|
||||
pub fencing_seconds: u32,
|
||||
pub leasing_seconds: u32,
|
||||
pub awaiting_ready_seconds: u32,
|
||||
pub routing_seconds: u32,
|
||||
pub committing_seconds: u32,
|
||||
}
|
||||
|
||||
impl PhaseDeadlines {
|
||||
pub const IDUNN_DEFAULTS: Self = Self {
|
||||
fencing_seconds: DEFAULT_FENCING_DEADLINE_SECONDS,
|
||||
leasing_seconds: DEFAULT_LEASING_DEADLINE_SECONDS,
|
||||
awaiting_ready_seconds: DEFAULT_AWAITING_READY_DEADLINE_SECONDS,
|
||||
routing_seconds: DEFAULT_ROUTING_DEADLINE_SECONDS,
|
||||
committing_seconds: DEFAULT_COMMITTING_DEADLINE_SECONDS,
|
||||
};
|
||||
|
||||
/// Binding overrides over Idunn's defaults.
|
||||
pub fn resolve(binding: Option<&DeadlineBinding>) -> Self {
|
||||
let defaults = Self::IDUNN_DEFAULTS;
|
||||
let Some(binding) = binding else {
|
||||
return defaults;
|
||||
};
|
||||
Self {
|
||||
fencing_seconds: binding.fencing_seconds.unwrap_or(defaults.fencing_seconds),
|
||||
leasing_seconds: binding.leasing_seconds.unwrap_or(defaults.leasing_seconds),
|
||||
awaiting_ready_seconds: binding
|
||||
.awaiting_ready_seconds
|
||||
.unwrap_or(defaults.awaiting_ready_seconds),
|
||||
routing_seconds: binding.routing_seconds.unwrap_or(defaults.routing_seconds),
|
||||
committing_seconds: binding
|
||||
.committing_seconds
|
||||
.unwrap_or(defaults.committing_seconds),
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_against(&self, binding: &OperatorBinding) -> Result<()> {
|
||||
let declared = binding.deadlines.clone().unwrap_or_default();
|
||||
for (name, frozen, declared) in [
|
||||
("fencing", self.fencing_seconds, declared.fencing_seconds),
|
||||
("leasing", self.leasing_seconds, declared.leasing_seconds),
|
||||
(
|
||||
"awaiting-ready",
|
||||
self.awaiting_ready_seconds,
|
||||
declared.awaiting_ready_seconds,
|
||||
),
|
||||
("routing", self.routing_seconds, declared.routing_seconds),
|
||||
(
|
||||
"committing",
|
||||
self.committing_seconds,
|
||||
declared.committing_seconds,
|
||||
),
|
||||
] {
|
||||
ensure!(
|
||||
(1..=crate::deployment::MAXIMUM_DEADLINE_SECONDS).contains(&frozen),
|
||||
"frozen {name} deadline is out of range"
|
||||
);
|
||||
if let Some(declared) = declared {
|
||||
ensure!(
|
||||
frozen == declared,
|
||||
"frozen {name} deadline differs from the binding"
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Private Idunn control-plane state. This contains host binding details and is
|
||||
/// never an Odin/CultMesh projection. The shared CultNet Expected incarnation
|
||||
/// is the sanitized topology projection derived only after a release validates
|
||||
|
|
@ -476,12 +571,17 @@ pub struct CompiledDeploymentPlan {
|
|||
pub binding_blob: Vec<u8>,
|
||||
pub dependencies: Vec<DependencySelection>,
|
||||
pub candidate_port: Option<u16>,
|
||||
/// Present exactly on v3 plans. Skipped when absent so a v2 plan re-encodes
|
||||
/// to its original bytes and keeps its plan id.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub deadlines: Option<PhaseDeadlines>,
|
||||
}
|
||||
|
||||
impl CompiledDeploymentPlan {
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
ensure!(
|
||||
self.schema == COMPILED_DEPLOYMENT_PLAN_SCHEMA,
|
||||
self.schema == COMPILED_DEPLOYMENT_PLAN_SCHEMA
|
||||
|| self.schema == COMPILED_DEPLOYMENT_PLAN_SCHEMA_V2,
|
||||
"unsupported deployment-plan schema"
|
||||
);
|
||||
require_sha256(&self.plan_id, "plan id")?;
|
||||
|
|
@ -492,6 +592,11 @@ impl CompiledDeploymentPlan {
|
|||
);
|
||||
let (declaration, binding) = self.parsed_inputs()?;
|
||||
self.source.validate_against(&binding)?;
|
||||
match (&self.schema[..], &self.deadlines) {
|
||||
(COMPILED_DEPLOYMENT_PLAN_SCHEMA, Some(deadlines)) => deadlines.validate_against(&binding)?,
|
||||
(COMPILED_DEPLOYMENT_PLAN_SCHEMA_V2, None) => {}
|
||||
_ => bail!("deployment plan deadlines differ from its schema"),
|
||||
}
|
||||
ensure!(
|
||||
sha256_id(&self.recipe_blob) == self.source.recipe_blob_sha256,
|
||||
"selected recipe blob differs from the exact plan recipe bytes"
|
||||
|
|
@ -527,6 +632,12 @@ impl CompiledDeploymentPlan {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// The deadline durations this plan runs under: frozen for v3, Idunn's
|
||||
/// defaults for a v2 plan that predates them.
|
||||
pub fn phase_deadlines(&self) -> PhaseDeadlines {
|
||||
self.deadlines.unwrap_or(PhaseDeadlines::IDUNN_DEFAULTS)
|
||||
}
|
||||
|
||||
pub(crate) fn parsed_inputs(&self) -> Result<(TargetDeclaration, OperatorBinding)> {
|
||||
let recipe_text = std::str::from_utf8(&self.recipe_blob)
|
||||
.context("stored deployment recipe is not UTF-8")?;
|
||||
|
|
@ -538,6 +649,25 @@ impl CompiledDeploymentPlan {
|
|||
Ok((declaration, binding))
|
||||
}
|
||||
|
||||
/// How this plan's target proves readiness, from its recipe's declarations
|
||||
/// and its binding's route, or the typed refusal that it declares none.
|
||||
/// Admission asks this before anything is sealed or installed.
|
||||
pub(crate) fn readiness_class(&self) -> Result<ReadinessClass> {
|
||||
let (declaration, binding) = self.parsed_inputs()?;
|
||||
Ok(ReadinessClass::declared(
|
||||
&declaration.target,
|
||||
declaration
|
||||
.provides
|
||||
.iter()
|
||||
.any(|provided| provided.capability == ODIN_RENDEZVOUS_CAPABILITY),
|
||||
declaration.dependencies.iter().any(|dependency| {
|
||||
dependency.kind == DependencyKind::SharedInfrastructure
|
||||
&& dependency.capability == ODIN_RENDEZVOUS_CAPABILITY
|
||||
}),
|
||||
binding.route.is_some(),
|
||||
)?)
|
||||
}
|
||||
|
||||
fn recomputed_plan_id(&self) -> Result<String> {
|
||||
let mut unsigned = self.clone();
|
||||
unsigned.plan_id.clear();
|
||||
|
|
@ -589,6 +719,7 @@ pub(crate) fn compile_deployment_plan(
|
|||
binding_blob: binding_bytes.to_vec(),
|
||||
dependencies,
|
||||
candidate_port,
|
||||
deadlines: Some(PhaseDeadlines::resolve(binding.deadlines.as_ref())),
|
||||
};
|
||||
plan.plan_id = plan.recomputed_plan_id()?;
|
||||
plan.validate()?;
|
||||
|
|
@ -603,6 +734,11 @@ pub struct ArtifactReceipt {
|
|||
pub sha256: String,
|
||||
pub size_bytes: u64,
|
||||
pub executable: bool,
|
||||
/// Files sealed executable inside a directory artifact. Skipped when empty
|
||||
/// so a receipt with no members keeps its original encoding, and so its
|
||||
/// sealed release id.
|
||||
#[serde(default, skip_serializing_if = "BTreeSet::is_empty")]
|
||||
pub executable_members: BTreeSet<PathBuf>,
|
||||
}
|
||||
|
||||
impl ArtifactReceipt {
|
||||
|
|
@ -616,7 +752,9 @@ impl ArtifactReceipt {
|
|||
.find(|artifact| artifact.id == self.artifact_id)
|
||||
.with_context(|| format!("plan declares no artifact {}", self.artifact_id))?;
|
||||
ensure!(
|
||||
self.destination == declared.destination && self.executable == declared.executable,
|
||||
self.destination == declared.destination
|
||||
&& self.executable == declared.executable
|
||||
&& self.executable_members == declared.executable_members,
|
||||
"artifact receipt differs from declared output"
|
||||
);
|
||||
if let Some(expected_sha256) = &declared.expected_sha256 {
|
||||
|
|
@ -716,14 +854,15 @@ impl SealedRelease {
|
|||
) -> Result<ExpectedIncarnation> {
|
||||
self.validate_against(plan)?;
|
||||
let (declaration, binding) = plan.parsed_inputs()?;
|
||||
let service = declaration.service()?;
|
||||
let route = match (&binding.route, plan.candidate_port) {
|
||||
(Some(bound), Some(port)) => Some(IdunnExpectedRoute {
|
||||
route_id: bound.route_id.clone(),
|
||||
transport: endpoint_scheme(declaration.service.transport)?.into(),
|
||||
transport: endpoint_scheme(service.transport)?.into(),
|
||||
stable_endpoint: bound.stable_endpoint.clone(),
|
||||
candidate_endpoint: format!(
|
||||
"{}://{}:{port}",
|
||||
endpoint_scheme(declaration.service.transport)?,
|
||||
endpoint_scheme(service.transport)?,
|
||||
bound.private_host
|
||||
),
|
||||
}),
|
||||
|
|
@ -733,7 +872,7 @@ impl SealedRelease {
|
|||
let artifact_sha256 = self
|
||||
.artifacts
|
||||
.iter()
|
||||
.find(|artifact| artifact.artifact_id == declaration.service.executable_artifact)
|
||||
.find(|artifact| artifact.artifact_id == service.executable_artifact)
|
||||
.context("sealed release has no executable service artifact")?
|
||||
.sha256
|
||||
.clone();
|
||||
|
|
@ -780,7 +919,7 @@ impl SealedRelease {
|
|||
.runtime_identity
|
||||
.expected_signer_identity_id
|
||||
.clone(),
|
||||
health_contract: declaration.service.health.contract.clone(),
|
||||
health_contract: service.health.contract.clone(),
|
||||
artifact_sha256,
|
||||
state_schema_generation: declaration
|
||||
.state
|
||||
|
|
@ -998,7 +1137,7 @@ fn sha256_id(bytes: &[u8]) -> String {
|
|||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
pub(crate) mod tests {
|
||||
use super::*;
|
||||
use cultnet_rs::{
|
||||
GameCultProviderHealthIdentity, GameCultRuntimeCapability, IdunnRuntimeActivationLaunch,
|
||||
|
|
@ -1009,7 +1148,7 @@ mod tests {
|
|||
verify_runtime_authority,
|
||||
};
|
||||
|
||||
const RECIPE: &str = r#"
|
||||
pub(crate) const RECIPE: &str = r#"
|
||||
schema = "gamecult.idunn.target_declaration.v1"
|
||||
target = "service"
|
||||
source_stamp_environment = "SERVICE_BUILD_COMMIT"
|
||||
|
|
@ -1072,7 +1211,7 @@ schema = "odin.verse-topology.v1"
|
|||
compatibility = "v1"
|
||||
"#;
|
||||
|
||||
const BINDING: &str = r#"
|
||||
pub(crate) const BINDING: &str = r#"
|
||||
schema = "gamecult.idunn.operator_binding.v2"
|
||||
target = "service"
|
||||
|
||||
|
|
@ -1147,7 +1286,7 @@ nodes = ["yggdrasil"]
|
|||
format!("sha256-{}", char::from(byte).to_string().repeat(64))
|
||||
}
|
||||
|
||||
fn source(recipe: &str) -> SourceSelectionFacts {
|
||||
pub(crate) fn source(recipe: &str) -> SourceSelectionFacts {
|
||||
SourceSelectionFacts {
|
||||
schema: SOURCE_SELECTION_FACTS_SCHEMA.into(),
|
||||
origin: "https://github.com/GameCult/Service.git".into(),
|
||||
|
|
@ -1287,17 +1426,18 @@ nodes = ["yggdrasil"]
|
|||
.unwrap()
|
||||
}
|
||||
|
||||
fn artifact_receipt() -> ArtifactReceipt {
|
||||
pub(crate) fn artifact_receipt() -> ArtifactReceipt {
|
||||
ArtifactReceipt {
|
||||
artifact_id: "daemon".into(),
|
||||
destination: "service".into(),
|
||||
sha256: digest(b'c'),
|
||||
size_bytes: 42,
|
||||
executable: true,
|
||||
executable_members: BTreeSet::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn external_input_receipt() -> ExternalInputMaterializationReceipt {
|
||||
pub(crate) fn external_input_receipt() -> ExternalInputMaterializationReceipt {
|
||||
ExternalInputMaterializationReceipt {
|
||||
input_id: "toolchain-index".into(),
|
||||
url: "https://example.invalid/toolchain-index".into(),
|
||||
|
|
@ -1444,6 +1584,71 @@ nodes = ["yggdrasil"]
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_plan_names_how_its_target_proves_readiness_and_agrees_with_its_expected() {
|
||||
use crate::control_plane::UndeclaredReadiness;
|
||||
let unrouted_binding = {
|
||||
let (head, tail) = BINDING.split_once("[route]").unwrap();
|
||||
format!("{head}[brakes]{}", tail.split_once("[brakes]").unwrap().1)
|
||||
};
|
||||
let unrouted = |recipe: &str| {
|
||||
recipe
|
||||
.replace("route_required = true", "route_required = false")
|
||||
.replace(
|
||||
r#"["GAMECULT_IDUNN_CANDIDATE_BIND", "GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#,
|
||||
r#"["GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#,
|
||||
)
|
||||
};
|
||||
let without_odin = RECIPE.split("[[dependencies]]").next().unwrap().to_owned();
|
||||
let provides_odin = |recipe: &str| {
|
||||
recipe.replace("capability = \"service.runtime\"", "capability = \"odin.verse-rendezvous\"")
|
||||
};
|
||||
let cases: [(&str, String, &str, Option<u16>, Result<ReadinessClass, ()>); 6] = [
|
||||
("depends on Odin, routed", RECIPE.to_owned(), BINDING, Some(18001), Ok(ReadinessClass::OdinCorrelated)),
|
||||
("routed, no Odin", without_odin.clone(), BINDING, Some(18001), Ok(ReadinessClass::RouteProof)),
|
||||
("unrouted, no Odin", unrouted(&without_odin), &unrouted_binding, None, Err(())),
|
||||
("provides Odin, unrouted", provides_odin(&unrouted(&without_odin)), &unrouted_binding, None, Ok(ReadinessClass::OdinSelf)),
|
||||
("depends on Odin, unrouted", unrouted(RECIPE), &unrouted_binding, None, Ok(ReadinessClass::OdinCorrelated)),
|
||||
// Only the shared-infrastructure declaration is the declaration.
|
||||
("requires the Odin capability, routed", RECIPE.replace("kind = \"shared-infrastructure\"", "kind = \"required\""), BINDING, Some(18001), Ok(ReadinessClass::RouteProof)),
|
||||
];
|
||||
for (name, recipe, binding, port, outcome) in cases {
|
||||
let providers = [ready_odin_provider("odin", "odin-yggdrasil", 1)];
|
||||
let plan = compile_deployment_plan(
|
||||
recipe.as_bytes(),
|
||||
binding.as_bytes(),
|
||||
source(&recipe),
|
||||
"service-incarnation-1",
|
||||
port,
|
||||
110,
|
||||
if recipe.contains("[[dependencies]]") { &providers } else { &[] },
|
||||
)
|
||||
.unwrap_or_else(|error| panic!("{name}: {error:#}"));
|
||||
let release = SealedRelease::new(
|
||||
&plan,
|
||||
vec![artifact_receipt()],
|
||||
vec![external_input_receipt()],
|
||||
120,
|
||||
)
|
||||
.unwrap();
|
||||
let expected = release.expected_projection(&plan).unwrap();
|
||||
match outcome {
|
||||
Ok(class) => {
|
||||
assert_eq!(plan.readiness_class().unwrap(), class, "{name}");
|
||||
assert_eq!(ReadinessClass::of(&expected), Ok(class), "{name}");
|
||||
}
|
||||
Err(()) => {
|
||||
let error = plan.readiness_class().expect_err(name);
|
||||
let undeclared = UndeclaredReadiness {
|
||||
target: "service".into(),
|
||||
};
|
||||
assert_eq!(error.downcast_ref::<UndeclaredReadiness>(), Some(&undeclared), "{name}");
|
||||
assert_eq!(ReadinessClass::of(&expected), Err(undeclared), "{name}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expected_projection_is_sanitized_and_names_the_selected_graph() {
|
||||
let plan = plan();
|
||||
|
|
@ -1594,4 +1799,199 @@ nodes = ["yggdrasil"]
|
|||
changed.external_inputs[0].size_bytes = 18;
|
||||
assert!(changed.validate_against(&plan).is_err());
|
||||
}
|
||||
|
||||
fn hex(bytes: &[u8]) -> String {
|
||||
bytes.iter().map(|byte| format!("{byte:02x}")).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn receipt_without_members_encodes_as_before() {
|
||||
// Both values were computed outside this code, from the five-field
|
||||
// msgpack array the receipt encoded to at base d74ac8d.
|
||||
const BASE_RECEIPT: &str = "95a66461656d6f6ea773657276696365d9477368613235362d636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363632ac3";
|
||||
const BASE_RELEASE_ID: &str =
|
||||
"sha256-b9143d8f546ad71c493c70037813547aaec10fd87994a6f3bdc77ac3a8b3495e";
|
||||
let receipt = artifact_receipt();
|
||||
let encoded = rmp_serde::to_vec(&receipt).unwrap();
|
||||
assert_eq!(hex(&encoded), BASE_RECEIPT);
|
||||
let bytes = (0..BASE_RECEIPT.len())
|
||||
.step_by(2)
|
||||
.map(|at| u8::from_str_radix(&BASE_RECEIPT[at..at + 2], 16).unwrap())
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(rmp_serde::from_slice::<ArtifactReceipt>(&bytes).unwrap(), receipt);
|
||||
let release = SealedRelease {
|
||||
schema: SEALED_RELEASE_SCHEMA.into(),
|
||||
sealed_release_id: String::new(),
|
||||
plan_id: digest(b'a'),
|
||||
artifacts: vec![receipt],
|
||||
external_inputs: vec![],
|
||||
sealed_at_unix_millis: 0,
|
||||
};
|
||||
assert_eq!(release.recomputed_release_id().unwrap(), BASE_RELEASE_ID);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sealed_release_address_covers_executable_members() {
|
||||
let release_with = |members: &[&str]| SealedRelease {
|
||||
schema: SEALED_RELEASE_SCHEMA.into(),
|
||||
sealed_release_id: String::new(),
|
||||
plan_id: digest(b'a'),
|
||||
artifacts: vec![ArtifactReceipt {
|
||||
executable: false,
|
||||
executable_members: members.iter().map(PathBuf::from).collect(),
|
||||
..artifact_receipt()
|
||||
}],
|
||||
external_inputs: vec![],
|
||||
sealed_at_unix_millis: 0,
|
||||
};
|
||||
let none = release_with(&[]).recomputed_release_id().unwrap();
|
||||
let one = release_with(&["bin/daemon"]).recomputed_release_id().unwrap();
|
||||
let other = release_with(&["bin/other"]).recomputed_release_id().unwrap();
|
||||
let two = release_with(&["bin/daemon", "bin/other"])
|
||||
.recomputed_release_id()
|
||||
.unwrap();
|
||||
let all = [&none, &one, &other, &two];
|
||||
for (index, left) in all.iter().enumerate() {
|
||||
for right in &all[index + 1..] {
|
||||
assert_ne!(left, right);
|
||||
}
|
||||
}
|
||||
|
||||
// The declaration is the other half: a receipt must carry exactly the
|
||||
// declared list, in a plan that declares one and in one that does not.
|
||||
let plain = plan();
|
||||
let mut with_members = artifact_receipt();
|
||||
with_members.executable_members = BTreeSet::from([PathBuf::from("bin/daemon")]);
|
||||
assert!(
|
||||
SealedRelease::new(&plain, vec![with_members], vec![external_input_receipt()], 120)
|
||||
.is_err()
|
||||
);
|
||||
let recipe = RECIPE.replace("executable = true", "executable_members = [\"bin/daemon\", \"lib/tool\"]");
|
||||
let declaring = compile_deployment_plan(
|
||||
recipe.as_bytes(),
|
||||
BINDING.as_bytes(),
|
||||
source(&recipe),
|
||||
"service-incarnation-1",
|
||||
Some(18001),
|
||||
110,
|
||||
&[ready_odin_provider("odin", "odin-yggdrasil", 1)],
|
||||
)
|
||||
.unwrap();
|
||||
let honest = ArtifactReceipt {
|
||||
executable: false,
|
||||
executable_members: BTreeSet::from([PathBuf::from("bin/daemon"), PathBuf::from("lib/tool")]),
|
||||
..artifact_receipt()
|
||||
};
|
||||
let release = SealedRelease::new(
|
||||
&declaring,
|
||||
vec![honest.clone()],
|
||||
vec![external_input_receipt()],
|
||||
120,
|
||||
)
|
||||
.unwrap();
|
||||
release.validate_against(&declaring).unwrap();
|
||||
for drifted in [
|
||||
BTreeSet::from([PathBuf::from("bin/daemon")]),
|
||||
BTreeSet::from([PathBuf::from("bin/daemon"), PathBuf::from("lib/other")]),
|
||||
BTreeSet::new(),
|
||||
] {
|
||||
let mut receipt = honest.clone();
|
||||
receipt.executable_members = drifted;
|
||||
assert!(
|
||||
SealedRelease::new(&declaring, vec![receipt], vec![external_input_receipt()], 120)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
let mut changed = release.clone();
|
||||
changed.artifacts[0].executable_members.insert(PathBuf::from("lib/extra"));
|
||||
assert!(changed.validate_against(&declaring).is_err());
|
||||
}
|
||||
|
||||
fn binding_with_deadlines(table: &str) -> String {
|
||||
BINDING.replace("[placement]", &format!("{table}\n[placement]"))
|
||||
}
|
||||
|
||||
fn plan_under(binding: &str) -> Result<CompiledDeploymentPlan> {
|
||||
let providers = [ready_odin_provider("odin", "odin-yggdrasil", 1)];
|
||||
compile_deployment_plan(
|
||||
RECIPE.as_bytes(),
|
||||
binding.as_bytes(),
|
||||
source(RECIPE),
|
||||
"service-incarnation-1",
|
||||
Some(18001),
|
||||
110,
|
||||
&providers,
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_compiled_plan_freezes_idunn_default_deadlines() {
|
||||
let plan = plan();
|
||||
assert_eq!(plan.schema, COMPILED_DEPLOYMENT_PLAN_SCHEMA);
|
||||
assert_eq!(plan.deadlines, Some(PhaseDeadlines::IDUNN_DEFAULTS));
|
||||
assert_eq!(plan.phase_deadlines(), PhaseDeadlines::IDUNN_DEFAULTS);
|
||||
plan.validate().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_binding_overrides_only_the_deadlines_it_names() {
|
||||
let binding = binding_with_deadlines("[deadlines]\nawaiting_ready_seconds = 45\n");
|
||||
let plan = plan_under(&binding).unwrap();
|
||||
let frozen = plan.phase_deadlines();
|
||||
assert_eq!(frozen.awaiting_ready_seconds, 45);
|
||||
assert_eq!(
|
||||
PhaseDeadlines {
|
||||
awaiting_ready_seconds: DEFAULT_AWAITING_READY_DEADLINE_SECONDS,
|
||||
..frozen
|
||||
},
|
||||
PhaseDeadlines::IDUNN_DEFAULTS
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deadline_bounds_and_unknown_phases_are_refused() {
|
||||
for table in [
|
||||
"[deadlines]\nrouting_seconds = 0\n",
|
||||
"[deadlines]\nrouting_seconds = 86401\n",
|
||||
"[deadlines]\nwarming_seconds = 5\n",
|
||||
] {
|
||||
assert!(
|
||||
OperatorBinding::parse(&binding_with_deadlines(table)).is_err(),
|
||||
"{table}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frozen_deadlines_must_agree_with_the_binding_and_the_plan_schema() {
|
||||
let binding = binding_with_deadlines("[deadlines]\nrouting_seconds = 45\n");
|
||||
let sealed = |mut plan: CompiledDeploymentPlan| {
|
||||
plan.plan_id = plan.recomputed_plan_id().unwrap();
|
||||
plan
|
||||
};
|
||||
let honest = plan_under(&binding).unwrap();
|
||||
|
||||
let mut drifted = honest.clone();
|
||||
drifted.deadlines.as_mut().unwrap().routing_seconds = 46;
|
||||
assert!(sealed(drifted).validate().is_err());
|
||||
|
||||
let mut absent = honest.clone();
|
||||
absent.deadlines = None;
|
||||
assert!(sealed(absent).validate().is_err());
|
||||
|
||||
let mut v2_with_deadlines = honest.clone();
|
||||
v2_with_deadlines.schema = COMPILED_DEPLOYMENT_PLAN_SCHEMA_V2.into();
|
||||
assert!(sealed(v2_with_deadlines).validate().is_err());
|
||||
|
||||
// A plan sealed before deadlines existed keeps its nine-field encoding,
|
||||
// so its plan id still holds, and it runs on Idunn's defaults.
|
||||
let mut v2 = plan();
|
||||
v2.schema = COMPILED_DEPLOYMENT_PLAN_SCHEMA_V2.into();
|
||||
v2.deadlines = None;
|
||||
let v2 = sealed(v2);
|
||||
v2.validate().unwrap();
|
||||
assert_eq!(v2.phase_deadlines(), PhaseDeadlines::IDUNN_DEFAULTS);
|
||||
assert_eq!(rmp_serde::to_vec(&v2).unwrap()[0], 0x99);
|
||||
assert_eq!(rmp_serde::to_vec(&honest).unwrap()[0], 0x9a);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
5696
src/drivers.rs
5696
src/drivers.rs
File diff suppressed because it is too large
Load diff
49
src/host.rs
49
src/host.rs
|
|
@ -30,13 +30,13 @@ use serde::Deserialize;
|
|||
|
||||
use crate::control_plane::read_trust_anchor;
|
||||
use crate::deployment::{
|
||||
ArtifactSource, HostWorkloadBinding, IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT,
|
||||
HostWorkloadBinding, IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT,
|
||||
IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, LaunchArgument, OperatorBinding, TargetDeclaration,
|
||||
};
|
||||
use crate::deployment_plan::{ArtifactReceipt, CompiledDeploymentPlan, SealedRelease};
|
||||
use crate::deployment_plan::{CompiledDeploymentPlan, SealedRelease};
|
||||
use crate::drivers::{
|
||||
HostWorkloadObservation, InstalledReleaseObservation, copy_artifact, copy_tree,
|
||||
digest_artifact, release_artifact, remove_tree_inside, sha256_id, write_runtime_bundle_records,
|
||||
HostWorkloadObservation, InstalledReleaseObservation, copy_tree, digest_artifact,
|
||||
release_artifact, remove_tree_inside, seal_artifact, sha256_id, write_runtime_bundle_records,
|
||||
};
|
||||
use crate::host_actuator::{
|
||||
HOST_ACTUATOR_CHANNEL, HOST_ACTUATOR_CONNECTION_ID, HOST_ACTUATOR_PING_MS,
|
||||
|
|
@ -631,11 +631,11 @@ impl Actuator {
|
|||
workload: &HostWorkloadBinding,
|
||||
installed: &Path,
|
||||
) -> Result<(PathBuf, Vec<String>)> {
|
||||
let artifact = release_artifact(declaration, &declaration.service.executable_artifact)?;
|
||||
let artifact = release_artifact(declaration, &declaration.service()?.executable_artifact)?;
|
||||
let executable = installed.join(&artifact.destination);
|
||||
ensure!(executable.is_file(), "sealed service executable is absent");
|
||||
let mut arguments = Vec::new();
|
||||
for argument in &declaration.service.arguments {
|
||||
for argument in &declaration.service()?.arguments {
|
||||
arguments.push(match argument {
|
||||
LaunchArgument::Literal { value } => value.clone(),
|
||||
LaunchArgument::Binding { name } => workload
|
||||
|
|
@ -1048,37 +1048,12 @@ fn materialize(
|
|||
fs::create_dir_all(&staging)?;
|
||||
let mut artifacts = Vec::new();
|
||||
for artifact in &declaration.artifacts {
|
||||
let source_path = match artifact.source_kind {
|
||||
ArtifactSource::RunnerOutput | ArtifactSource::WorktreeTree => {
|
||||
source.join(&artifact.source)
|
||||
}
|
||||
};
|
||||
ensure!(
|
||||
source_path.exists(),
|
||||
"declared artifact output {} is absent",
|
||||
artifact.id
|
||||
);
|
||||
let destination = staging.join(&artifact.destination);
|
||||
ensure!(
|
||||
destination.starts_with(&staging),
|
||||
"artifact destination escaped its staging root"
|
||||
);
|
||||
copy_artifact(&source, &source_path, &destination)?;
|
||||
let (sha256, size_bytes) = digest_artifact(&destination)?;
|
||||
if let Some(expected) = &artifact.expected_sha256 {
|
||||
ensure!(
|
||||
&sha256 == expected,
|
||||
"artifact {} differs from its recipe-pinned digest",
|
||||
artifact.id
|
||||
);
|
||||
}
|
||||
artifacts.push(ArtifactReceipt {
|
||||
artifact_id: artifact.id.clone(),
|
||||
destination: artifact.destination.clone(),
|
||||
sha256: format!("sha256-{sha256}"),
|
||||
size_bytes,
|
||||
executable: artifact.executable,
|
||||
});
|
||||
artifacts.push(seal_artifact(
|
||||
artifact,
|
||||
&source,
|
||||
&source.join(&artifact.source),
|
||||
&staging,
|
||||
)?);
|
||||
}
|
||||
let release = SealedRelease::new(plan, artifacts, Vec::new(), sealed_at_unix_millis)?;
|
||||
Ok((release, staging))
|
||||
|
|
|
|||
|
|
@ -963,6 +963,12 @@ impl WorkloadPort for HostActuatorWorkloadDriver<'_> {
|
|||
};
|
||||
Ok(!running)
|
||||
}
|
||||
|
||||
/// A managed host's reboot is not yet observable here: follow-up
|
||||
/// host-actuator-reboot. Until then continuity charges its restarts.
|
||||
fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -1044,6 +1050,43 @@ mod tests {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// A managed host's reboot is not observable yet (follow-up
|
||||
/// host-actuator-reboot): the driver never claims a reboot, so continuity
|
||||
/// never grants a free re-materialization it cannot justify.
|
||||
#[test]
|
||||
fn a_managed_host_workload_is_never_claimed_lost_to_a_reboot() -> Result<()> {
|
||||
let (_temp, idunn, _host) = identities()?;
|
||||
let hub = HostActuatorHub::bind("127.0.0.1:0".parse()?)?;
|
||||
let driver = HostActuatorWorkloadDriver {
|
||||
access: HostActuatorAccess {
|
||||
hub: Arc::new(Mutex::new(hub)),
|
||||
anchors: BTreeMap::new(),
|
||||
signer: &idunn,
|
||||
},
|
||||
};
|
||||
let observation = WorkloadObservation::Host(HostWorkloadObservation {
|
||||
host: "raven".into(),
|
||||
actuator_identity_id: "actuator".into(),
|
||||
process_id: 1,
|
||||
process_creation_time: 1,
|
||||
session_id: 1,
|
||||
user_sid: "S-1-5-21-1-2-3-1001".into(),
|
||||
executable: "C:/GameCult/idunn/releases/muninn/x/muninn.exe".into(),
|
||||
executable_sha256: format!("sha256-{}", "1".repeat(64)),
|
||||
command_line_sha256: format!("sha256-{}", "2".repeat(64)),
|
||||
environment_names: Vec::new(),
|
||||
environment_contract_sha256: format!("sha256-{}", "3".repeat(64)),
|
||||
runtime_bundle: "C:/GameCult/idunn/runtime/muninn/x".into(),
|
||||
runtime_instance_id: format!("sha256-{}", "a".repeat(64)),
|
||||
activation_signer_identity_id: "activation".into(),
|
||||
activation_signer_public_key: vec![1; 32],
|
||||
exit_code: Some(1),
|
||||
});
|
||||
assert!(!driver.lost_to_host_reboot(&observation, 1)?);
|
||||
assert!(!driver.lost_to_host_reboot(&observation, u64::MAX)?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A real hub and a real client socket on loopback: Hello attaches under
|
||||
/// the bound anchor, a wrong host is refused, a request is answered.
|
||||
#[test]
|
||||
|
|
@ -1108,7 +1151,7 @@ mod tests {
|
|||
let answerer = thread::spawn(move || -> Result<()> {
|
||||
for _ in 0..500 {
|
||||
client.poll_resends()?;
|
||||
while let Some(frame) = client.receive_once()? {
|
||||
if let Some(frame) = client.receive_once()? {
|
||||
let envelope = HostActuatorEnvelope::decode(&frame.payload)?;
|
||||
let request = envelope.open_request("raven", &idunn_anchor)?;
|
||||
assert_eq!(envelope.session_nonce, nonce);
|
||||
|
|
|
|||
1046
src/incident.rs
Normal file
1046
src/incident.rs
Normal file
File diff suppressed because it is too large
Load diff
3238
src/launcher.rs
Normal file
3238
src/launcher.rs
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -1,7 +1,14 @@
|
|||
mod brake;
|
||||
pub mod control_plane;
|
||||
pub mod deployment;
|
||||
pub mod deployment_plan;
|
||||
pub mod drivers;
|
||||
pub mod host;
|
||||
pub mod host_actuator;
|
||||
#[cfg(unix)]
|
||||
pub mod launcher;
|
||||
mod incident;
|
||||
mod runner_evidence;
|
||||
pub mod provisioning;
|
||||
#[cfg(unix)]
|
||||
mod steward_orders;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
use anyhow::{Context, Result, anyhow, bail};
|
||||
use crate::brake::{Stop, StopState, process_uid, read_stop, replace_stop};
|
||||
use anyhow::{Context, Result, anyhow, bail, ensure};
|
||||
use cultcache_rs::{
|
||||
CultCacheEnvelope, CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore,
|
||||
};
|
||||
|
|
@ -6,15 +7,13 @@ use cultnet_rs::{
|
|||
GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA, GameCultProviderHealthIdentity,
|
||||
GameCultServiceTrustAnchorRecord, IDUNN_AUTHENTICATED_PROVIDER_HEALTH_PROJECTION_SCHEMA,
|
||||
IDUNN_AUTHENTICATED_PROVIDER_HEALTH_PROJECTION_SIGNING_PURPOSE,
|
||||
IDUNN_DAEMON_HEALTH_TRUST_BINDING_SCHEMA, IDUNN_DEPLOYMENT_BRAKE_AUTHORITY,
|
||||
IDUNN_DEPLOYMENT_BRAKE_ID, IDUNN_DEPLOYMENT_BRAKE_SCHEMA, IDUNN_DEPLOYMENT_BRAKE_SCOPE,
|
||||
IDUNN_DEPLOYMENT_RELEASE_PURPOSE, IDUNN_LIFECYCLE_BRAKE_AUTHORITY,
|
||||
IDUNN_DAEMON_HEALTH_TRUST_BINDING_SCHEMA,
|
||||
IDUNN_LIFECYCLE_BRAKE_AUTHORITY,
|
||||
IDUNN_LIFECYCLE_BRAKE_SCHEMA, IDUNN_LIFECYCLE_BRAKE_SCOPE, IdunnDaemonHealthTrustBindingRecord,
|
||||
IdunnDeploymentBrakeObservation, IdunnDeploymentBrakeOperatorIdentity,
|
||||
IdunnDeploymentBrakeRecord, IdunnDeploymentBrakeReleasePurpose, IdunnLifecycleBrakeObservation,
|
||||
IdunnLifecycleBrakeObservation,
|
||||
IdunnLifecycleBrakeRecord, IdunnServiceIdentity, OdinTopologyIdentity, ServiceIdentityProfile,
|
||||
ServiceIdentityTrustAnchor, derive_service_identity_id, enroll_service_identity_at,
|
||||
evaluate_idunn_continuity_restart, evaluate_idunn_deployment_brake,
|
||||
evaluate_idunn_continuity_restart,
|
||||
export_service_identity_trust_anchor, open_service_identity_at,
|
||||
};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
|
@ -80,17 +79,8 @@ pub fn run(args: impl IntoIterator<Item = String>) -> Result<()> {
|
|||
"export-odin-topology-public-anchor" => {
|
||||
export_identity_anchor::<OdinTopologyIdentity>(&options, "Odin topology public anchor")?
|
||||
}
|
||||
"enroll-deployment-brake-operator" => {
|
||||
enroll_identity::<IdunnDeploymentBrakeOperatorIdentity>(&options)?
|
||||
}
|
||||
"export-deployment-brake-operator-anchor" => {
|
||||
export_identity_anchor::<IdunnDeploymentBrakeOperatorIdentity>(
|
||||
&options,
|
||||
"deployment brake operator anchor",
|
||||
)?
|
||||
}
|
||||
"deployment-brake-engage" => deployment_brake_engage(&options)?,
|
||||
"deployment-brake-release" => deployment_brake_release(&options)?,
|
||||
"deployment-brake-engage" => deployment_brake_write(&options, StopState::Engaged)?,
|
||||
"deployment-brake-clear" => deployment_brake_write(&options, StopState::Clear)?,
|
||||
"deployment-brake-status" => deployment_brake_status(&options)?,
|
||||
"lifecycle-brake-engage" => lifecycle_brake_engage(&options)?,
|
||||
"lifecycle-brake-release" => lifecycle_brake_release(&options)?,
|
||||
|
|
@ -157,153 +147,60 @@ fn provider_health_public_key(options: &BTreeMap<String, String>) -> Result<Stri
|
|||
encode_public_key(&signer.entry().public_key)
|
||||
}
|
||||
|
||||
fn deployment_brake_engage(options: &BTreeMap<String, String>) -> Result<()> {
|
||||
require_only(
|
||||
options,
|
||||
&[
|
||||
"store",
|
||||
"runtime-id",
|
||||
"owner",
|
||||
"reason",
|
||||
"observed-at-unix-millis",
|
||||
],
|
||||
/// Engage or clear the deployment stop. The author is the process uid's login
|
||||
/// name and the time is this verb's clock: neither is an argument.
|
||||
fn deployment_brake_write(options: &BTreeMap<String, String>, state: StopState) -> Result<()> {
|
||||
require_only(options, &["store", "runtime-id", "reason"])?;
|
||||
let now = u64::try_from(
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)?
|
||||
.as_millis(),
|
||||
)?;
|
||||
let observed = parse_u64(options, "observed-at-unix-millis")?;
|
||||
let record = IdunnDeploymentBrakeRecord {
|
||||
schema_version: IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into(),
|
||||
brake_id: IDUNN_DEPLOYMENT_BRAKE_ID.into(),
|
||||
authority: IDUNN_DEPLOYMENT_BRAKE_AUTHORITY.into(),
|
||||
runtime_id: required(options, "runtime-id")?.into(),
|
||||
status: "engaged".into(),
|
||||
scope: IDUNN_DEPLOYMENT_BRAKE_SCOPE.into(),
|
||||
reason: required(options, "reason")?.into(),
|
||||
observed_at_unix_millis: observed,
|
||||
expires_at_unix_millis: None,
|
||||
authorization_id: None,
|
||||
authorization_purpose: None,
|
||||
authorized_release_id: None,
|
||||
authorized_deployment_id: None,
|
||||
authorized_by: None,
|
||||
authorization_issued_at_unix_millis: None,
|
||||
authorization_expires_at_unix_millis: None,
|
||||
signature_algorithm: None,
|
||||
signature: None,
|
||||
private_state_exposed: false,
|
||||
updated_by: required(options, "owner")?.into(),
|
||||
};
|
||||
record.validate()?;
|
||||
replace_brake(&path(options, "store")?, record, observed)
|
||||
}
|
||||
|
||||
fn deployment_brake_release(options: &BTreeMap<String, String>) -> Result<()> {
|
||||
require_only(
|
||||
options,
|
||||
&[
|
||||
"store",
|
||||
"private-store",
|
||||
"runtime-id",
|
||||
"owner",
|
||||
"reason",
|
||||
"authorization-id",
|
||||
"release-id",
|
||||
"deployment-id",
|
||||
"issued-at-unix-millis",
|
||||
"expires-at-unix-millis",
|
||||
],
|
||||
)?;
|
||||
let issued = parse_u64(options, "issued-at-unix-millis")?;
|
||||
let expires = parse_u64(options, "expires-at-unix-millis")?;
|
||||
let signer = open_service_identity_at::<IdunnDeploymentBrakeOperatorIdentity>(&path(
|
||||
options,
|
||||
"private-store",
|
||||
)?)?;
|
||||
let mut record = IdunnDeploymentBrakeRecord {
|
||||
schema_version: IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into(),
|
||||
brake_id: IDUNN_DEPLOYMENT_BRAKE_ID.into(),
|
||||
authority: IDUNN_DEPLOYMENT_BRAKE_AUTHORITY.into(),
|
||||
runtime_id: required(options, "runtime-id")?.into(),
|
||||
status: "released".into(),
|
||||
scope: IDUNN_DEPLOYMENT_BRAKE_SCOPE.into(),
|
||||
reason: required(options, "reason")?.into(),
|
||||
observed_at_unix_millis: issued,
|
||||
expires_at_unix_millis: Some(expires),
|
||||
authorization_id: Some(required(options, "authorization-id")?.into()),
|
||||
authorization_purpose: Some(IDUNN_DEPLOYMENT_RELEASE_PURPOSE.into()),
|
||||
authorized_release_id: Some(required(options, "release-id")?.into()),
|
||||
authorized_deployment_id: Some(required(options, "deployment-id")?.into()),
|
||||
authorized_by: Some(signer.trust_anchor()?.identity_id),
|
||||
authorization_issued_at_unix_millis: Some(issued),
|
||||
authorization_expires_at_unix_millis: Some(expires),
|
||||
signature_algorithm: Some("ed25519".into()),
|
||||
signature: None,
|
||||
private_state_exposed: false,
|
||||
updated_by: required(options, "owner")?.into(),
|
||||
};
|
||||
record.signature = Some(
|
||||
signer
|
||||
.sign::<IdunnDeploymentBrakeReleasePurpose>(&rmp_serde::to_vec(&record)?)
|
||||
.signature,
|
||||
);
|
||||
record.validate()?;
|
||||
replace_brake(&path(options, "store")?, record, issued)
|
||||
}
|
||||
|
||||
fn deployment_brake_status(options: &BTreeMap<String, String>) -> Result<()> {
|
||||
allow_only(
|
||||
options,
|
||||
&[
|
||||
"store",
|
||||
"operator-anchor",
|
||||
"runtime-id",
|
||||
"release-id",
|
||||
"deployment-id",
|
||||
"now-unix-millis",
|
||||
],
|
||||
)?;
|
||||
let exact_actuation_fields = ["release-id", "deployment-id", "now-unix-millis"];
|
||||
let exact_actuation_field_count = exact_actuation_fields
|
||||
.into_iter()
|
||||
.filter(|field| options.contains_key(*field))
|
||||
.count();
|
||||
if exact_actuation_field_count != 0
|
||||
&& exact_actuation_field_count != exact_actuation_fields.len()
|
||||
{
|
||||
bail!("exact deployment-brake actuation validation requires release, deployment, and time");
|
||||
}
|
||||
let record = read_brake(&path(options, "store")?)?;
|
||||
if record.runtime_id != required(options, "runtime-id")? {
|
||||
bail!("deployment brake belongs to another runtime")
|
||||
}
|
||||
if record.status == "engaged" {
|
||||
if exact_actuation_field_count != 0 {
|
||||
bail!("engaged deployment brake denies exact actuation validation");
|
||||
}
|
||||
println!(
|
||||
"engaged runtime={} scope={} owner={} reason={}",
|
||||
record.runtime_id, record.scope, record.updated_by, record.reason
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
let anchor = read_operator_anchor(&path(options, "operator-anchor")?)?;
|
||||
evaluate_idunn_deployment_brake(
|
||||
IdunnDeploymentBrakeObservation::Present(&record),
|
||||
&anchor,
|
||||
let record = replace_stop(
|
||||
&path(options, "store")?,
|
||||
required(options, "runtime-id")?,
|
||||
required(options, "release-id")?,
|
||||
required(options, "deployment-id")?,
|
||||
parse_u64(options, "now-unix-millis")?,
|
||||
)
|
||||
.map_err(|d| anyhow!("deployment brake denied actuation: {d:?}"))?;
|
||||
state,
|
||||
required(options, "reason")?,
|
||||
now,
|
||||
)?;
|
||||
println!(
|
||||
"released runtime={} release={} deployment={} owner={}",
|
||||
"{} runtime={} by={} sequence={}",
|
||||
record.state.name(),
|
||||
record.runtime_id,
|
||||
record.authorized_release_id.unwrap(),
|
||||
record.authorized_deployment_id.unwrap(),
|
||||
record.updated_by
|
||||
record.by,
|
||||
record.sequence
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Print the stop and exit nonzero unless deployments admit, so a script can
|
||||
/// gate on it as it could on the retired exact-actuation validation.
|
||||
fn deployment_brake_status(options: &BTreeMap<String, String>) -> Result<()> {
|
||||
allow_only(options, &["store", "runtime-id", "owner-user"])?;
|
||||
// A steward's store is owned by the steward: the operator names that user
|
||||
// to read it, and the owner check is the same one Idunn makes.
|
||||
let owner = match options.get("owner-user") {
|
||||
Some(user) => crate::steward_orders::user_uid(user)?,
|
||||
None => process_uid(),
|
||||
};
|
||||
let stop = read_stop(&path(options, "store")?, required(options, "runtime-id")?, owner);
|
||||
match &stop {
|
||||
Stop::Absent => println!("absent: deployments admit"),
|
||||
Stop::Unreadable(why) => println!("unreadable: deployments wait ({why})"),
|
||||
Stop::Clear(record) | Stop::Engaged(record) => println!(
|
||||
"{} runtime={} by={} at={} sequence={} reason={}",
|
||||
record.state.name(),
|
||||
record.runtime_id,
|
||||
record.by,
|
||||
record.at_unix_millis,
|
||||
record.sequence,
|
||||
record.reason
|
||||
),
|
||||
}
|
||||
ensure!(stop.admits(), "the deployment brake does not admit deployments");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn lifecycle_brake_engage(options: &BTreeMap<String, String>) -> Result<()> {
|
||||
require_only(
|
||||
options,
|
||||
|
|
@ -407,7 +304,7 @@ fn lifecycle_brake_status(options: &BTreeMap<String, String>) -> Result<()> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn replace_lifecycle_brake(path: &Path, record: IdunnLifecycleBrakeRecord) -> Result<()> {
|
||||
pub(crate) fn replace_lifecycle_brake(path: &Path, record: IdunnLifecycleBrakeRecord) -> Result<()> {
|
||||
record.validate()?;
|
||||
let store = SingleFileMessagePackBackingStore::new(path);
|
||||
let entries = store.pull_all_read_only_snapshot()?;
|
||||
|
|
@ -482,70 +379,6 @@ fn lifecycle_identifier<'a>(options: &'a BTreeMap<String, String>, name: &str) -
|
|||
Ok(value)
|
||||
}
|
||||
|
||||
fn replace_brake(path: &Path, record: IdunnDeploymentBrakeRecord, millis: u64) -> Result<()> {
|
||||
let store = SingleFileMessagePackBackingStore::new(path);
|
||||
let entries = store.pull_all_read_only_snapshot()?;
|
||||
let current = match entries.as_slice() {
|
||||
[] => None,
|
||||
[entry]
|
||||
if entry.r#type == IdunnDeploymentBrakeRecord::TYPE
|
||||
&& entry.key == IDUNN_DEPLOYMENT_BRAKE_ID =>
|
||||
{
|
||||
Some(entry.clone())
|
||||
}
|
||||
_ => bail!("deployment brake store is corrupt or ambiguous"),
|
||||
};
|
||||
let next = typed_envelope(
|
||||
IDUNN_DEPLOYMENT_BRAKE_ID,
|
||||
&record,
|
||||
IDUNN_DEPLOYMENT_BRAKE_SCHEMA,
|
||||
millis,
|
||||
)?;
|
||||
if !store.compare_exchange(
|
||||
&[CultCacheExpectedEnvelope {
|
||||
r#type: IdunnDeploymentBrakeRecord::TYPE.into(),
|
||||
key: IDUNN_DEPLOYMENT_BRAKE_ID.into(),
|
||||
current,
|
||||
}],
|
||||
&[next],
|
||||
)? {
|
||||
bail!("deployment brake changed during transition");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn read_brake(path: &Path) -> Result<IdunnDeploymentBrakeRecord> {
|
||||
let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?;
|
||||
let [entry] = entries.as_slice() else {
|
||||
bail!("deployment brake is missing or ambiguous")
|
||||
};
|
||||
if entry.r#type != IdunnDeploymentBrakeRecord::TYPE
|
||||
|| entry.key != IDUNN_DEPLOYMENT_BRAKE_ID
|
||||
|| entry.schema_id.as_deref() != Some(IDUNN_DEPLOYMENT_BRAKE_SCHEMA)
|
||||
{
|
||||
bail!("deployment brake is foreign")
|
||||
}
|
||||
let record: IdunnDeploymentBrakeRecord = rmp_serde::from_slice(&entry.payload)?;
|
||||
if rmp_serde::to_vec(&record)? != entry.payload {
|
||||
bail!("deployment brake is noncanonical")
|
||||
}
|
||||
record.validate()?;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
fn read_operator_anchor(path: &Path) -> Result<ServiceIdentityTrustAnchor> {
|
||||
let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?;
|
||||
let [entry] = entries.as_slice() else {
|
||||
bail!("operator anchor is missing or ambiguous")
|
||||
};
|
||||
if entry.r#type != IdunnDeploymentBrakeOperatorIdentity::TRUST_ANCHOR_TYPE
|
||||
|| entry.key != IdunnDeploymentBrakeOperatorIdentity::TRUST_ANCHOR_KEY
|
||||
{
|
||||
bail!("operator anchor is foreign")
|
||||
}
|
||||
Ok(rmp_serde::from_slice(&entry.payload)?)
|
||||
}
|
||||
|
||||
fn validate_health_binding_store(path: &Path) -> Result<()> {
|
||||
let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?;
|
||||
validated_health_bindings(&entries)?;
|
||||
|
|
@ -1127,7 +960,7 @@ fn normalized(path: &Path) -> Result<PathBuf> {
|
|||
}
|
||||
|
||||
fn usage() -> &'static str {
|
||||
"Usage: idunn-provision enroll-provider-health-identity --private-store <path>\n idunn-provision provider-health-public-key --private-store <path>\n idunn-provision export-provider-health-public-anchor --private-store <path> --public-anchor <path>\n idunn-provision enroll-idunn-identity --private-store <path>\n idunn-provision export-idunn-public-anchor --private-store <path> --public-anchor <path>\n idunn-provision enroll-odin-topology-identity --private-store <path>\n idunn-provision export-odin-topology-public-anchor --private-store <path> --public-anchor <path>\n idunn-provision enroll-deployment-brake-operator --private-store <path>\n idunn-provision export-deployment-brake-operator-anchor --private-store <path> --public-anchor <path>\n idunn-provision deployment-brake-engage --store <path> --runtime-id <id> --owner <principal> --reason <text> --observed-at-unix-millis <u64>\n idunn-provision deployment-brake-release --store <path> --private-store <path> --runtime-id <id> --owner <principal> --reason <text> --authorization-id <id> --release-id <id> --deployment-id <id> --issued-at-unix-millis <u64> --expires-at-unix-millis <u64>\n idunn-provision deployment-brake-status --store <path> --operator-anchor <path> --runtime-id <id> [--release-id <id> --deployment-id <id> --now-unix-millis <u64>]\n idunn-provision lifecycle-brake-engage --store <path> --runtime-id <id> --target <id> --reason <text> --updated-at-unix-millis <u64>\n idunn-provision lifecycle-brake-release --store <path> --runtime-id <id> --target <id> --reason <text> --updated-at-unix-millis <u64> [--released-until-unix-millis <u64>]\n idunn-provision lifecycle-brake-status --store <path> --runtime-id <id> --target <id> --now-unix-millis <u64>\n idunn-provision create-daemon-health-trust-binding --output <path> --binding-id <id> --daemon <id> --health-contract <id> --source-runtime <id> --signer-public-key-hex <hex> --bound-at-unix-millis <u64> --release-binding-required <true|false>\n idunn-provision add-daemon-health-trust-binding --output <path> --binding-id <id> --daemon <id> --health-contract <id> --source-runtime <id> --signer-public-key-hex <hex> --bound-at-unix-millis <u64> --release-binding-required <true|false>\n idunn-provision require-daemon-health-release-binding --store <path> --binding-id <id> --daemon <id> --health-contract <id> --source-runtime <id> --signer-public-key-hex <hex>\n idunn-provision rotate-daemon-health-trust-signer --output <path> --binding-id <id> --signer-public-key-hex <hex> --bound-at-unix-millis <u64>\n idunn-provision validate-daemon-health-trust-binding --input <path>\n idunn-provision create-provider-projection-trust-anchor --output <path> --trust-anchor-id <id> --runtime-id <id> --idunn-public-anchor <path> --bound-at-unix-millis <u64> --expires-at-unix-millis <u64>\n idunn-provision validate-provider-projection-trust-anchor --input <path> --idunn-public-anchor <path>"
|
||||
"Usage: idunn-provision enroll-provider-health-identity --private-store <path>\n idunn-provision provider-health-public-key --private-store <path>\n idunn-provision export-provider-health-public-anchor --private-store <path> --public-anchor <path>\n idunn-provision enroll-idunn-identity --private-store <path>\n idunn-provision export-idunn-public-anchor --private-store <path> --public-anchor <path>\n idunn-provision enroll-odin-topology-identity --private-store <path>\n idunn-provision export-odin-topology-public-anchor --private-store <path> --public-anchor <path>\n idunn-provision deployment-brake-engage --store <path> --runtime-id <id> --reason <text>\n idunn-provision deployment-brake-clear --store <path> --runtime-id <id> --reason <text>\n idunn-provision deployment-brake-status --store <path> --runtime-id <id> [--owner-user <login>]\n idunn-provision lifecycle-brake-engage --store <path> --runtime-id <id> --target <id> --reason <text> --updated-at-unix-millis <u64>\n idunn-provision lifecycle-brake-release --store <path> --runtime-id <id> --target <id> --reason <text> --updated-at-unix-millis <u64> [--released-until-unix-millis <u64>]\n idunn-provision lifecycle-brake-status --store <path> --runtime-id <id> --target <id> --now-unix-millis <u64>\n idunn-provision create-daemon-health-trust-binding --output <path> --binding-id <id> --daemon <id> --health-contract <id> --source-runtime <id> --signer-public-key-hex <hex> --bound-at-unix-millis <u64> --release-binding-required <true|false>\n idunn-provision add-daemon-health-trust-binding --output <path> --binding-id <id> --daemon <id> --health-contract <id> --source-runtime <id> --signer-public-key-hex <hex> --bound-at-unix-millis <u64> --release-binding-required <true|false>\n idunn-provision require-daemon-health-release-binding --store <path> --binding-id <id> --daemon <id> --health-contract <id> --source-runtime <id> --signer-public-key-hex <hex>\n idunn-provision rotate-daemon-health-trust-signer --output <path> --binding-id <id> --signer-public-key-hex <hex> --bound-at-unix-millis <u64>\n idunn-provision validate-daemon-health-trust-binding --input <path>\n idunn-provision create-provider-projection-trust-anchor --output <path> --trust-anchor-id <id> --runtime-id <id> --idunn-public-anchor <path> --bound-at-unix-millis <u64> --expires-at-unix-millis <u64>\n idunn-provision validate-provider-projection-trust-anchor --input <path> --idunn-public-anchor <path>"
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -2157,120 +1990,74 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn brake_operator_can_issue_one_exact_bounded_rollout_grant() -> Result<()> {
|
||||
fn provision_engage_clear_sequence() -> Result<()> {
|
||||
let temp = TempDir::new()?;
|
||||
let private = temp.path().join("operator-private.cc");
|
||||
let anchor = temp.path().join("operator-public.cc");
|
||||
let store = temp.path().join("brake.cc");
|
||||
invoke(&[
|
||||
"enroll-deployment-brake-operator",
|
||||
"--private-store",
|
||||
private.to_str().unwrap(),
|
||||
])?;
|
||||
invoke(&[
|
||||
"export-deployment-brake-operator-anchor",
|
||||
"--private-store",
|
||||
private.to_str().unwrap(),
|
||||
"--public-anchor",
|
||||
anchor.to_str().unwrap(),
|
||||
])?;
|
||||
invoke(&[
|
||||
"deployment-brake-engage",
|
||||
"--store",
|
||||
store.to_str().unwrap(),
|
||||
"--runtime-id",
|
||||
"yggdrasil",
|
||||
"--owner",
|
||||
"operator/metacrat",
|
||||
"--reason",
|
||||
"sleep mode",
|
||||
"--observed-at-unix-millis",
|
||||
"100",
|
||||
])?;
|
||||
assert_eq!(read_brake(&store)?.status, "engaged");
|
||||
let me = process_uid();
|
||||
let engage = |verb: &str, reason: &str| {
|
||||
invoke(&[
|
||||
verb,
|
||||
"--store",
|
||||
store.to_str().unwrap(),
|
||||
"--runtime-id",
|
||||
"yggdrasil",
|
||||
"--reason",
|
||||
reason,
|
||||
])
|
||||
};
|
||||
assert_eq!(read_stop(&store, "yggdrasil", me), Stop::Absent);
|
||||
invoke(&[
|
||||
"deployment-brake-status",
|
||||
"--store",
|
||||
store.to_str().unwrap(),
|
||||
"--operator-anchor",
|
||||
anchor.to_str().unwrap(),
|
||||
"--runtime-id",
|
||||
"yggdrasil",
|
||||
])?;
|
||||
assert!(
|
||||
engage("deployment-brake-engage", "freeze")?;
|
||||
let status = || {
|
||||
invoke(&[
|
||||
"deployment-brake-status",
|
||||
"--store",
|
||||
store.to_str().unwrap(),
|
||||
"--operator-anchor",
|
||||
anchor.to_str().unwrap(),
|
||||
"--runtime-id",
|
||||
"yggdrasil",
|
||||
"--release-id",
|
||||
"commit-4",
|
||||
"--deployment-id",
|
||||
"request-4",
|
||||
"--now-unix-millis",
|
||||
"150",
|
||||
])
|
||||
.is_err()
|
||||
);
|
||||
invoke(&[
|
||||
"deployment-brake-release",
|
||||
"--store",
|
||||
store.to_str().unwrap(),
|
||||
"--private-store",
|
||||
private.to_str().unwrap(),
|
||||
"--runtime-id",
|
||||
"yggdrasil",
|
||||
"--owner",
|
||||
"operator/metacrat",
|
||||
"--reason",
|
||||
"one rollout",
|
||||
"--authorization-id",
|
||||
"auth/r4",
|
||||
"--release-id",
|
||||
"commit-4",
|
||||
"--deployment-id",
|
||||
"request-4",
|
||||
"--issued-at-unix-millis",
|
||||
"200",
|
||||
"--expires-at-unix-millis",
|
||||
"800",
|
||||
])?;
|
||||
invoke(&[
|
||||
"deployment-brake-status",
|
||||
"--store",
|
||||
store.to_str().unwrap(),
|
||||
"--operator-anchor",
|
||||
anchor.to_str().unwrap(),
|
||||
"--runtime-id",
|
||||
"yggdrasil",
|
||||
"--release-id",
|
||||
"commit-4",
|
||||
"--deployment-id",
|
||||
"request-4",
|
||||
"--now-unix-millis",
|
||||
"500",
|
||||
])?;
|
||||
assert!(
|
||||
invoke(&[
|
||||
"deployment-brake-status",
|
||||
"--store",
|
||||
store.to_str().unwrap(),
|
||||
"--operator-anchor",
|
||||
anchor.to_str().unwrap(),
|
||||
"--runtime-id",
|
||||
"yggdrasil",
|
||||
"--release-id",
|
||||
"substituted",
|
||||
"--deployment-id",
|
||||
"request-4",
|
||||
"--now-unix-millis",
|
||||
"500"
|
||||
])
|
||||
.is_err()
|
||||
);
|
||||
};
|
||||
assert!(status().is_err(), "an engaged stop must not pass status");
|
||||
let Stop::Engaged(first) = read_stop(&store, "yggdrasil", me) else {
|
||||
bail!("engage did not engage")
|
||||
};
|
||||
assert_eq!((first.sequence, first.reason.as_str()), (1, "freeze"));
|
||||
assert!(!first.by.is_empty() && first.at_unix_millis > 0);
|
||||
engage("deployment-brake-clear", "thawed")?;
|
||||
status()?;
|
||||
let Stop::Clear(second) = read_stop(&store, "yggdrasil", me) else {
|
||||
bail!("clear did not clear")
|
||||
};
|
||||
assert_eq!((second.sequence, second.by), (2, first.by));
|
||||
assert!(second.at_unix_millis >= first.at_unix_millis);
|
||||
// The author and the time are not arguments, and the retired
|
||||
// signing arguments are refused.
|
||||
for stray in ["--owner", "--observed-at-unix-millis", "--private-store"] {
|
||||
assert!(
|
||||
invoke(&[
|
||||
"deployment-brake-engage",
|
||||
"--store",
|
||||
store.to_str().unwrap(),
|
||||
"--runtime-id",
|
||||
"yggdrasil",
|
||||
"--reason",
|
||||
"x",
|
||||
stray,
|
||||
"y",
|
||||
])
|
||||
.is_err(),
|
||||
"{stray}"
|
||||
);
|
||||
}
|
||||
for gone in ["deployment-brake-release", "enroll-deployment-brake-operator"] {
|
||||
assert!(invoke(&[gone]).is_err(), "{gone}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -2309,7 +2096,6 @@ mod tests {
|
|||
assert_eq!(engaged.status, "engaged");
|
||||
assert_eq!(engaged.runtime_id, "idunn-yggdrasil");
|
||||
assert_eq!(engaged.target, "ghostlight");
|
||||
assert!(read_brake(&store).is_err());
|
||||
assert!(
|
||||
invoke(&[
|
||||
"lifecycle-brake-status",
|
||||
|
|
@ -2426,57 +2212,4 @@ mod tests {
|
|||
assert_eq!(std::fs::read(&store)?, released_bytes);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn brake_compare_exchange_rejects_stale_transition_snapshot() -> Result<()> {
|
||||
let temp = TempDir::new()?;
|
||||
let store_path = temp.path().join("brake.cc");
|
||||
let base = IdunnDeploymentBrakeRecord {
|
||||
schema_version: IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into(),
|
||||
brake_id: IDUNN_DEPLOYMENT_BRAKE_ID.into(),
|
||||
authority: IDUNN_DEPLOYMENT_BRAKE_AUTHORITY.into(),
|
||||
runtime_id: "yggdrasil".into(),
|
||||
status: "engaged".into(),
|
||||
scope: IDUNN_DEPLOYMENT_BRAKE_SCOPE.into(),
|
||||
reason: "first".into(),
|
||||
observed_at_unix_millis: 100,
|
||||
expires_at_unix_millis: None,
|
||||
authorization_id: None,
|
||||
authorization_purpose: None,
|
||||
authorized_release_id: None,
|
||||
authorized_deployment_id: None,
|
||||
authorized_by: None,
|
||||
authorization_issued_at_unix_millis: None,
|
||||
authorization_expires_at_unix_millis: None,
|
||||
signature_algorithm: None,
|
||||
signature: None,
|
||||
private_state_exposed: false,
|
||||
updated_by: "operator/a".into(),
|
||||
};
|
||||
replace_brake(&store_path, base.clone(), 100)?;
|
||||
let stale = SingleFileMessagePackBackingStore::new(&store_path)
|
||||
.pull_all_read_only_snapshot()?
|
||||
.remove(0);
|
||||
let mut changed = base.clone();
|
||||
changed.reason = "racing writer".into();
|
||||
changed.observed_at_unix_millis = 101;
|
||||
replace_brake(&store_path, changed, 101)?;
|
||||
let candidate = typed_envelope(
|
||||
IDUNN_DEPLOYMENT_BRAKE_ID,
|
||||
&base,
|
||||
IDUNN_DEPLOYMENT_BRAKE_SCHEMA,
|
||||
102,
|
||||
)?;
|
||||
assert!(
|
||||
!SingleFileMessagePackBackingStore::new(&store_path).compare_exchange(
|
||||
&[CultCacheExpectedEnvelope {
|
||||
r#type: IdunnDeploymentBrakeRecord::TYPE.into(),
|
||||
key: IDUNN_DEPLOYMENT_BRAKE_ID.into(),
|
||||
current: Some(stale)
|
||||
}],
|
||||
&[candidate]
|
||||
)?
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
|
|||
805
src/runner_evidence.rs
Normal file
805
src/runner_evidence.rs
Normal file
|
|
@ -0,0 +1,805 @@
|
|||
//! The output tail of a failed runner step.
|
||||
//!
|
||||
//! One file, `runner-evidence.cc`, beside `control.cc`, holds one type: the
|
||||
//! last bytes of a failed runner container's stdout and stderr, one record per
|
||||
//! transaction. The engine's pre-fence failure path is its only writer, and
|
||||
//! `idunn status --command` its only reader. It is never `control.cc`, which
|
||||
//! refuses any type it does not know, and never `incidents.cc`, which is
|
||||
//! published and carries no free text.
|
||||
//!
|
||||
//! The tail is raw runner output. It is kept unredacted and readable by root
|
||||
//! alone (mode 0600): root can already read every secret the runner was given.
|
||||
//! It never enters an error, a transaction record or the journal. The status
|
||||
//! reader is a terminal, so everything it prints from this store goes through
|
||||
//! one allowlist sink (`emit`): printable ASCII passes, all else is escaped.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use anyhow::{Context, Result, bail, ensure};
|
||||
use cultcache_rs::{
|
||||
CultCacheEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore,
|
||||
TryCompareExchangeSnapshotOutcome,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::control_plane::{decode_record, require_id, typed_envelope};
|
||||
use crate::drivers::{
|
||||
LockContended, RUNNER_OUTPUT_TAIL_BYTES, RunnerStepFailure, RunnerSubject,
|
||||
exchange_behind_private_lock, runner_status,
|
||||
};
|
||||
|
||||
const RUNNER_FAILURE_SCHEMA: &str = "idunn.runner_failure.v1";
|
||||
|
||||
/// Records kept; the writer drops the oldest past this, so the store needs no
|
||||
/// other owner.
|
||||
pub(crate) const RUNNER_EVIDENCE_RECORDS: usize = 32;
|
||||
|
||||
/// `runner-evidence.cc`, beside the control store.
|
||||
pub(crate) fn runner_evidence_store_path(state_store: &Path) -> PathBuf {
|
||||
state_store.with_file_name("runner-evidence.cc")
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub(crate) enum SubjectKind {
|
||||
Step,
|
||||
ExternalInput,
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq, Eq, DatabaseEntry)]
|
||||
#[cultcache(type = "idunn.runner_failure", schema = "idunn.runner_failure.v1")]
|
||||
pub(crate) struct RunnerFailureRecord {
|
||||
#[cultcache(key = 0)]
|
||||
pub(crate) schema_version: String,
|
||||
#[cultcache(key = 1)]
|
||||
pub(crate) transaction_id: String,
|
||||
#[cultcache(key = 2)]
|
||||
pub(crate) command_id: String,
|
||||
#[cultcache(key = 3)]
|
||||
pub(crate) target: String,
|
||||
#[cultcache(key = 4)]
|
||||
pub(crate) subject_kind: SubjectKind,
|
||||
#[cultcache(key = 5)]
|
||||
pub(crate) subject_id: String,
|
||||
#[cultcache(key = 6)]
|
||||
pub(crate) runner: String,
|
||||
#[cultcache(key = 7)]
|
||||
pub(crate) exit_code: Option<i32>,
|
||||
#[cultcache(key = 8)]
|
||||
pub(crate) signal: Option<i32>,
|
||||
#[cultcache(key = 9)]
|
||||
pub(crate) stdout_tail: Vec<u8>,
|
||||
#[cultcache(key = 10)]
|
||||
pub(crate) stdout_total_bytes: u64,
|
||||
#[cultcache(key = 11)]
|
||||
pub(crate) stderr_tail: Vec<u8>,
|
||||
#[cultcache(key = 12)]
|
||||
pub(crate) stderr_total_bytes: u64,
|
||||
#[cultcache(key = 13)]
|
||||
pub(crate) recorded_at_unix_millis: u64,
|
||||
}
|
||||
|
||||
/// Tails stay out of `Debug`, so a failed assertion or a logged record never
|
||||
/// prints runner output.
|
||||
impl std::fmt::Debug for RunnerFailureRecord {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("RunnerFailureRecord")
|
||||
.field("transaction_id", &self.transaction_id)
|
||||
.field("subject_id", &self.subject_id)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl RunnerFailureRecord {
|
||||
pub(crate) fn from_failure(
|
||||
transaction_id: &str,
|
||||
command_id: &str,
|
||||
target: &str,
|
||||
failure: &RunnerStepFailure,
|
||||
now: u64,
|
||||
) -> Self {
|
||||
let (subject_kind, subject_id) = match &failure.subject {
|
||||
RunnerSubject::Step(id) => (SubjectKind::Step, id),
|
||||
RunnerSubject::ExternalInput(id) => (SubjectKind::ExternalInput, id),
|
||||
};
|
||||
Self {
|
||||
schema_version: RUNNER_FAILURE_SCHEMA.to_owned(),
|
||||
transaction_id: transaction_id.to_owned(),
|
||||
command_id: command_id.to_owned(),
|
||||
target: target.to_owned(),
|
||||
subject_kind,
|
||||
subject_id: subject_id.clone(),
|
||||
runner: failure.runner.clone(),
|
||||
exit_code: failure.exit_code,
|
||||
signal: failure.signal,
|
||||
stdout_tail: failure.stdout_tail.clone(),
|
||||
stdout_total_bytes: failure.stdout_total_bytes,
|
||||
stderr_tail: failure.stderr_tail.clone(),
|
||||
stderr_total_bytes: failure.stderr_total_bytes,
|
||||
recorded_at_unix_millis: now,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn validate(&self) -> Result<()> {
|
||||
ensure!(
|
||||
self.schema_version == RUNNER_FAILURE_SCHEMA,
|
||||
"runner failure schema is unsupported"
|
||||
);
|
||||
require_id(&self.transaction_id, "runner failure transaction")?;
|
||||
require_id(&self.command_id, "runner failure command")?;
|
||||
require_id(&self.target, "runner failure target")?;
|
||||
require_id(&self.subject_id, "runner failure subject")?;
|
||||
require_id(&self.runner, "runner failure runner")?;
|
||||
for (tail, total, stream) in [
|
||||
(&self.stdout_tail, self.stdout_total_bytes, "stdout"),
|
||||
(&self.stderr_tail, self.stderr_total_bytes, "stderr"),
|
||||
] {
|
||||
ensure!(
|
||||
tail.len() <= RUNNER_OUTPUT_TAIL_BYTES,
|
||||
"runner failure {stream} tail exceeds its bound"
|
||||
);
|
||||
ensure!(
|
||||
tail.len() as u64 <= total,
|
||||
"runner failure {stream} tail exceeds its stream"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn envelope(&self) -> Result<CultCacheEnvelope> {
|
||||
self.validate()?;
|
||||
typed_envelope(
|
||||
&self.transaction_id,
|
||||
Self::TYPE,
|
||||
RUNNER_FAILURE_SCHEMA,
|
||||
self,
|
||||
self.recorded_at_unix_millis,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// One read of the store: the envelopes exactly as stored, and the records
|
||||
/// decoded from them in the same order.
|
||||
fn snapshot(path: &Path) -> Result<(Vec<CultCacheEnvelope>, Vec<RunnerFailureRecord>)> {
|
||||
let envelopes = SingleFileMessagePackBackingStore::new(path)
|
||||
.pull_all_read_only_snapshot()
|
||||
.context("reading the runner evidence store")?;
|
||||
let mut records = Vec::new();
|
||||
for envelope in &envelopes {
|
||||
ensure!(
|
||||
envelope.r#type == RunnerFailureRecord::TYPE
|
||||
&& envelope.schema_id.as_deref() == Some(RUNNER_FAILURE_SCHEMA),
|
||||
"the runner evidence store holds a document that is not a runner failure"
|
||||
);
|
||||
let record: RunnerFailureRecord = decode_record(envelope)?;
|
||||
record.validate()?;
|
||||
ensure!(
|
||||
envelope.key == record.transaction_id,
|
||||
"runner evidence store key differs from the record's transaction"
|
||||
);
|
||||
records.push(record);
|
||||
}
|
||||
Ok((envelopes, records))
|
||||
}
|
||||
|
||||
/// How a write left the store.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum Kept {
|
||||
Stored,
|
||||
/// The store held bytes that do not decode; it was moved to
|
||||
/// `unreadable_store_path` and the record opened a fresh one. `replaced`
|
||||
/// is the last-written time (unix milliseconds) of an earlier set-aside
|
||||
/// store this one displaced, which is gone: one slot, and it says so.
|
||||
SetAside { replaced: Option<u64> },
|
||||
}
|
||||
|
||||
/// Where an undecodable store is kept. One slot: the next one replaces it, so
|
||||
/// the damage a bad store can cost is one extra file.
|
||||
pub(crate) fn unreadable_store_path(path: &Path) -> PathBuf {
|
||||
path.with_file_name("runner-evidence.unreadable.cc")
|
||||
}
|
||||
|
||||
/// Moves the store over the previous set-aside one, root-only, and reports
|
||||
/// what it displaced. Nothing of the store's content is named: a decode error
|
||||
/// can quote stored values, and none is carried here.
|
||||
fn set_aside_unreadable(path: &Path) -> Result<Option<u64>> {
|
||||
let aside = unreadable_store_path(path);
|
||||
let replaced = std::fs::symlink_metadata(&aside).ok().map(|earlier| {
|
||||
earlier
|
||||
.modified()
|
||||
.ok()
|
||||
.and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok())
|
||||
.map_or(0, |since| since.as_millis() as u64)
|
||||
});
|
||||
std::fs::rename(path, &aside)
|
||||
.context("the runner evidence store is undecodable and could not be set aside")?;
|
||||
keep_private(&aside)?;
|
||||
Ok(replaced)
|
||||
}
|
||||
|
||||
/// How the writer reads the store's bytes; tests inject an I/O error here.
|
||||
type ReadStore<'a> = &'a dyn Fn(&Path) -> std::io::Result<Vec<u8>>;
|
||||
|
||||
fn unreadable_fault(error: &std::io::Error) -> anyhow::Error {
|
||||
anyhow::anyhow!("the runner evidence store could not be read: {}", error.kind())
|
||||
}
|
||||
|
||||
/// The one writer. Upsert by transaction id; past `RUNNER_EVIDENCE_RECORDS` the
|
||||
/// oldest records go in the same exchange. The file ends at mode 0600.
|
||||
///
|
||||
/// A store is set aside only when bytes that were read do not decode. An I/O
|
||||
/// error leaves the store in place, keeps no evidence for that failure, and
|
||||
/// the error names its kind for the journal.
|
||||
pub(crate) fn record(path: &Path, record: &RunnerFailureRecord) -> Result<Kept> {
|
||||
record_reading(path, record, &|path| std::fs::read(path))
|
||||
}
|
||||
|
||||
fn record_reading(path: &Path, record: &RunnerFailureRecord, read: ReadStore) -> Result<Kept> {
|
||||
let first = match read(path) {
|
||||
Ok(bytes) => Some(bytes),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
|
||||
Err(error) => return Err(unreadable_fault(&error)),
|
||||
};
|
||||
let (envelopes, records, how) = match snapshot(path) {
|
||||
Ok((envelopes, records)) => (envelopes, records, Kept::Stored),
|
||||
Err(_) => {
|
||||
let Some(first) = first else {
|
||||
bail!("the runner evidence store could not be decoded");
|
||||
};
|
||||
// Undecodable only if the same bytes are still there to read: a
|
||||
// second read that fails, or finds other bytes, is not a verdict.
|
||||
match read(path) {
|
||||
Ok(second) if second == first => {}
|
||||
Ok(_) => bail!("the runner evidence store changed while it was being read"),
|
||||
Err(error) => return Err(unreadable_fault(&error)),
|
||||
}
|
||||
let replaced = set_aside_unreadable(path)?;
|
||||
(Vec::new(), Vec::new(), Kept::SetAside { replaced })
|
||||
}
|
||||
};
|
||||
let mut kept: Vec<(u64, CultCacheEnvelope)> = envelopes
|
||||
.iter()
|
||||
.zip(&records)
|
||||
.filter(|(_, stored)| stored.transaction_id != record.transaction_id)
|
||||
.map(|(envelope, stored)| (stored.recorded_at_unix_millis, envelope.clone()))
|
||||
.collect();
|
||||
kept.push((record.recorded_at_unix_millis, record.envelope()?));
|
||||
// Stable: records of one instant keep their stored order, so the record
|
||||
// being written survives a tie.
|
||||
kept.sort_by_key(|(recorded_at, _)| *recorded_at);
|
||||
let excess = kept.len().saturating_sub(RUNNER_EVIDENCE_RECORDS);
|
||||
let replacements: Vec<_> = kept
|
||||
.into_iter()
|
||||
.skip(excess)
|
||||
.map(|(_, envelope)| envelope)
|
||||
.collect();
|
||||
match exchange_behind_private_lock(path, &envelopes, &replacements)? {
|
||||
TryCompareExchangeSnapshotOutcome::Exchanged => {}
|
||||
TryCompareExchangeSnapshotOutcome::Mismatch => {
|
||||
bail!("the runner evidence store changed while it was being written")
|
||||
}
|
||||
TryCompareExchangeSnapshotOutcome::LockContended => {
|
||||
return Err(LockContended {
|
||||
store: "runner evidence store",
|
||||
}
|
||||
.into());
|
||||
}
|
||||
}
|
||||
keep_private(path)?;
|
||||
Ok(how)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn keep_private(path: &Path) -> Result<()> {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600))
|
||||
.context("making the runner evidence store private")
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn keep_private(_path: &Path) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The one reader. An absent store holds nothing.
|
||||
pub(crate) fn for_transaction(
|
||||
path: &Path,
|
||||
transaction_id: &str,
|
||||
) -> Result<Option<RunnerFailureRecord>> {
|
||||
Ok(snapshot(path)?
|
||||
.1
|
||||
.into_iter()
|
||||
.find(|stored| stored.transaction_id == transaction_id))
|
||||
}
|
||||
|
||||
/// The record as `idunn status --command` shows it, as raw lines. They are
|
||||
/// not text yet: `show_status` is the only caller, and `emit` the only
|
||||
/// writer.
|
||||
fn render(record: &RunnerFailureRecord) -> Vec<Vec<u8>> {
|
||||
let noun = match record.subject_kind {
|
||||
SubjectKind::Step => "step",
|
||||
SubjectKind::ExternalInput => "external input",
|
||||
};
|
||||
let mut lines: Vec<Vec<u8>> = vec![
|
||||
format!(" failed {noun} {}", record.subject_id).into_bytes(),
|
||||
format!(" runner {}", record.runner).into_bytes(),
|
||||
format!(" status {}", runner_status(record.exit_code, record.signal)).into_bytes(),
|
||||
];
|
||||
for (stream, tail, total) in [
|
||||
("stdout", &record.stdout_tail, record.stdout_total_bytes),
|
||||
("stderr", &record.stderr_tail, record.stderr_total_bytes),
|
||||
] {
|
||||
lines.push(format!(" {stream} (last {} of {total} bytes)", tail.len()).into_bytes());
|
||||
let text = tail.strip_suffix(b"\n").unwrap_or(tail);
|
||||
if !text.is_empty() {
|
||||
lines.extend(text.split(|byte| *byte == b'\n').map(|line| {
|
||||
let mut shown = b" | ".to_vec();
|
||||
shown.extend_from_slice(line);
|
||||
shown
|
||||
}));
|
||||
}
|
||||
}
|
||||
lines
|
||||
}
|
||||
|
||||
/// What `idunn status --command` prints for a transaction's runner evidence:
|
||||
/// its record, or one line naming why the store is unreadable. Nothing is
|
||||
/// written when the transaction has no record.
|
||||
pub(crate) fn show_status(
|
||||
out: &mut impl std::io::Write,
|
||||
store: &Path,
|
||||
transaction_id: &str,
|
||||
) -> std::io::Result<()> {
|
||||
let lines = match for_transaction(store, transaction_id) {
|
||||
Ok(Some(record)) => render(&record),
|
||||
Ok(None) => return Ok(()),
|
||||
Err(error) => {
|
||||
vec![format!(" runner evidence unreadable: {}", error.root_cause()).into_bytes()]
|
||||
}
|
||||
};
|
||||
emit(out, &lines)
|
||||
}
|
||||
|
||||
/// The one sink: every byte the status read prints from the evidence store,
|
||||
/// or from an error about it, is written here and nowhere else. An allowlist:
|
||||
/// printable ASCII and tab pass; every other code point is a visible
|
||||
/// `\u{..}` escape and every invalid UTF-8 byte a `\x..` escape. A newline
|
||||
/// inside a line is escaped too: this function writes the only ones.
|
||||
pub(crate) fn emit(out: &mut impl std::io::Write, lines: &[Vec<u8>]) -> std::io::Result<()> {
|
||||
for line in lines {
|
||||
out.write_all(escape(line).as_bytes())?;
|
||||
out.write_all(b"\n")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn escape(bytes: &[u8]) -> String {
|
||||
let mut shown = String::with_capacity(bytes.len());
|
||||
for chunk in bytes.utf8_chunks() {
|
||||
for character in chunk.valid().chars() {
|
||||
match character {
|
||||
' '..='~' | '\t' => shown.push(character),
|
||||
_ => shown.push_str(&format!("\\u{{{:x}}}", u32::from(character))),
|
||||
}
|
||||
}
|
||||
for byte in chunk.invalid() {
|
||||
shown.push_str(&format!("\\x{byte:02x}"));
|
||||
}
|
||||
}
|
||||
shown
|
||||
}
|
||||
|
||||
#[cfg(all(test, unix))]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
fn failure(subject: RunnerSubject, stdout: &[u8], stderr: &[u8]) -> RunnerStepFailure {
|
||||
RunnerStepFailure {
|
||||
subject,
|
||||
runner: "rust".into(),
|
||||
exit_code: Some(3),
|
||||
signal: None,
|
||||
stdout_tail: stdout.to_vec(),
|
||||
stdout_total_bytes: stdout.len() as u64,
|
||||
stderr_tail: stderr.to_vec(),
|
||||
stderr_total_bytes: stderr.len() as u64,
|
||||
}
|
||||
}
|
||||
|
||||
/// What status writes for a record, through the one sink.
|
||||
fn shown(record: &RunnerFailureRecord) -> String {
|
||||
let mut written = Vec::new();
|
||||
emit(&mut written, &render(record)).expect("a Vec takes any write");
|
||||
String::from_utf8(written).expect("the sink writes ASCII only")
|
||||
}
|
||||
|
||||
fn shown_lines(record: &RunnerFailureRecord) -> Vec<String> {
|
||||
shown(record).lines().map(str::to_owned).collect()
|
||||
}
|
||||
|
||||
fn record_at(transaction: &str, now: u64) -> RunnerFailureRecord {
|
||||
RunnerFailureRecord::from_failure(
|
||||
transaction,
|
||||
"up-service",
|
||||
"service",
|
||||
&failure(RunnerSubject::Step("build".into()), b"out\n", b"err\n"),
|
||||
now,
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_evidence_store_keeps_the_newest_32() -> Result<()> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let path = runner_evidence_store_path(&temp.path().join("control.cc"));
|
||||
// The literal, not the constant: the spec's bound is 32.
|
||||
for index in 0..=32 {
|
||||
record(&path, &record_at(&format!("tx-{index:02}"), 1000 + index as u64))?;
|
||||
}
|
||||
let kept: Vec<_> = snapshot(&path)?.1.into_iter().map(|stored| stored.transaction_id).collect();
|
||||
assert_eq!(kept.len(), 32);
|
||||
assert!(!kept.contains(&"tx-00".to_owned()), "the oldest survived");
|
||||
assert!(kept.contains(&"tx-32".to_owned()), "the newest was evicted");
|
||||
assert!(kept.contains(&"tx-01".to_owned()), "the second oldest was evicted early");
|
||||
// Rewriting a resident transaction replaces it and drops nothing.
|
||||
let mut again = record_at("tx-05", 5000);
|
||||
again.stdout_tail = b"rewritten\n".to_vec();
|
||||
again.stdout_total_bytes = 10;
|
||||
record(&path, &again)?;
|
||||
let kept = snapshot(&path)?.1;
|
||||
assert_eq!(kept.len(), 32);
|
||||
assert!(kept.iter().any(|stored| stored.transaction_id == "tx-01"));
|
||||
let stored = for_transaction(&path, "tx-05")?.context("rewritten record is gone")?;
|
||||
assert_eq!(stored.stdout_tail, b"rewritten\n");
|
||||
assert_eq!(for_transaction(&path, "tx-00")?, None);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_record_round_trips_byte_for_byte() -> Result<()> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let path = runner_evidence_store_path(&temp.path().join("control.cc"));
|
||||
let written = RunnerFailureRecord::from_failure(
|
||||
"tx-a",
|
||||
"up-service",
|
||||
"service",
|
||||
&failure(RunnerSubject::ExternalInput("toolchain".into()), &[0, 1, 255, 10], b""),
|
||||
77,
|
||||
);
|
||||
record(&path, &written)?;
|
||||
assert_eq!(for_transaction(&path, "tx-a")?, Some(written));
|
||||
assert_eq!(for_transaction(&path, "tx-b")?, None);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_evidence_store_is_private() -> Result<()> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let path = runner_evidence_store_path(&temp.path().join("control.cc"));
|
||||
// Whatever the umask, the mode is the store's own rule, on every write.
|
||||
for index in 0..2 {
|
||||
record(&path, &record_at(&format!("tx-{index}"), 10 + index))?;
|
||||
assert_eq!(std::fs::metadata(&path)?.permissions().mode() & 0o777, 0o600);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_tail_over_its_bound_or_its_stream_is_refused() {
|
||||
let mut oversize = record_at("tx-a", 1);
|
||||
oversize.stdout_tail = vec![b'x'; RUNNER_OUTPUT_TAIL_BYTES + 1];
|
||||
oversize.stdout_total_bytes = u64::MAX;
|
||||
assert!(oversize.validate().is_err());
|
||||
let mut impossible = record_at("tx-a", 1);
|
||||
impossible.stderr_total_bytes = 1;
|
||||
assert!(impossible.validate().is_err());
|
||||
let mut exact = record_at("tx-a", 1);
|
||||
exact.stdout_tail = vec![b'x'; RUNNER_OUTPUT_TAIL_BYTES];
|
||||
exact.stdout_total_bytes = RUNNER_OUTPUT_TAIL_BYTES as u64;
|
||||
assert!(exact.validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn status_renders_the_tail_with_control_bytes_escaped() {
|
||||
let mut stored = record_at("tx-a", 1);
|
||||
stored.stdout_tail = b"first\n\x1b[2Jsecond\x00end\tTab\nlast\n".to_vec();
|
||||
stored.stdout_total_bytes = 5000;
|
||||
stored.stderr_tail = b"bad\r\x07".to_vec();
|
||||
stored.stderr_total_bytes = 6;
|
||||
let lines = shown_lines(&stored);
|
||||
let text = lines.join("\n");
|
||||
assert!(!text.contains('\u{1b}') && !text.contains('\0') && !text.contains('\r'));
|
||||
assert!(!text.contains('\u{7}'));
|
||||
assert_eq!(
|
||||
lines,
|
||||
vec![
|
||||
" failed step build",
|
||||
" runner rust",
|
||||
" status exit status 3",
|
||||
" stdout (last 30 of 5000 bytes)",
|
||||
" | first",
|
||||
" | \\u{1b}[2Jsecond\\u{0}end\tTab",
|
||||
" | last",
|
||||
" stderr (last 5 of 6 bytes)",
|
||||
" | bad\\u{d}\\u{7}",
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn debug_names_the_record_and_prints_no_tail() {
|
||||
let mut stored = record_at("tx-a", 1);
|
||||
stored.stdout_tail = b"SECRET-OUTPUT".to_vec();
|
||||
stored.stdout_total_bytes = 13;
|
||||
let debug = format!("{stored:?}");
|
||||
assert!(debug.contains("tx-a") && !debug.contains("SECRET"), "{debug}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_tail_renders_only_its_header() {
|
||||
let mut stored = record_at("tx-a", 1);
|
||||
stored.stdout_tail = Vec::new();
|
||||
stored.stdout_total_bytes = 0;
|
||||
let lines = shown_lines(&stored);
|
||||
assert!(lines.contains(&" stdout (last 0 of 0 bytes)".to_owned()));
|
||||
assert!(!lines.iter().any(|line| line == " | "));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_reader_names_an_unreadable_store_as_an_error() -> Result<()> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let path = runner_evidence_store_path(&temp.path().join("control.cc"));
|
||||
std::fs::write(&path, b"\xc1 not a store")?;
|
||||
assert!(for_transaction(&path, "tx-a").is_err());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn files_beside(path: &Path) -> Result<Vec<String>> {
|
||||
let mut names: Vec<String> = std::fs::read_dir(path.parent().context("no parent")?)?
|
||||
.map(|entry| Ok(entry?.file_name().to_string_lossy().into_owned()))
|
||||
.collect::<Result<_>>()?;
|
||||
names.retain(|name| name.starts_with("runner-evidence") && !name.contains("lock"));
|
||||
names.sort();
|
||||
Ok(names)
|
||||
}
|
||||
|
||||
/// A store a newer Idunn wrote: well-formed, but a schema this one does not know.
|
||||
fn write_future_store(path: &Path) -> Result<()> {
|
||||
let mut future = record_at("tx-future", 1);
|
||||
future.schema_version = "idunn.runner_failure.v2".into();
|
||||
let envelope = typed_envelope(
|
||||
&future.transaction_id,
|
||||
RunnerFailureRecord::TYPE,
|
||||
"idunn.runner_failure.v2",
|
||||
&future,
|
||||
1,
|
||||
)?;
|
||||
let outcome = exchange_behind_private_lock(path, &[], &[envelope])?;
|
||||
ensure!(
|
||||
matches!(outcome, TryCompareExchangeSnapshotOutcome::Exchanged),
|
||||
"the future store was not written"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_corrupt_store_is_set_aside_and_the_next_failure_keeps_its_evidence() -> Result<()> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let path = runner_evidence_store_path(&temp.path().join("control.cc"));
|
||||
let damage = b"\xc1 not a store \x00 damage";
|
||||
std::fs::write(&path, damage)?;
|
||||
let kept = record(&path, &record_at("tx-a", 5))?;
|
||||
assert_eq!(kept, Kept::SetAside { replaced: None });
|
||||
// Kept, not deleted: the damage is where the journal says, root-only.
|
||||
let aside = unreadable_store_path(&path);
|
||||
assert_eq!(std::fs::read(&aside)?, damage);
|
||||
assert_eq!(std::fs::metadata(&aside)?.permissions().mode() & 0o777, 0o600);
|
||||
// The failure's evidence is in a fresh, private store.
|
||||
assert_eq!(
|
||||
for_transaction(&path, "tx-a")?.map(|stored| stored.transaction_id),
|
||||
Some("tx-a".to_owned())
|
||||
);
|
||||
assert_eq!(std::fs::metadata(&path)?.permissions().mode() & 0o777, 0o600);
|
||||
// A healthy store is not set aside again.
|
||||
assert_eq!(record(&path, &record_at("tx-b", 6))?, Kept::Stored);
|
||||
assert_eq!(std::fs::read(&aside)?, damage);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_future_version_store_is_set_aside_and_the_next_failure_keeps_its_evidence() -> Result<()> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let path = runner_evidence_store_path(&temp.path().join("control.cc"));
|
||||
write_future_store(&path)?;
|
||||
let before = std::fs::read(&path)?;
|
||||
assert!(for_transaction(&path, "tx-future").is_err());
|
||||
assert_eq!(
|
||||
record(&path, &record_at("tx-a", 5))?,
|
||||
Kept::SetAside { replaced: None }
|
||||
);
|
||||
assert_eq!(std::fs::read(unreadable_store_path(&path))?, before);
|
||||
assert!(for_transaction(&path, "tx-a")?.is_some());
|
||||
assert_eq!(for_transaction(&path, "tx-future")?, None);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn at_most_one_unreadable_store_is_kept() -> Result<()> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let path = runner_evidence_store_path(&temp.path().join("control.cc"));
|
||||
let damages: [&[u8]; 3] = [b"\xc1 first", b"\xc1 second", b"\xc1 third"];
|
||||
for (index, damage) in damages.into_iter().enumerate() {
|
||||
std::fs::write(&path, damage)?;
|
||||
record(&path, &record_at(&format!("tx-{index}"), 5 + index as u64))?;
|
||||
assert_eq!(
|
||||
std::fs::read(unreadable_store_path(&path))?,
|
||||
damage,
|
||||
"the newest damage is the one kept"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
files_beside(&path)?,
|
||||
vec![
|
||||
"runner-evidence.cc".to_owned(),
|
||||
"runner-evidence.unreadable.cc".to_owned()
|
||||
]
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_store_that_cannot_be_read_is_left_in_place_and_the_write_fails() -> Result<()> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let path = runner_evidence_store_path(&temp.path().join("control.cc"));
|
||||
// A directory where the store should be: unreadable, not undecodable.
|
||||
std::fs::create_dir(&path)?;
|
||||
assert!(record(&path, &record_at("tx-a", 1)).is_err());
|
||||
assert!(path.is_dir());
|
||||
assert!(!unreadable_store_path(&path).exists());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_code_point_but_printable_ascii_is_a_visible_escape() {
|
||||
// The allowlist, exhaustively: no class list decides anything.
|
||||
for code in 0..=0x10ffffu32 {
|
||||
let Some(character) = char::from_u32(code) else {
|
||||
continue;
|
||||
};
|
||||
let text = character.to_string();
|
||||
let shown = escape(text.as_bytes());
|
||||
if matches!(character, ' '..='~' | '\t') {
|
||||
assert_eq!(shown, text, "U+{code:04X} should pass");
|
||||
} else {
|
||||
assert_eq!(shown, format!("\\u{{{code:x}}}"), "U+{code:04X}");
|
||||
}
|
||||
}
|
||||
// Invalid UTF-8 is escaped byte by byte; valid text round it passes.
|
||||
assert_eq!(escape(b"a\xc1\xffb\xe2\x82"), "a\\xc1\\xffb\\xe2\\x82");
|
||||
assert_eq!(escape(b"ok \xe2\x82\xac"), "ok \\u{20ac}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_newline_inside_a_line_cannot_forge_a_line() {
|
||||
let mut stored = record_at("tx-a", 1);
|
||||
stored.subject_id = "build\n status exit status 0".into();
|
||||
let lines = shown_lines(&stored);
|
||||
assert_eq!(lines[0], " failed step build\\u{a} status exit status 0");
|
||||
assert_eq!(
|
||||
lines.iter().filter(|line| line.starts_with(" status ")).count(),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
/// A store whose first record is well-formed except that its subject kind
|
||||
/// holds `marker`: the decode error quotes it.
|
||||
fn store_with_hostile_subject_kind(path: &Path, marker: &[u8; 4]) -> Result<()> {
|
||||
record(path, &record_at("tx-a", 1))?;
|
||||
let mut bytes = std::fs::read(path)?;
|
||||
let at = bytes
|
||||
.windows(4)
|
||||
.position(|window| window == b"step")
|
||||
.context("no step marker")?;
|
||||
bytes[at..at + 4].copy_from_slice(marker);
|
||||
std::fs::write(path, bytes)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_unreadable_store_line_is_escaped_with_the_rest() -> Result<()> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let path = runner_evidence_store_path(&temp.path().join("control.cc"));
|
||||
store_with_hostile_subject_kind(&path, b"\x1b[2J")?;
|
||||
let mut written = Vec::new();
|
||||
show_status(&mut written, &path, "tx-a")?;
|
||||
assert!(!written.contains(&0x1b), "an escape byte reached the terminal");
|
||||
assert!(written.iter().all(|byte| matches!(byte, b' '..=b'~' | b'\t' | b'\n')));
|
||||
let text = String::from_utf8(written)?;
|
||||
assert!(text.starts_with(" runner evidence unreadable: "), "{text}");
|
||||
assert!(text.contains("\\u{1b}[2J"), "the quoted value was not shown escaped: {text}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failed_set_aside_names_itself_keeps_the_store_and_echoes_nothing() -> Result<()> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let path = runner_evidence_store_path(&temp.path().join("control.cc"));
|
||||
store_with_hostile_subject_kind(&path, b"CNRY")?;
|
||||
let damage = std::fs::read(&path)?;
|
||||
// Sanity: this damage decodes to an error that quotes the canary.
|
||||
assert!(format!("{:#}", snapshot(&path).unwrap_err()).contains("CNRY"));
|
||||
// A non-empty directory where the aside goes: rename fails even as root.
|
||||
let aside = unreadable_store_path(&path);
|
||||
std::fs::create_dir(&aside)?;
|
||||
std::fs::write(aside.join("occupied"), b"x")?;
|
||||
let error = record(&path, &record_at("tx-b", 2)).unwrap_err();
|
||||
for text in [format!("{error:#}"), format!("{error:?}")] {
|
||||
assert!(text.contains("could not be set aside"), "{text}");
|
||||
assert!(!text.contains("CNRY"), "stored content reached the error: {text}");
|
||||
}
|
||||
// The failure cost nothing: the store is as it was, with no evidence
|
||||
// written over it.
|
||||
assert_eq!(std::fs::read(&path)?, damage);
|
||||
assert!(aside.join("occupied").exists());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_io_error_leaves_the_store_in_place_and_keeps_no_evidence() -> Result<()> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let path = runner_evidence_store_path(&temp.path().join("control.cc"));
|
||||
record(&path, &record_at("tx-a", 1))?;
|
||||
let before = std::fs::read(&path)?;
|
||||
let denied = || std::io::Error::from(std::io::ErrorKind::PermissionDenied);
|
||||
// The first read fails on a healthy store.
|
||||
let error = record_reading(&path, &record_at("tx-b", 2), &|_| Err(denied())).unwrap_err();
|
||||
assert!(format!("{error:#}").contains("permission denied"), "{error:#}");
|
||||
assert_eq!(std::fs::read(&path)?, before, "the store changed");
|
||||
assert!(!unreadable_store_path(&path).exists());
|
||||
assert!(for_transaction(&path, "tx-b")?.is_none());
|
||||
// The second read, the one that confirms the bytes do not decode, fails.
|
||||
std::fs::write(&path, b"\xc1 not a store")?;
|
||||
let calls = std::cell::Cell::new(0usize);
|
||||
let read = |path: &Path| {
|
||||
calls.set(calls.get() + 1);
|
||||
if calls.get() == 2 {
|
||||
return Err(denied());
|
||||
}
|
||||
std::fs::read(path)
|
||||
};
|
||||
let error = record_reading(&path, &record_at("tx-b", 2), &read).unwrap_err();
|
||||
assert!(format!("{error:#}").contains("permission denied"), "{error:#}");
|
||||
assert_eq!(std::fs::read(&path)?, b"\xc1 not a store");
|
||||
assert!(!unreadable_store_path(&path).exists());
|
||||
// Bytes that changed between the reads are no verdict either.
|
||||
let calls = std::cell::Cell::new(0usize);
|
||||
let read = |path: &Path| -> std::io::Result<Vec<u8>> {
|
||||
calls.set(calls.get() + 1);
|
||||
let mut bytes = std::fs::read(path)?;
|
||||
bytes.push(calls.get() as u8);
|
||||
Ok(bytes)
|
||||
};
|
||||
assert!(record_reading(&path, &record_at("tx-b", 2), &read).is_err());
|
||||
assert_eq!(std::fs::read(&path)?, b"\xc1 not a store");
|
||||
assert!(!unreadable_store_path(&path).exists());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_second_set_aside_says_it_replaced_the_first() -> Result<()> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let path = runner_evidence_store_path(&temp.path().join("control.cc"));
|
||||
std::fs::write(&path, b"\xc1 first")?;
|
||||
assert_eq!(
|
||||
record(&path, &record_at("tx-a", 1))?,
|
||||
Kept::SetAside { replaced: None }
|
||||
);
|
||||
std::fs::write(&path, b"\xc1 second")?;
|
||||
let Kept::SetAside { replaced: Some(_) } = record(&path, &record_at("tx-b", 2))? else {
|
||||
bail!("a displaced set-aside store went unreported");
|
||||
};
|
||||
assert_eq!(std::fs::read(unreadable_store_path(&path))?, b"\xc1 second");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
4086
src/steward_orders.rs
Normal file
4086
src/steward_orders.rs
Normal file
File diff suppressed because it is too large
Load diff
16
tests/fixtures/README.md
vendored
16
tests/fixtures/README.md
vendored
|
|
@ -10,3 +10,19 @@ It is checked in because the TypeScript runtime emits a camelCase MessagePack
|
|||
map while Rust `DatabaseEntry` derives tuple payloads. Idunn must prove it can
|
||||
decode the producer's real wire shape rather than round-tripping a fixture it
|
||||
serialized itself.
|
||||
|
||||
# systemd 257 `systemctl show` captures
|
||||
|
||||
`systemd-257/*.show` are the verbatim output of systemd 257.13 for the 27
|
||||
properties `show_unit` requested at `112a7a5`, captured read-only on Yggdrasil
|
||||
on 2026-10-24: `vili-presence-identity-only.show` (a failed idunn-vili unit whose
|
||||
only credential is the presence identity) and
|
||||
`odin-presence-and-one-service-credential.show` (the live Odin unit). systemd 257
|
||||
prints `LoadCredential=[unprintable]` in both. Tests read these bytes; do not edit.
|
||||
|
||||
# Pre-cut systemd observation records
|
||||
|
||||
`pre-cut-systemd-observation.{compact,named}.hex` are the two MessagePack
|
||||
encodings of a `SystemdWorkloadObservation` with `load_credential` set to
|
||||
`[unprintable]`, produced by the code at `112a7a5` (the form live incumbents
|
||||
persisted). They pin that later Idunns still decode such records.
|
||||
|
|
|
|||
62
tests/fixtures/idunn-control-legacy/README.md
vendored
Normal file
62
tests/fixtures/idunn-control-legacy/README.md
vendored
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
# Legacy control-record wire fixtures
|
||||
|
||||
Each `*.hex` file is two lines: the CultCache envelope `schema_id`, then the
|
||||
record's MessagePack payload in hex. They are the exact bytes the encoder at
|
||||
Idunn `9001b58` (the base of the F0 cut) wrote for records built through that
|
||||
revision's own constructors: `DeploymentTransaction::new`,
|
||||
`AdmittedGeneration::from_transaction`, `SealedRelease::new`,
|
||||
`compile_deployment_plan`, and `rmp_serde::to_vec`. A throwaway test on a
|
||||
scratch commit over `9001b58` printed them; it is not kept.
|
||||
|
||||
Everything in them is synthetic: identities were enrolled fresh in a temporary
|
||||
directory, the Odin receipts are placeholder bytes with correct digests, and
|
||||
the plan is the `deployment_plan` test recipe. No byte comes from a live store.
|
||||
|
||||
| File | Schema | What it is |
|
||||
|---|---|---|
|
||||
| `generation-with-receipts` | `idunn.admitted_generation.v2` | routed, stateless generation with Ready and latest Odin receipts |
|
||||
| `generation-route-repair-started` | `idunn.admitted_generation.v2` | the same with `route_repair_started_at_unix_millis` set |
|
||||
| `transaction-fencing` .. `transaction-committing` | `idunn.deployment_transaction.v3` | one continuity transaction in each post-fencing phase |
|
||||
| `transaction-committing-v2` | `idunn.deployment_transaction.v2` | the Committing record in the v2 layout: the v3 tuple without key 34 (base code stops writing v2, so this one is the v3 encoding with its trailing nil slot dropped and the header count lowered) |
|
||||
| `transaction-failed-after-fencing` | `idunn.deployment_transaction.v3` | terminal post-fencing failure with complete abort evidence |
|
||||
|
||||
`control_plane.rs` decodes these through `LegacyDeploymentTransaction` and
|
||||
`LegacyAdmittedGeneration`. The fixtures die with those structs.
|
||||
|
||||
## Pre-B1 continuity aborts
|
||||
|
||||
The `transaction-*-abort*` files and `provider-anchor` were printed by a second
|
||||
throwaway test over `9001b58`. It built a continuity transaction that issued an
|
||||
activation (the recipe and binding of the `deployment_plan` tests, a provider
|
||||
identity enrolled fresh, the trust-anchor path in the binding fixed at
|
||||
`/tmp/idunn-b1-legacy/provider-anchor.cc`) and gave it the aborts the base code
|
||||
writes: the pre-fence abort literal of `begin_pre_fencing_abort`, and the base
|
||||
`post_fencing_abort_intent`. In each, the transaction issued an activation and
|
||||
its abort records `topology_reconciliation = Skipped`, because the base rule
|
||||
owed a continuity no projection cleanup. The workload and Odin evidence are
|
||||
borrowed from `transaction-fencing`.
|
||||
|
||||
| File | What it is |
|
||||
|---|---|
|
||||
| `transaction-pre-fence-abort` | Warming continuity, pre-fencing abort in flight |
|
||||
| `transaction-pre-fence-abort-terminal` | the same abort finished: Complete, `FailedBeforeFencing` |
|
||||
| `transaction-post-fence-abort` | Fencing continuity, post-fencing abort in flight |
|
||||
| `provider-anchor` | one line: the provider trust-anchor file bytes, hex. A test writes it to the fixed path so the Engine can read the plan's anchor |
|
||||
|
||||
## Generation v3 (the B2 lift)
|
||||
|
||||
`generation-v3-odin` and `generation-v3-route-proof` are the exact
|
||||
`idunn.admitted_generation.v3` payloads the encoder at Idunn `583b2a7` wrote
|
||||
(`admitted_envelope` over `rmp_serde::to_vec`), printed by a throwaway test that
|
||||
is not kept. The first is the v2 `generation-with-receipts` record lifted by that
|
||||
revision's own `LegacyAdmittedGeneration::into_current`, the second is the
|
||||
generation `route_proof::committed_generation` admits through a real route-proof
|
||||
deployment. Both are synthetic.
|
||||
|
||||
| File | Schema | What it is |
|
||||
|---|---|---|
|
||||
| `generation-v3-odin` | `idunn.admitted_generation.v3` | routed, Odin-correlated generation |
|
||||
| `generation-v3-route-proof` | `idunn.admitted_generation.v3` | routed, route-proof generation, no Odin receipts |
|
||||
|
||||
`control_plane.rs` decodes them through `LegacyAdmittedGenerationV3`; the
|
||||
fixtures die with that struct.
|
||||
2
tests/fixtures/idunn-control-legacy/generation-route-repair-started.hex
vendored
Normal file
2
tests/fixtures/idunn-control-legacy/generation-route-repair-started.hex
vendored
Normal file
File diff suppressed because one or more lines are too long
2
tests/fixtures/idunn-control-legacy/generation-v3-odin.hex
vendored
Normal file
2
tests/fixtures/idunn-control-legacy/generation-v3-odin.hex
vendored
Normal file
File diff suppressed because one or more lines are too long
2
tests/fixtures/idunn-control-legacy/generation-v3-route-proof.hex
vendored
Normal file
2
tests/fixtures/idunn-control-legacy/generation-v3-route-proof.hex
vendored
Normal file
File diff suppressed because one or more lines are too long
2
tests/fixtures/idunn-control-legacy/generation-with-receipts.hex
vendored
Normal file
2
tests/fixtures/idunn-control-legacy/generation-with-receipts.hex
vendored
Normal file
File diff suppressed because one or more lines are too long
1
tests/fixtures/idunn-control-legacy/provider-anchor.hex
vendored
Normal file
1
tests/fixtures/idunn-control-legacy/provider-anchor.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
93b263756c7463616368652e73746f72652e76319197d93167616d6563756c742e70726f76696465725f6865616c74685f6964656e746974792e74727573745f616e63686f722e7631d93167616d6563756c742e70726f76696465725f6865616c74685f6964656e746974792e74727573745f616e63686f722e7631d93467616d6563756c742e70726f76696465725f6865616c74685f6964656e746974792e74727573745f616e63686f722e76312e7631d93167616d6563756c742e70726f76696465725f6865616c74685f6964656e746974792e74727573745f616e63686f722e7631d9967b22736368656d614e616d65223a2267616d6563756c742e70726f76696465725f6865616c74685f6964656e746974792e74727573745f616e63686f722e7631222c22736368656d6156657273696f6e223a2267616d6563756c742e70726f76696465725f6865616c74685f6964656e746974792e74727573745f616e63686f722e76312e7631222c226d656d62657273223a5b5d7d91d93167616d6563756c742e70726f76696465725f6865616c74685f6964656e746974792e74727573745f616e63686f722e7631909194d92867616d6563756c742d70726f76696465722d6865616c74682d6964656e746974792d7075626c6963d93167616d6563756c742e70726f76696465725f6865616c74685f6964656e746974792e74727573745f616e63686f722e7631d923323032362d30392d32395430333a34353a31352e3031353232313330302b30303a3030c5014596d93167616d6563756c742e70726f76696465725f6865616c74685f6964656e746974792e74727573745f616e63686f722e7631d94031626661653932326138626630313132373562353536623236393135303363386366643538313234383931373134323365373736663133386463313235376337dc00205858cca9ccbb7dccb9cc8d01cccfccfccc9153ccdbcce23c224bcca757617dcce5cc8a163a6516cc9dccee2c2acce9d92d6f735f696e7374616c6c6174696f6e5f66696c655f626f756e645f636c6f6e6561626c655f626173656c696e65d923323032362d30392d32395430333a34353a31352e3031353232313330302b30303a3030d9477368613235362d63663933623766656534396235313133343736366538613937623432303065653636396338353164386464336634353366623537666664326233366630653430
|
||||
2
tests/fixtures/idunn-control-legacy/transaction-awaiting-ready.hex
vendored
Normal file
2
tests/fixtures/idunn-control-legacy/transaction-awaiting-ready.hex
vendored
Normal file
File diff suppressed because one or more lines are too long
2
tests/fixtures/idunn-control-legacy/transaction-committing-v2.hex
vendored
Normal file
2
tests/fixtures/idunn-control-legacy/transaction-committing-v2.hex
vendored
Normal file
File diff suppressed because one or more lines are too long
2
tests/fixtures/idunn-control-legacy/transaction-committing.hex
vendored
Normal file
2
tests/fixtures/idunn-control-legacy/transaction-committing.hex
vendored
Normal file
File diff suppressed because one or more lines are too long
2
tests/fixtures/idunn-control-legacy/transaction-failed-after-fencing.hex
vendored
Normal file
2
tests/fixtures/idunn-control-legacy/transaction-failed-after-fencing.hex
vendored
Normal file
File diff suppressed because one or more lines are too long
2
tests/fixtures/idunn-control-legacy/transaction-fencing.hex
vendored
Normal file
2
tests/fixtures/idunn-control-legacy/transaction-fencing.hex
vendored
Normal file
File diff suppressed because one or more lines are too long
2
tests/fixtures/idunn-control-legacy/transaction-leasing.hex
vendored
Normal file
2
tests/fixtures/idunn-control-legacy/transaction-leasing.hex
vendored
Normal file
File diff suppressed because one or more lines are too long
2
tests/fixtures/idunn-control-legacy/transaction-post-fence-abort.hex
vendored
Normal file
2
tests/fixtures/idunn-control-legacy/transaction-post-fence-abort.hex
vendored
Normal file
File diff suppressed because one or more lines are too long
2
tests/fixtures/idunn-control-legacy/transaction-pre-fence-abort-terminal.hex
vendored
Normal file
2
tests/fixtures/idunn-control-legacy/transaction-pre-fence-abort-terminal.hex
vendored
Normal file
File diff suppressed because one or more lines are too long
2
tests/fixtures/idunn-control-legacy/transaction-pre-fence-abort.hex
vendored
Normal file
2
tests/fixtures/idunn-control-legacy/transaction-pre-fence-abort.hex
vendored
Normal file
File diff suppressed because one or more lines are too long
2
tests/fixtures/idunn-control-legacy/transaction-routing.hex
vendored
Normal file
2
tests/fixtures/idunn-control-legacy/transaction-routing.hex
vendored
Normal file
File diff suppressed because one or more lines are too long
BIN
tests/fixtures/idunn.operator_incident.v1.cc
vendored
Normal file
BIN
tests/fixtures/idunn.operator_incident.v1.cc
vendored
Normal file
Binary file not shown.
1
tests/fixtures/pre-cut-systemd-observation.compact.hex
vendored
Normal file
1
tests/fixtures/pre-cut-systemd-observation.compact.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
dc0033d94e6964756e6e2d616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e73657276696365aa4964756e6e2074657374a3696e7601a465786563a26e6fa56d69786564c3a6753631303030a6753631303030a0a0a0c3c3a9696e76697369626c65a3616c6cc3a430303037a0ad5b756e7072696e7461626c655dcd10920194cdee48cdee48cdee48cdee4894cdee48cdee48cdee48cdee4891cdee480000000000c391010708b52f6f70742f746573742f62696e2f736572766963650101d9477368613235362d31313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131d9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161a92f6f70742f74657374a92f72756e2f74657374d9477368613235362d3232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323290d9477368613235362d33333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333d9202f73797374656d2e736c6963652f6964756e6e2d746573742e73657276696365c090aa61637469766174696f6edc0020010101010101010101010101010101010101010101010101010101010101010190
|
||||
1
tests/fixtures/pre-cut-systemd-observation.named.hex
vendored
Normal file
1
tests/fixtures/pre-cut-systemd-observation.named.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
de0033a4756e6974d94e6964756e6e2d616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e73657276696365b0756e69745f6465736372697074696f6eaa4964756e6e2074657374ad696e766f636174696f6e5f6964a3696e76d923657865635f6d61696e5f73746172745f74696d657374616d705f6d6f6e6f746f6e696301ac736572766963655f74797065a465786563ae726573746172745f706f6c696379a26e6fa96b696c6c5f6d6f6465a56d69786564ac64796e616d69635f75736572c3ac73797374656d645f75736572a6753631303030ad73797374656d645f67726f7570a6753631303030b4737570706c656d656e746172795f67726f757073a0b76361706162696c6974795f626f756e64696e675f736574a0b4616d6269656e745f6361706162696c6974696573a0ae707269766174655f6d6f756e7473c3ac707269766174655f70696473c3ac70726f746563745f70726f63a9696e76697369626c65ab70726f635f737562736574a3616c6cb16e6f5f6e65775f70726976696c65676573c3a5756d61736ba430303037b2696e61636365737369626c655f7061746873a0af6c6f61645f63726564656e7469616cad5b756e7072696e7461626c655da86d61696e5f706964cd1092b270726f636573735f73746172745f74696d6501ac70726f636573735f7569647394cdee48cdee48cdee48cdee48ac70726f636573735f6769647394cdee48cdee48cdee48cdee48ae70726f636573735f67726f75707391cdee48b770726f636573735f6361705f696e686572697461626c6500b570726f636573735f6361705f7065726d697474656400b570726f636573735f6361705f65666665637469766500b470726f636573735f6361705f626f756e64696e6700b370726f636573735f6361705f616d6269656e7400b970726f636573735f6e6f5f6e65775f70726976696c65676573c3b670726f636573735f6e616d6573706163655f706964739101b26d6f756e745f6e616d6573706163655f696407b07069645f6e616d6573706163655f696408aa65786563757461626c65b52f6f70742f746573742f62696e2f73657276696365b165786563757461626c655f64657669636501b065786563757461626c655f696e6f646501b165786563757461626c655f736861323536d9477368613235362d31313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131b372756e74696d655f696e7374616e63655f6964d9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161b1776f726b696e675f6469726563746f7279a92f6f70742f74657374ae72756e74696d655f62756e646c65a92f72756e2f74657374b3636f6d6d616e645f6c696e655f736861323536d9477368613235362d32323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232b1656e7669726f6e6d656e745f6e616d657390bb656e7669726f6e6d656e745f636f6e74726163745f736861323536d9477368613235362d33333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333ad636f6e74726f6c5f67726f7570d9202f73797374656d2e736c6963652f6964756e6e2d746573742e73657276696365b563726564656e7469616c735f6469726563746f7279c0bc706172656e745f6f6e6c795f66696c655f64657363726970746f727390bd61637469766174696f6e5f7369676e65725f6964656e746974795f6964aa61637469766174696f6ebc61637469766174696f6e5f7369676e65725f7075626c69635f6b6579dc00200101010101010101010101010101010101010101010101010101010101010101b3736572766963655f63726564656e7469616c7390
|
||||
28
tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show
vendored
Normal file
28
tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show
vendored
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
Type=exec
|
||||
Restart=no
|
||||
MainPID=2571
|
||||
OpenFile=/run/idunn/activation-credentials/sha256-7cfe5cd1e87839e029c7425eb1b586b6b0d94dd79d7376eed3b156386f2c7af8.credential:gamecult-idunn-runtime-activation-key:read-only
|
||||
OpenFile=/etc/gamecult/odin/runtime-presence-identity.cc:gamecult-runtime-presence-identity:read-only
|
||||
ExecMainStartTimestampMonotonic=20630000
|
||||
ControlGroup=/system.slice/idunn-odin-7cfe5cd1e87839e029c7425eb1b586b6b0d94dd79d7376eed3b156386f2c7af8.service
|
||||
UMask=0007
|
||||
WorkingDirectory=/srv/odin/releases/sha256-e5c7d0b2b9c0a8ffba368aeb08abdbe37ae0d17484a785e60a66ec4a62b2f18c
|
||||
CapabilityBoundingSet=
|
||||
AmbientCapabilities=
|
||||
User=_duc23369f1b8b167db
|
||||
Group=odin
|
||||
DynamicUser=yes
|
||||
LoadCredential=[unprintable]
|
||||
SupplementaryGroups=
|
||||
InaccessiblePaths=/run/idunn/activation-credentials
|
||||
PrivateMounts=yes
|
||||
PrivatePIDs=yes
|
||||
NoNewPrivileges=yes
|
||||
ProtectProc=invisible
|
||||
ProcSubset=all
|
||||
KillMode=mixed
|
||||
Description=Idunn odin sha256-7cfe5cd1e87839e029c7425eb1b586b6b0d94dd79d7376eed3b156386f2c7af8
|
||||
LoadState=loaded
|
||||
ActiveState=active
|
||||
SubState=running
|
||||
InvocationID=ada48921043449f39dacc87b14a1bf70
|
||||
28
tests/fixtures/systemd-257/vili-presence-identity-only.show
vendored
Normal file
28
tests/fixtures/systemd-257/vili-presence-identity-only.show
vendored
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
Type=exec
|
||||
Restart=no
|
||||
MainPID=0
|
||||
OpenFile=/run/idunn/activation-credentials/sha256-b507e1705a70f8eee72f355e7db637acab8eb37880b5d25245fda68d80f63041.credential:gamecult-idunn-runtime-activation-key:read-only
|
||||
OpenFile=/etc/gamecult/vili/runtime-presence-identity.cc:gamecult-runtime-presence-identity:read-only
|
||||
ExecMainStartTimestampMonotonic=520083420000
|
||||
ControlGroup=
|
||||
UMask=0007
|
||||
WorkingDirectory=/srv/vili-idunn/releases/sha256-aeaee4e1669742572c45a3f26bc86893c44b2a4149520975eb7d8a28aa931906
|
||||
CapabilityBoundingSet=
|
||||
AmbientCapabilities=
|
||||
User=_du8c06c563aac6d205
|
||||
Group=vili-state
|
||||
DynamicUser=yes
|
||||
LoadCredential=[unprintable]
|
||||
SupplementaryGroups=
|
||||
InaccessiblePaths=/run/idunn/activation-credentials
|
||||
PrivateMounts=yes
|
||||
PrivatePIDs=yes
|
||||
NoNewPrivileges=yes
|
||||
ProtectProc=invisible
|
||||
ProcSubset=all
|
||||
KillMode=mixed
|
||||
Description=Idunn vili sha256-b507e1705a70f8eee72f355e7db637acab8eb37880b5d25245fda68d80f63041
|
||||
LoadState=loaded
|
||||
ActiveState=failed
|
||||
SubState=failed
|
||||
InvocationID=8f7a349f2f8f49fe87ac5d89239260b4
|
||||
Loading…
Reference in a new issue