GameCult/Odin
Watch
9
0
Fork
You've already forked Odin
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Make Idunn health contracts explicit

Browse source
This commit is contained in:
Meta Vens 2026-06-15 20:50:57 +01:00
commit 83760e9cd8
4 changed files with 193 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

141
crates/idunn-daemon/src/main.rs
View file

@ -17,6 +17,7 @@ struct DaemonTarget {
daemon_id: String,
verse_id: String,
name: String,
health_contract: HealthContract,
health_command: Option<String>,
deploy_command: Option<String>,
restart_command: Option<String>,
@ -24,6 +25,19 @@ struct DaemonTarget {
interval_seconds: u64,
}
#[derive(Clone, Debug)]
struct HealthContract {
id: String,
default_failure_state: String,
}
fn health_contract(id: &str, default_failure_state: &str) -> HealthContract {
HealthContract {
id: id.to_string(),
default_failure_state: default_failure_state.to_string(),
}
}
#[derive(Clone, Debug)]
struct CommonOptions {
store_path: PathBuf,
@ -75,6 +89,7 @@ fn run_swarm(options: &SwarmOptions, common: &CommonOptions) -> Result<()> {
options.profile
));
}
validate_targets(&targets)?;
println!(
"Idunn swarm profile {} starting with {} targets.",
@ -104,6 +119,35 @@ fn run_swarm(options: &SwarmOptions, common: &CommonOptions) -> Result<()> {
Ok(())
}
fn validate_targets(targets: &[DaemonTarget]) -> Result<()> {
let mut issues = Vec::new();
for target in targets {
if target.health_contract.id.trim().is_empty() {
issues.push(format!("{} has no health contract", target.daemon_id));
}
if target.health_command.is_none() {
issues.push(format!("{} has no health command", target.daemon_id));
}
if target.health_contract.default_failure_state == "stale-deployment"
&& target.deploy_command.is_none()
{
issues.push(format!(
"{} treats probe failure as stale deployment but has no deploy command",
target.daemon_id
));
}
}
if issues.is_empty() {
Ok(())
} else {
Err(anyhow!(
"Idunn target catalog is incoherent: {}",
issues.join("; ")
))
}
}
fn run_target_loop(
target: DaemonTarget,
options: CommonOptions,
@ -135,6 +179,7 @@ fn run_target_cycle(
health_command: target.health_command.clone(),
restart_command: target.restart_command.clone(),
deploy_command: target.deploy_command.clone(),
health_contract: target.health_contract.id.clone(),
authority: "idunn.local-command".to_string(),
max_silence_seconds: 60,
observed_at: now.clone(),
@ -242,11 +287,7 @@ fn run_target_cycle(
Ok(())
}
fn with_store_node<F>(
options: &CommonOptions,
store_lock: &Arc<Mutex<()>>,
write: F,
) -> Result<()>
fn with_store_node<F>(options: &CommonOptions, store_lock: &Arc<Mutex<()>>, write: F) -> Result<()>
where
F: FnOnce(&mut CultMeshNode) -> Result<()>,
{
@ -274,6 +315,7 @@ fn swarm_targets(options: &SwarmOptions) -> Result<Vec<DaemonTarget>> {
daemon_id: "odin".to_string(),
verse_id: "starfire.local".to_string(),
name: "Odin all-seer".to_string(),
health_contract: health_contract("odin.http-health-and-deck-publication", "failed"),
health_command: Some(script("health-odin.cmd")),
deploy_command: None,
restart_command: Some(script("restart-odin.cmd")),
@ -284,6 +326,7 @@ fn swarm_targets(options: &SwarmOptions) -> Result<Vec<DaemonTarget>> {
daemon_id: "mimir-eve-dashboard".to_string(),
verse_id: "starfire.local".to_string(),
name: "Mimir Eve dashboard".to_string(),
health_contract: health_contract("mimir.http-dashboard-health", "failed"),
health_command: Some(script("health-mimir-eve-dashboard.cmd")),
deploy_command: None,
restart_command: None,
@ -294,6 +337,7 @@ fn swarm_targets(options: &SwarmOptions) -> Result<Vec<DaemonTarget>> {
daemon_id: "stonks".to_string(),
verse_id: "starfire.local".to_string(),
name: "Stonks market pulse".to_string(),
health_contract: health_contract("stonks.http-health", "failed"),
health_command: Some(script("health-stonks.cmd")),
deploy_command: None,
restart_command: Some(script("restart-stonks.cmd")),
@ -304,6 +348,7 @@ fn swarm_targets(options: &SwarmOptions) -> Result<Vec<DaemonTarget>> {
daemon_id: "voidbot".to_string(),
verse_id: "starfire.local".to_string(),
name: "VoidBot local stack".to_string(),
health_contract: health_contract("voidbot.stack-health", "failed"),
health_command: Some(script("health-voidbot.cmd")),
deploy_command: None,
restart_command: Some(script("restart-voidbot.cmd")),
@ -314,6 +359,7 @@ fn swarm_targets(options: &SwarmOptions) -> Result<Vec<DaemonTarget>> {
daemon_id: "weksa".to_string(),
verse_id: "starfire.local".to_string(),
name: "Weksa intent and utterance lowering service".to_string(),
health_contract: health_contract("weksa.provider-health", "failed"),
health_command: Some(script("health-weksa.cmd")),
deploy_command: None,
restart_command: Some(script("restart-weksa.cmd")),
@ -324,6 +370,10 @@ fn swarm_targets(options: &SwarmOptions) -> Result<Vec<DaemonTarget>> {
daemon_id: "starfire-muninn".to_string(),
verse_id: "starfire.local".to_string(),
name: "Starfire Muninn telemetry and Quest access daemon".to_string(),
health_contract: health_contract(
"muninn.local-telemetry-and-quest-access",
"failed",
),
health_command: Some(script("health-starfire-muninn.cmd")),
deploy_command: None,
restart_command: Some(script("restart-starfire-muninn.cmd")),
@ -334,6 +384,7 @@ fn swarm_targets(options: &SwarmOptions) -> Result<Vec<DaemonTarget>> {
daemon_id: "muninn".to_string(),
verse_id: "raven.local".to_string(),
name: "Muninn telemetry Verse assembler".to_string(),
health_contract: health_contract("muninn.remote-telemetry-health", "failed"),
health_command: Some(script("health-muninn.cmd")),
deploy_command: None,
restart_command: Some(script("restart-muninn.cmd")),
@ -344,6 +395,7 @@ fn swarm_targets(options: &SwarmOptions) -> Result<Vec<DaemonTarget>> {
daemon_id: "vili".to_string(),
verse_id: "raven.local".to_string(),
name: "Vili Persona animation daemon".to_string(),
health_contract: health_contract("vili.http-animation-health", "failed"),
health_command: Some(script("health-vili.cmd")),
deploy_command: None,
restart_command: Some(script("restart-vili.cmd")),
@ -354,6 +406,7 @@ fn swarm_targets(options: &SwarmOptions) -> Result<Vec<DaemonTarget>> {
daemon_id: "idunn-swarm-deployment-coverage".to_string(),
verse_id: "starfire.local".to_string(),
name: "Idunn swarm deployment coverage".to_string(),
health_contract: health_contract("idunn.deployment-catalog-coherence", "degraded"),
health_command: Some(script("health-idunn-swarm-deployment-coverage.cmd")),
deploy_command: None,
restart_command: None,
@ -364,6 +417,10 @@ fn swarm_targets(options: &SwarmOptions) -> Result<Vec<DaemonTarget>> {
daemon_id: "yggdrasil-heimdall".to_string(),
verse_id: "yggdrasil.local".to_string(),
name: "Yggdrasil Heimdall".to_string(),
health_contract: health_contract(
"yggdrasil.source-deployment-freshness",
"stale-deployment",
),
health_command: Some(script("health-yggdrasil-heimdall.cmd")),
deploy_command: Some(script("deploy-yggdrasil-heimdall.cmd")),
restart_command: None,
@ -374,6 +431,10 @@ fn swarm_targets(options: &SwarmOptions) -> Result<Vec<DaemonTarget>> {
daemon_id: "yggdrasil-repixelizer".to_string(),
verse_id: "yggdrasil.local".to_string(),
name: "Yggdrasil Repixelizer".to_string(),
health_contract: health_contract(
"yggdrasil.source-deployment-freshness",
"stale-deployment",
),
health_command: Some(script("health-yggdrasil-repixelizer.cmd")),
deploy_command: Some(script("deploy-yggdrasil-repixelizer.cmd")),
restart_command: None,
@ -384,6 +445,10 @@ fn swarm_targets(options: &SwarmOptions) -> Result<Vec<DaemonTarget>> {
daemon_id: "yggdrasil-streampixels".to_string(),
verse_id: "yggdrasil.local".to_string(),
name: "Yggdrasil StreamPixels".to_string(),
health_contract: health_contract(
"yggdrasil.source-deployment-freshness",
"stale-deployment",
),
health_command: Some(script("health-yggdrasil-streampixels.cmd")),
deploy_command: Some(script("deploy-yggdrasil-streampixels.cmd")),
restart_command: None,
@ -394,6 +459,10 @@ fn swarm_targets(options: &SwarmOptions) -> Result<Vec<DaemonTarget>> {
daemon_id: "nightwing-gjallar".to_string(),
verse_id: "nightwing.local".to_string(),
name: "Nightwing Gjallar framebuffer compositor".to_string(),
health_contract: health_contract(
"gjallar.framebuffer-composition-witness",
"dependency-unavailable",
),
health_command: Some(script("health-nightwing-gjallar.cmd")),
deploy_command: Some(script("deploy-nightwing-gjallar.cmd")),
restart_command: Some(script("restart-nightwing-gjallar.cmd")),
@ -404,6 +473,7 @@ fn swarm_targets(options: &SwarmOptions) -> Result<Vec<DaemonTarget>> {
daemon_id: "nightwing-muninn".to_string(),
verse_id: "nightwing.local".to_string(),
name: "Nightwing Muninn telemetry and Move HID daemon".to_string(),
health_contract: health_contract("muninn.remote-telemetry-and-move-hid", "failed"),
health_command: Some(script("health-nightwing-muninn.cmd")),
deploy_command: None,
restart_command: Some(script("restart-nightwing-muninn.cmd")),
@ -414,6 +484,7 @@ fn swarm_targets(options: &SwarmOptions) -> Result<Vec<DaemonTarget>> {
daemon_id: "nightwing-eve-dashboard".to_string(),
verse_id: "nightwing.local".to_string(),
name: "Nightwing Eve dashboard broker".to_string(),
health_contract: health_contract("nightwing.systemd-service-liveness", "failed"),
health_command: Some(script("health-nightwing-eve-dashboard.cmd")),
deploy_command: None,
restart_command: Some(script("restart-nightwing-eve-dashboard.cmd")),
@ -424,6 +495,7 @@ fn swarm_targets(options: &SwarmOptions) -> Result<Vec<DaemonTarget>> {
daemon_id: "nightwing-eve-browser-reference".to_string(),
verse_id: "nightwing.local".to_string(),
name: "Nightwing Eve browser reference".to_string(),
health_contract: health_contract("nightwing.systemd-service-liveness", "failed"),
health_command: Some(script("health-nightwing-eve-browser-reference.cmd")),
deploy_command: None,
restart_command: Some(script("restart-nightwing-eve-browser-reference.cmd")),
@ -564,6 +636,7 @@ impl Options {
daemon_id: daemon_id.clone(),
verse_id,
name: name.unwrap_or(daemon_id),
health_contract: health_contract("manual.command-health", "failed"),
health_command,
deploy_command,
restart_command,
@ -599,6 +672,7 @@ fn probe_health(
daemon_id: target.daemon_id.clone(),
state: "active".to_string(),
detail: command_output_detail("health command exited successfully", &output),
health_contract: target.health_contract.id.clone(),
observed_at: observed_at.to_string(),
},
Ok(output) => {
@ -608,8 +682,9 @@ fn probe_health(
);
IdunnDaemonHealthRecord {
daemon_id: target.daemon_id.clone(),
state: health_state_from_detail(&detail).to_string(),
state: health_state_from_detail(&detail, target).to_string(),
detail,
health_contract: target.health_contract.id.clone(),
observed_at: observed_at.to_string(),
}
}
@ -617,6 +692,7 @@ fn probe_health(
daemon_id: target.daemon_id.clone(),
state: "failed".to_string(),
detail: format!("health command could not run: {error}"),
health_contract: target.health_contract.id.clone(),
observed_at: observed_at.to_string(),
},
},
@ -624,12 +700,13 @@ fn probe_health(
daemon_id: target.daemon_id.clone(),
state: "unknown".to_string(),
detail: "no health command was provided".to_string(),
health_contract: target.health_contract.id.clone(),
observed_at: observed_at.to_string(),
},
}
}
fn health_state_from_detail(detail: &str) -> &str {
fn health_state_from_detail<'a>(detail: &'a str, target: &'a DaemonTarget) -> &'a str {
for state in [
"stale-deployment",
"dependency-unavailable",
@ -642,7 +719,7 @@ fn health_state_from_detail(detail: &str) -> &str {
}
}
"failed"
&target.health_contract.default_failure_state
}
fn run_restart(
@ -902,3 +979,51 @@ fn timestamp() -> Result<String> {
fn help_text() -> &'static str {
"Usage: idunn --daemon <id> [--name <name>] [--verse <verse>] [--store <path>] [--health-command <command>] [--deploy-command <command>] [--restart-command <command>] [--operator-alarm-command <command>] [--execute] [--interval-seconds <seconds>] [--command-timeout-seconds <seconds>] [--repo-root <path>] [--swarm-profile <profile>]\n\nIdunn can run one manual daemon probe lane with --daemon, or one built-in swarm supervisor with --swarm-profile starfire-local."
}
#[cfg(test)]
mod tests {
use super::*;
fn target(default_failure_state: &str, deploy_command: Option<&str>) -> DaemonTarget {
DaemonTarget {
daemon_id: "test-daemon".to_string(),
verse_id: "test.local".to_string(),
name: "Test daemon".to_string(),
health_contract: health_contract("test.contract", default_failure_state),
health_command: Some("exit 0".to_string()),
deploy_command: deploy_command.map(ToString::to_string),
restart_command: Some("restart test".to_string()),
enabled: true,
interval_seconds: 30,
}
}
#[test]
fn target_catalog_requires_health_contract() {
let mut target = target("failed", None);
target.health_contract.id = String::new();
let error = validate_targets(&[target]).unwrap_err().to_string();
assert!(error.contains("has no health contract"));
}
#[test]
fn stale_deployment_contract_requires_deploy_authority() {
let error = validate_targets(&[target("stale-deployment", None)])
.unwrap_err()
.to_string();
assert!(error.contains("has no deploy command"));
}
#[test]
fn health_state_uses_contract_default_when_probe_has_no_marker() {
let target = target("dependency-unavailable", None);
assert_eq!(
health_state_from_detail("plain command failure", &target),
"dependency-unavailable"
);
}
}

4
crates/odin-core/src/documents.rs
View file

@ -215,6 +215,8 @@ pub struct IdunnDesiredDaemonRecord {
pub observed_at: String,
#[cultcache(key = 9)]
pub deploy_command: Option<String>,
#[cultcache(key = 10)]
pub health_contract: String,
}
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
@ -228,6 +230,8 @@ pub struct IdunnDaemonHealthRecord {
pub detail: String,
#[cultcache(key = 3)]
pub observed_at: String,
#[cultcache(key = 4)]
pub health_contract: String,
}
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]

37
crates/odin-core/src/idunn.rs
View file

@ -85,6 +85,27 @@ pub fn plan_keepalive(
}
}
if health.state == "stale-deployment" {
let alarm_id = format!("alarm:{}:{}", desired.daemon_id, now);
let reason = format!(
"health is stale-deployment under {}; no deploy command authority is available",
desired.health_contract
);
return IdunnPlan {
decision: decision(&decision_id, desired, "alarm", &reason, &now),
deployment_request: None,
restart_request: None,
operator_alarm: Some(IdunnOperatorAlarmRecord {
alarm_id,
daemon_id: desired.daemon_id.clone(),
severity: "operator-action-required".to_string(),
reason,
escalation_target: "bifrost.operator-notification".to_string(),
raised_at: now,
}),
};
}
match desired.restart_command.as_deref() {
Some(command) if !command.trim().is_empty() => {
let request_id = format!("restart:{}:{}", desired.daemon_id, now);
@ -170,6 +191,7 @@ mod tests {
max_silence_seconds: 60,
observed_at: "2026-06-04T00:00:00Z".to_string(),
deploy_command: None,
health_contract: "test.command-exit".to_string(),
}
}
@ -179,6 +201,7 @@ mod tests {
state: state.to_string(),
detail: "unit probe".to_string(),
observed_at: "2026-06-04T00:00:01Z".to_string(),
health_contract: "test.command-exit".to_string(),
}
}
@ -257,6 +280,20 @@ mod tests {
assert!(plan.operator_alarm.is_some());
}
#[test]
fn stale_deployment_without_deploy_authority_raises_alarm_instead_of_restarting() {
let plan = plan_keepalive(
&desired(Some("systemctl restart old-app")),
&health("stale-deployment"),
"2026-06-04T00:00:02Z",
);
assert_eq!(plan.decision.action, "alarm");
assert!(plan.deployment_request.is_none());
assert!(plan.restart_request.is_none());
assert!(plan.operator_alarm.is_some());
}
#[test]
fn unhealthy_daemon_without_authority_raises_operator_alarm() {
let plan = plan_keepalive(&desired(None), &health("failed"), "2026-06-04T00:00:02Z");

26
docs/idunn.md
View file

@ -17,9 +17,9 @@ escalation, and continuity witness state.
daemon lives: bring-up after host reboot, deployment freshness, crash
recovery, health watching, deploy/restart policy, and operator escalation.
- Inputs: Odin's accepted service records, provider advertisements, `.cc`
witnesses, advertised command boundaries, freshness windows, operator policy,
local service manager state, and direct local service probes only when no
provider advertisement exists.
witnesses, advertised command boundaries, health contracts, freshness
windows, operator policy, local service manager state, and direct local
service probes only when no provider advertisement exists.
- Outputs: typed keepalive observations, deployment requests/results, restart
requests/results, denied-action records, operator alarms, Bifrost
operator-notification requests, and an Eve/CultUI keepalive surface.
@ -103,6 +103,17 @@ idunn.operator_alarm.v1
Odin refresh or manual click, Idunn's ownership path is still incomplete.
- Restart attempts must be witnessed: requested by whom, against which service,
through which command boundary, with what result and timestamp.
- Health command exit status is not daemon awareness by itself. Every Idunn
target must declare a health contract naming what the probe proves and what
unmarked failure means. `idunn.desired_daemon.v1` and
`idunn.daemon_health.v1` both record that contract so later readers can
distinguish process liveness, HTTP health, source deployment freshness,
framebuffer composition, telemetry capture, and catalog coherence.
- A stale deployment is not restartable liveness. If a target reports
`stale-deployment` without deploy authority, Idunn must alarm instead of
restarting the stale artifact. If a target reports `dependency-unavailable`
or `degraded`, Idunn must alarm instead of treating local deploy/restart as
the owner.
- Idunn should fail closed when authority is unclear. Unknown ownership,
repeated restart failure, missing command authority, or degraded health that
needs a human becomes an operator alarm.
@ -163,10 +174,11 @@ Idunn's current Rust runtime now has two postures:
The local swarm mode owns:
1. the built-in Starfire-local target catalog;
2. one in-process schedule per target instead of one watchdog process per target;
3. shared typed keepalive records in one CultMesh store;
4. deploy/restart/alarm execution through the same Rust decision engine;
5. recovery of fast local targets like Odin without waiting behind slow remote
2. a mandatory health contract per target;
3. one in-process schedule per target instead of one watchdog process per target;
4. shared typed keepalive records in one CultMesh store;
5. deploy/restart/alarm execution through the same Rust decision engine;
6. recovery of fast local targets like Odin without waiting behind slow remote
Yggdrasil checks.
Next: ingest Odin-owned service/provider advertisements directly, promote the