2026-06-04 21:07:42 +01:00
|
|
|
use anyhow::{Context, Result, anyhow};
|
2026-06-15 17:06:32 +01:00
|
|
|
use cultmesh_rs::{CultMesh, CultMeshNode, CultMeshNodeOptions};
|
2026-06-04 21:07:42 +01:00
|
|
|
use odin_core::{
|
2026-06-09 15:32:20 +01:00
|
|
|
IdunnDaemonHealthRecord, IdunnDeploymentResultRecord, IdunnDesiredDaemonRecord,
|
|
|
|
|
IdunnOperatorAlarmRecord, IdunnRestartResultRecord, OdinDocuments, plan_keepalive,
|
2026-06-04 21:07:42 +01:00
|
|
|
};
|
|
|
|
|
use std::env;
|
|
|
|
|
use std::path::PathBuf;
|
2026-06-11 16:41:21 +01:00
|
|
|
use std::process::{Child, Command, Stdio};
|
2026-06-15 17:06:32 +01:00
|
|
|
use std::sync::{Arc, Mutex};
|
2026-06-04 21:07:42 +01:00
|
|
|
use std::thread;
|
|
|
|
|
use std::time::Duration;
|
2026-06-11 16:41:21 +01:00
|
|
|
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
2026-06-04 21:07:42 +01:00
|
|
|
|
|
|
|
|
#[derive(Clone, Debug)]
|
2026-06-11 16:41:21 +01:00
|
|
|
struct DaemonTarget {
|
2026-06-04 21:07:42 +01:00
|
|
|
daemon_id: String,
|
|
|
|
|
verse_id: String,
|
|
|
|
|
name: String,
|
2026-06-15 20:50:57 +01:00
|
|
|
health_contract: HealthContract,
|
2026-06-04 21:07:42 +01:00
|
|
|
health_command: Option<String>,
|
2026-06-09 15:32:20 +01:00
|
|
|
deploy_command: Option<String>,
|
2026-06-04 21:07:42 +01:00
|
|
|
restart_command: Option<String>,
|
|
|
|
|
enabled: bool,
|
2026-06-11 16:41:21 +01:00
|
|
|
interval_seconds: u64,
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-15 20:50:57 +01:00
|
|
|
#[derive(Clone, Debug)]
|
|
|
|
|
struct HealthContract {
|
|
|
|
|
id: String,
|
|
|
|
|
default_failure_state: String,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn health_contract(id: &str, default_failure_state: &str) -> HealthContract {
|
|
|
|
|
HealthContract {
|
|
|
|
|
id: id.to_string(),
|
|
|
|
|
default_failure_state: default_failure_state.to_string(),
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-11 16:41:21 +01:00
|
|
|
#[derive(Clone, Debug)]
|
|
|
|
|
struct CommonOptions {
|
|
|
|
|
store_path: PathBuf,
|
|
|
|
|
operator_alarm_command: Option<String>,
|
2026-06-04 21:07:42 +01:00
|
|
|
execute: bool,
|
2026-06-11 16:41:21 +01:00
|
|
|
command_timeout_seconds: u64,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[derive(Clone, Debug)]
|
|
|
|
|
enum Mode {
|
|
|
|
|
Single(DaemonTarget),
|
|
|
|
|
Swarm(SwarmOptions),
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[derive(Clone, Debug)]
|
|
|
|
|
struct SwarmOptions {
|
|
|
|
|
profile: String,
|
|
|
|
|
repo_root: PathBuf,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[derive(Clone, Debug)]
|
|
|
|
|
struct Options {
|
|
|
|
|
common: CommonOptions,
|
|
|
|
|
mode: Mode,
|
2026-06-04 21:07:42 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn main() -> Result<()> {
|
|
|
|
|
let options = Options::parse(env::args().skip(1))?;
|
|
|
|
|
|
2026-06-11 16:41:21 +01:00
|
|
|
if let Some(parent) = options.common.store_path.parent() {
|
2026-06-04 21:07:42 +01:00
|
|
|
std::fs::create_dir_all(parent)
|
|
|
|
|
.with_context(|| format!("creating {}", parent.display()))?;
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-11 16:41:21 +01:00
|
|
|
match &options.mode {
|
2026-06-15 17:06:32 +01:00
|
|
|
Mode::Single(target) => {
|
|
|
|
|
let store_lock = Arc::new(Mutex::new(()));
|
|
|
|
|
run_target_cycle(target, &options.common, &store_lock)
|
|
|
|
|
}
|
2026-06-11 16:41:21 +01:00
|
|
|
Mode::Swarm(swarm) => run_swarm(swarm, &options.common),
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn run_swarm(options: &SwarmOptions, common: &CommonOptions) -> Result<()> {
|
|
|
|
|
let targets = swarm_targets(options)?;
|
|
|
|
|
if targets.is_empty() {
|
|
|
|
|
return Err(anyhow!(
|
|
|
|
|
"Idunn swarm profile {} resolved to no targets",
|
|
|
|
|
options.profile
|
|
|
|
|
));
|
|
|
|
|
}
|
2026-06-15 20:50:57 +01:00
|
|
|
validate_targets(&targets)?;
|
2026-06-11 16:41:21 +01:00
|
|
|
|
|
|
|
|
println!(
|
|
|
|
|
"Idunn swarm profile {} starting with {} targets.",
|
|
|
|
|
options.profile,
|
|
|
|
|
targets.len()
|
|
|
|
|
);
|
|
|
|
|
println!("CultMesh store: {}", common.store_path.display());
|
|
|
|
|
|
2026-06-15 17:06:32 +01:00
|
|
|
let store_lock = Arc::new(Mutex::new(()));
|
2026-06-11 16:41:21 +01:00
|
|
|
let mut workers = Vec::with_capacity(targets.len());
|
|
|
|
|
for target in targets {
|
|
|
|
|
let worker_common = common.clone();
|
2026-06-15 17:06:32 +01:00
|
|
|
let worker_store_lock = Arc::clone(&store_lock);
|
2026-06-11 16:41:21 +01:00
|
|
|
workers.push(thread::spawn(move || {
|
2026-06-15 17:06:32 +01:00
|
|
|
run_target_loop(target, worker_common, worker_store_lock)
|
2026-06-11 16:41:21 +01:00
|
|
|
}));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for worker in workers {
|
|
|
|
|
match worker.join() {
|
|
|
|
|
Ok(Ok(())) => {}
|
|
|
|
|
Ok(Err(error)) => return Err(error),
|
|
|
|
|
Err(_) => return Err(anyhow!("Idunn swarm worker thread panicked")),
|
2026-06-04 21:07:42 +01:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-11 16:41:21 +01:00
|
|
|
Ok(())
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-15 20:50:57 +01:00
|
|
|
fn validate_targets(targets: &[DaemonTarget]) -> Result<()> {
|
|
|
|
|
let mut issues = Vec::new();
|
|
|
|
|
for target in targets {
|
|
|
|
|
if target.health_contract.id.trim().is_empty() {
|
|
|
|
|
issues.push(format!("{} has no health contract", target.daemon_id));
|
|
|
|
|
}
|
|
|
|
|
if target.health_command.is_none() {
|
|
|
|
|
issues.push(format!("{} has no health command", target.daemon_id));
|
|
|
|
|
}
|
|
|
|
|
if target.health_contract.default_failure_state == "stale-deployment"
|
|
|
|
|
&& target.deploy_command.is_none()
|
|
|
|
|
{
|
|
|
|
|
issues.push(format!(
|
|
|
|
|
"{} treats probe failure as stale deployment but has no deploy command",
|
|
|
|
|
target.daemon_id
|
|
|
|
|
));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if issues.is_empty() {
|
|
|
|
|
Ok(())
|
|
|
|
|
} else {
|
|
|
|
|
Err(anyhow!(
|
|
|
|
|
"Idunn target catalog is incoherent: {}",
|
|
|
|
|
issues.join("; ")
|
|
|
|
|
))
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-15 17:06:32 +01:00
|
|
|
fn run_target_loop(
|
|
|
|
|
target: DaemonTarget,
|
|
|
|
|
options: CommonOptions,
|
|
|
|
|
store_lock: Arc<Mutex<()>>,
|
|
|
|
|
) -> Result<()> {
|
2026-06-11 16:41:21 +01:00
|
|
|
loop {
|
2026-06-15 17:06:32 +01:00
|
|
|
if let Err(error) = run_target_cycle(&target, &options, &store_lock) {
|
2026-06-11 16:41:21 +01:00
|
|
|
eprintln!(
|
|
|
|
|
"Idunn swarm target {} cycle failed: {}",
|
|
|
|
|
target.daemon_id, error
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
thread::sleep(Duration::from_secs(target.interval_seconds));
|
|
|
|
|
}
|
2026-06-04 21:07:42 +01:00
|
|
|
}
|
|
|
|
|
|
2026-06-15 17:06:32 +01:00
|
|
|
fn run_target_cycle(
|
|
|
|
|
target: &DaemonTarget,
|
|
|
|
|
options: &CommonOptions,
|
|
|
|
|
store_lock: &Arc<Mutex<()>>,
|
|
|
|
|
) -> Result<()> {
|
2026-06-04 21:07:42 +01:00
|
|
|
let now = timestamp()?;
|
|
|
|
|
|
|
|
|
|
let desired = IdunnDesiredDaemonRecord {
|
2026-06-11 16:41:21 +01:00
|
|
|
daemon_id: target.daemon_id.clone(),
|
|
|
|
|
verse_id: target.verse_id.clone(),
|
|
|
|
|
name: target.name.clone(),
|
|
|
|
|
enabled: target.enabled,
|
|
|
|
|
health_command: target.health_command.clone(),
|
|
|
|
|
restart_command: target.restart_command.clone(),
|
|
|
|
|
deploy_command: target.deploy_command.clone(),
|
2026-06-15 20:50:57 +01:00
|
|
|
health_contract: target.health_contract.id.clone(),
|
2026-06-04 21:07:42 +01:00
|
|
|
authority: "idunn.local-command".to_string(),
|
|
|
|
|
max_silence_seconds: 60,
|
|
|
|
|
observed_at: now.clone(),
|
|
|
|
|
};
|
|
|
|
|
|
2026-06-11 16:41:21 +01:00
|
|
|
let health = probe_health(target, options.command_timeout_seconds, &now);
|
2026-06-04 21:07:42 +01:00
|
|
|
let plan = plan_keepalive(&desired, &health, now.clone());
|
|
|
|
|
|
2026-06-15 17:06:32 +01:00
|
|
|
with_store_node(options, store_lock, |node| {
|
|
|
|
|
node.put(&desired.daemon_id, &desired)?;
|
|
|
|
|
node.put(&health.daemon_id, &health)?;
|
|
|
|
|
node.put(&plan.decision.decision_id, &plan.decision)?;
|
|
|
|
|
if let Some(request) = &plan.deployment_request {
|
|
|
|
|
node.put(&request.request_id, request)?;
|
|
|
|
|
}
|
|
|
|
|
if let Some(request) = &plan.restart_request {
|
|
|
|
|
node.put(&request.request_id, request)?;
|
|
|
|
|
}
|
|
|
|
|
if let Some(alarm) = &plan.operator_alarm {
|
|
|
|
|
node.put(&alarm.alarm_id, alarm)?;
|
|
|
|
|
}
|
|
|
|
|
Ok(())
|
|
|
|
|
})?;
|
2026-06-04 21:07:42 +01:00
|
|
|
|
2026-06-09 15:32:20 +01:00
|
|
|
if let Some(request) = &plan.deployment_request {
|
|
|
|
|
if options.execute {
|
2026-06-11 16:41:21 +01:00
|
|
|
let result = run_deployment(request, &now, options.command_timeout_seconds);
|
2026-06-15 17:06:32 +01:00
|
|
|
let alarm = if result.state != "succeeded" {
|
|
|
|
|
Some(deployment_failure_alarm(&result, &now))
|
|
|
|
|
} else {
|
|
|
|
|
None
|
|
|
|
|
};
|
|
|
|
|
with_store_node(options, store_lock, |node| {
|
|
|
|
|
node.put(&result.result_id, &result)?;
|
|
|
|
|
if let Some(alarm) = &alarm {
|
|
|
|
|
node.put(&alarm.alarm_id, alarm)?;
|
|
|
|
|
}
|
|
|
|
|
Ok(())
|
|
|
|
|
})?;
|
2026-06-09 15:32:20 +01:00
|
|
|
println!(
|
|
|
|
|
"Idunn deployment {} for {}: {}",
|
|
|
|
|
result.state, result.daemon_id, result.detail
|
|
|
|
|
);
|
2026-06-15 17:06:32 +01:00
|
|
|
if let Some(alarm) = alarm {
|
2026-06-09 15:32:20 +01:00
|
|
|
println!(
|
|
|
|
|
"Idunn raised operator alarm for {} through {}: {}",
|
|
|
|
|
alarm.daemon_id, alarm.escalation_target, alarm.reason
|
|
|
|
|
);
|
2026-06-11 16:41:21 +01:00
|
|
|
run_operator_alarm_command(options, &alarm);
|
2026-06-09 15:32:20 +01:00
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
println!(
|
|
|
|
|
"Idunn requested deployment for {} but did not execute it. Pass --execute to actuate.",
|
|
|
|
|
request.daemon_id
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-04 21:07:42 +01:00
|
|
|
if let Some(request) = &plan.restart_request {
|
|
|
|
|
if options.execute {
|
2026-06-11 16:41:21 +01:00
|
|
|
let result = run_restart(request, &now, options.command_timeout_seconds);
|
2026-06-15 17:06:32 +01:00
|
|
|
let alarm = if result.state != "succeeded" {
|
|
|
|
|
Some(restart_failure_alarm(&result, &now))
|
|
|
|
|
} else {
|
|
|
|
|
None
|
|
|
|
|
};
|
|
|
|
|
with_store_node(options, store_lock, |node| {
|
|
|
|
|
node.put(&result.result_id, &result)?;
|
|
|
|
|
if let Some(alarm) = &alarm {
|
|
|
|
|
node.put(&alarm.alarm_id, alarm)?;
|
|
|
|
|
}
|
|
|
|
|
Ok(())
|
|
|
|
|
})?;
|
2026-06-04 21:07:42 +01:00
|
|
|
println!(
|
|
|
|
|
"Idunn restart {} for {}: {}",
|
|
|
|
|
result.state, result.daemon_id, result.detail
|
|
|
|
|
);
|
2026-06-15 17:06:32 +01:00
|
|
|
if let Some(alarm) = alarm {
|
2026-06-07 02:43:24 +01:00
|
|
|
println!(
|
|
|
|
|
"Idunn raised operator alarm for {} through {}: {}",
|
|
|
|
|
alarm.daemon_id, alarm.escalation_target, alarm.reason
|
|
|
|
|
);
|
2026-06-11 16:41:21 +01:00
|
|
|
run_operator_alarm_command(options, &alarm);
|
2026-06-07 02:43:24 +01:00
|
|
|
}
|
2026-06-04 21:07:42 +01:00
|
|
|
} else {
|
|
|
|
|
println!(
|
|
|
|
|
"Idunn requested restart for {} but did not execute it. Pass --execute to actuate.",
|
|
|
|
|
request.daemon_id
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if let Some(alarm) = &plan.operator_alarm {
|
|
|
|
|
println!(
|
|
|
|
|
"Idunn raised operator alarm for {} through {}: {}",
|
|
|
|
|
alarm.daemon_id, alarm.escalation_target, alarm.reason
|
|
|
|
|
);
|
2026-06-07 04:21:41 +01:00
|
|
|
run_operator_alarm_command(options, alarm);
|
2026-06-04 21:07:42 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
println!(
|
|
|
|
|
"Idunn decision for {}: {} ({})",
|
|
|
|
|
plan.decision.daemon_id, plan.decision.action, plan.decision.reason
|
|
|
|
|
);
|
|
|
|
|
Ok(())
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-15 20:50:57 +01:00
|
|
|
fn with_store_node<F>(options: &CommonOptions, store_lock: &Arc<Mutex<()>>, write: F) -> Result<()>
|
2026-06-15 17:06:32 +01:00
|
|
|
where
|
|
|
|
|
F: FnOnce(&mut CultMeshNode) -> Result<()>,
|
|
|
|
|
{
|
|
|
|
|
let _store_guard = store_lock
|
|
|
|
|
.lock()
|
|
|
|
|
.map_err(|_| anyhow!("Idunn store lock is poisoned"))?;
|
|
|
|
|
let mut node = CultMesh::create_node(
|
|
|
|
|
&options.store_path,
|
|
|
|
|
OdinDocuments,
|
|
|
|
|
CultMeshNodeOptions {
|
|
|
|
|
runtime_id: "idunn-daemon".to_string(),
|
|
|
|
|
pull_on_start: true,
|
|
|
|
|
},
|
|
|
|
|
)?;
|
|
|
|
|
write(&mut node)
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-11 16:41:21 +01:00
|
|
|
fn swarm_targets(options: &SwarmOptions) -> Result<Vec<DaemonTarget>> {
|
|
|
|
|
let repo_root = options.repo_root.display().to_string();
|
|
|
|
|
let script = |name: &str| format!(r"{}\scripts\{}", repo_root, name);
|
|
|
|
|
|
|
|
|
|
match options.profile.as_str() {
|
|
|
|
|
"starfire-local" => Ok(vec![
|
|
|
|
|
DaemonTarget {
|
|
|
|
|
daemon_id: "odin".to_string(),
|
|
|
|
|
verse_id: "starfire.local".to_string(),
|
|
|
|
|
name: "Odin all-seer".to_string(),
|
2026-06-15 21:09:37 +01:00
|
|
|
health_contract: health_contract("odin.cultnet-rudp-provider-health", "failed"),
|
2026-06-11 16:41:21 +01:00
|
|
|
health_command: Some(script("health-odin.cmd")),
|
|
|
|
|
deploy_command: None,
|
|
|
|
|
restart_command: Some(script("restart-odin.cmd")),
|
|
|
|
|
enabled: true,
|
|
|
|
|
interval_seconds: 30,
|
|
|
|
|
},
|
|
|
|
|
DaemonTarget {
|
|
|
|
|
daemon_id: "mimir-eve-dashboard".to_string(),
|
|
|
|
|
verse_id: "starfire.local".to_string(),
|
|
|
|
|
name: "Mimir Eve dashboard".to_string(),
|
2026-06-15 21:09:37 +01:00
|
|
|
health_contract: health_contract("mimir.cultnet-rudp-provider-health", "failed"),
|
2026-06-11 16:41:21 +01:00
|
|
|
health_command: Some(script("health-mimir-eve-dashboard.cmd")),
|
|
|
|
|
deploy_command: None,
|
|
|
|
|
restart_command: None,
|
|
|
|
|
enabled: true,
|
|
|
|
|
interval_seconds: 30,
|
|
|
|
|
},
|
|
|
|
|
DaemonTarget {
|
|
|
|
|
daemon_id: "stonks".to_string(),
|
|
|
|
|
verse_id: "starfire.local".to_string(),
|
|
|
|
|
name: "Stonks market pulse".to_string(),
|
2026-06-15 21:09:37 +01:00
|
|
|
health_contract: health_contract("stonks.cultnet-rudp-market-health", "failed"),
|
2026-06-11 16:41:21 +01:00
|
|
|
health_command: Some(script("health-stonks.cmd")),
|
|
|
|
|
deploy_command: None,
|
|
|
|
|
restart_command: Some(script("restart-stonks.cmd")),
|
|
|
|
|
enabled: true,
|
|
|
|
|
interval_seconds: 30,
|
|
|
|
|
},
|
|
|
|
|
DaemonTarget {
|
|
|
|
|
daemon_id: "voidbot".to_string(),
|
|
|
|
|
verse_id: "starfire.local".to_string(),
|
|
|
|
|
name: "VoidBot local stack".to_string(),
|
2026-06-15 21:09:37 +01:00
|
|
|
health_contract: health_contract("voidbot.cultnet-rudp-stack-health", "failed"),
|
2026-06-11 16:41:21 +01:00
|
|
|
health_command: Some(script("health-voidbot.cmd")),
|
|
|
|
|
deploy_command: None,
|
|
|
|
|
restart_command: Some(script("restart-voidbot.cmd")),
|
|
|
|
|
enabled: true,
|
|
|
|
|
interval_seconds: 300,
|
|
|
|
|
},
|
2026-06-11 20:27:58 +01:00
|
|
|
DaemonTarget {
|
|
|
|
|
daemon_id: "weksa".to_string(),
|
|
|
|
|
verse_id: "starfire.local".to_string(),
|
|
|
|
|
name: "Weksa intent and utterance lowering service".to_string(),
|
2026-06-15 21:09:37 +01:00
|
|
|
health_contract: health_contract("weksa.cultnet-rudp-provider-health", "failed"),
|
2026-06-11 20:27:58 +01:00
|
|
|
health_command: Some(script("health-weksa.cmd")),
|
|
|
|
|
deploy_command: None,
|
|
|
|
|
restart_command: Some(script("restart-weksa.cmd")),
|
|
|
|
|
enabled: true,
|
|
|
|
|
interval_seconds: 60,
|
|
|
|
|
},
|
2026-06-12 16:09:03 +01:00
|
|
|
DaemonTarget {
|
|
|
|
|
daemon_id: "starfire-muninn".to_string(),
|
|
|
|
|
verse_id: "starfire.local".to_string(),
|
|
|
|
|
name: "Starfire Muninn telemetry and Quest access daemon".to_string(),
|
2026-06-15 20:50:57 +01:00
|
|
|
health_contract: health_contract(
|
2026-06-15 21:09:37 +01:00
|
|
|
"muninn.cultnet-rudp-local-telemetry-and-quest-access",
|
2026-06-15 20:50:57 +01:00
|
|
|
"failed",
|
|
|
|
|
),
|
2026-06-12 16:09:03 +01:00
|
|
|
health_command: Some(script("health-starfire-muninn.cmd")),
|
|
|
|
|
deploy_command: None,
|
|
|
|
|
restart_command: Some(script("restart-starfire-muninn.cmd")),
|
|
|
|
|
enabled: true,
|
|
|
|
|
interval_seconds: 30,
|
|
|
|
|
},
|
2026-06-11 16:41:21 +01:00
|
|
|
DaemonTarget {
|
|
|
|
|
daemon_id: "muninn".to_string(),
|
|
|
|
|
verse_id: "raven.local".to_string(),
|
|
|
|
|
name: "Muninn telemetry Verse assembler".to_string(),
|
2026-06-15 21:09:37 +01:00
|
|
|
health_contract: health_contract("muninn.cultnet-rudp-remote-telemetry-health", "failed"),
|
2026-06-11 16:41:21 +01:00
|
|
|
health_command: Some(script("health-muninn.cmd")),
|
|
|
|
|
deploy_command: None,
|
|
|
|
|
restart_command: Some(script("restart-muninn.cmd")),
|
|
|
|
|
enabled: true,
|
|
|
|
|
interval_seconds: 30,
|
|
|
|
|
},
|
2026-06-12 12:56:40 +01:00
|
|
|
DaemonTarget {
|
|
|
|
|
daemon_id: "vili".to_string(),
|
|
|
|
|
verse_id: "raven.local".to_string(),
|
|
|
|
|
name: "Vili Persona animation daemon".to_string(),
|
2026-06-15 21:09:37 +01:00
|
|
|
health_contract: health_contract("vili.cultnet-rudp-animation-health", "failed"),
|
2026-06-12 12:56:40 +01:00
|
|
|
health_command: Some(script("health-vili.cmd")),
|
|
|
|
|
deploy_command: None,
|
|
|
|
|
restart_command: Some(script("restart-vili.cmd")),
|
|
|
|
|
enabled: true,
|
|
|
|
|
interval_seconds: 60,
|
|
|
|
|
},
|
2026-06-11 16:41:21 +01:00
|
|
|
DaemonTarget {
|
|
|
|
|
daemon_id: "idunn-swarm-deployment-coverage".to_string(),
|
|
|
|
|
verse_id: "starfire.local".to_string(),
|
|
|
|
|
name: "Idunn swarm deployment coverage".to_string(),
|
2026-06-15 20:50:57 +01:00
|
|
|
health_contract: health_contract("idunn.deployment-catalog-coherence", "degraded"),
|
2026-06-11 16:41:21 +01:00
|
|
|
health_command: Some(script("health-idunn-swarm-deployment-coverage.cmd")),
|
|
|
|
|
deploy_command: None,
|
|
|
|
|
restart_command: None,
|
|
|
|
|
enabled: true,
|
|
|
|
|
interval_seconds: 30,
|
|
|
|
|
},
|
|
|
|
|
DaemonTarget {
|
|
|
|
|
daemon_id: "yggdrasil-heimdall".to_string(),
|
|
|
|
|
verse_id: "yggdrasil.local".to_string(),
|
|
|
|
|
name: "Yggdrasil Heimdall".to_string(),
|
2026-06-15 20:50:57 +01:00
|
|
|
health_contract: health_contract(
|
|
|
|
|
"yggdrasil.source-deployment-freshness",
|
|
|
|
|
"stale-deployment",
|
|
|
|
|
),
|
2026-06-11 16:41:21 +01:00
|
|
|
health_command: Some(script("health-yggdrasil-heimdall.cmd")),
|
|
|
|
|
deploy_command: Some(script("deploy-yggdrasil-heimdall.cmd")),
|
|
|
|
|
restart_command: None,
|
|
|
|
|
enabled: true,
|
|
|
|
|
interval_seconds: 300,
|
|
|
|
|
},
|
|
|
|
|
DaemonTarget {
|
|
|
|
|
daemon_id: "yggdrasil-repixelizer".to_string(),
|
|
|
|
|
verse_id: "yggdrasil.local".to_string(),
|
|
|
|
|
name: "Yggdrasil Repixelizer".to_string(),
|
2026-06-15 20:50:57 +01:00
|
|
|
health_contract: health_contract(
|
|
|
|
|
"yggdrasil.source-deployment-freshness",
|
|
|
|
|
"stale-deployment",
|
|
|
|
|
),
|
2026-06-11 16:41:21 +01:00
|
|
|
health_command: Some(script("health-yggdrasil-repixelizer.cmd")),
|
|
|
|
|
deploy_command: Some(script("deploy-yggdrasil-repixelizer.cmd")),
|
|
|
|
|
restart_command: None,
|
|
|
|
|
enabled: true,
|
|
|
|
|
interval_seconds: 300,
|
|
|
|
|
},
|
|
|
|
|
DaemonTarget {
|
|
|
|
|
daemon_id: "yggdrasil-streampixels".to_string(),
|
|
|
|
|
verse_id: "yggdrasil.local".to_string(),
|
|
|
|
|
name: "Yggdrasil StreamPixels".to_string(),
|
2026-06-15 20:50:57 +01:00
|
|
|
health_contract: health_contract(
|
|
|
|
|
"yggdrasil.source-deployment-freshness",
|
|
|
|
|
"stale-deployment",
|
|
|
|
|
),
|
2026-06-11 16:41:21 +01:00
|
|
|
health_command: Some(script("health-yggdrasil-streampixels.cmd")),
|
|
|
|
|
deploy_command: Some(script("deploy-yggdrasil-streampixels.cmd")),
|
|
|
|
|
restart_command: None,
|
|
|
|
|
enabled: true,
|
|
|
|
|
interval_seconds: 300,
|
|
|
|
|
},
|
|
|
|
|
DaemonTarget {
|
|
|
|
|
daemon_id: "nightwing-gjallar".to_string(),
|
|
|
|
|
verse_id: "nightwing.local".to_string(),
|
|
|
|
|
name: "Nightwing Gjallar framebuffer compositor".to_string(),
|
2026-06-15 20:50:57 +01:00
|
|
|
health_contract: health_contract(
|
2026-06-15 21:09:37 +01:00
|
|
|
"gjallar.cultnet-rudp-framebuffer-composition-health",
|
2026-06-15 20:50:57 +01:00
|
|
|
"dependency-unavailable",
|
|
|
|
|
),
|
2026-06-11 16:41:21 +01:00
|
|
|
health_command: Some(script("health-nightwing-gjallar.cmd")),
|
|
|
|
|
deploy_command: Some(script("deploy-nightwing-gjallar.cmd")),
|
|
|
|
|
restart_command: Some(script("restart-nightwing-gjallar.cmd")),
|
|
|
|
|
enabled: true,
|
|
|
|
|
interval_seconds: 30,
|
|
|
|
|
},
|
|
|
|
|
DaemonTarget {
|
|
|
|
|
daemon_id: "nightwing-muninn".to_string(),
|
|
|
|
|
verse_id: "nightwing.local".to_string(),
|
|
|
|
|
name: "Nightwing Muninn telemetry and Move HID daemon".to_string(),
|
2026-06-15 21:09:37 +01:00
|
|
|
health_contract: health_contract(
|
|
|
|
|
"muninn.cultnet-rudp-remote-telemetry-and-move-hid",
|
|
|
|
|
"failed",
|
|
|
|
|
),
|
2026-06-11 16:41:21 +01:00
|
|
|
health_command: Some(script("health-nightwing-muninn.cmd")),
|
|
|
|
|
deploy_command: None,
|
|
|
|
|
restart_command: Some(script("restart-nightwing-muninn.cmd")),
|
|
|
|
|
enabled: true,
|
|
|
|
|
interval_seconds: 30,
|
|
|
|
|
},
|
|
|
|
|
DaemonTarget {
|
|
|
|
|
daemon_id: "nightwing-eve-dashboard".to_string(),
|
|
|
|
|
verse_id: "nightwing.local".to_string(),
|
|
|
|
|
name: "Nightwing Eve dashboard broker".to_string(),
|
2026-06-15 21:09:37 +01:00
|
|
|
health_contract: health_contract("nightwing.cultnet-rudp-eve-dashboard-health", "failed"),
|
2026-06-11 16:41:21 +01:00
|
|
|
health_command: Some(script("health-nightwing-eve-dashboard.cmd")),
|
|
|
|
|
deploy_command: None,
|
|
|
|
|
restart_command: Some(script("restart-nightwing-eve-dashboard.cmd")),
|
|
|
|
|
enabled: true,
|
|
|
|
|
interval_seconds: 30,
|
|
|
|
|
},
|
|
|
|
|
DaemonTarget {
|
|
|
|
|
daemon_id: "nightwing-eve-browser-reference".to_string(),
|
|
|
|
|
verse_id: "nightwing.local".to_string(),
|
|
|
|
|
name: "Nightwing Eve browser reference".to_string(),
|
2026-06-15 21:09:37 +01:00
|
|
|
health_contract: health_contract("nightwing.cultnet-rudp-browser-reference-health", "failed"),
|
2026-06-11 16:41:21 +01:00
|
|
|
health_command: Some(script("health-nightwing-eve-browser-reference.cmd")),
|
|
|
|
|
deploy_command: None,
|
|
|
|
|
restart_command: Some(script("restart-nightwing-eve-browser-reference.cmd")),
|
|
|
|
|
enabled: true,
|
|
|
|
|
interval_seconds: 30,
|
|
|
|
|
},
|
|
|
|
|
]),
|
|
|
|
|
other => Err(anyhow!("unknown Idunn swarm profile: {other}")),
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-07 02:43:24 +01:00
|
|
|
fn restart_failure_alarm(
|
|
|
|
|
result: &IdunnRestartResultRecord,
|
|
|
|
|
raised_at: &str,
|
|
|
|
|
) -> IdunnOperatorAlarmRecord {
|
|
|
|
|
IdunnOperatorAlarmRecord {
|
|
|
|
|
alarm_id: format!("alarm:restart-failed:{}:{}", result.daemon_id, raised_at),
|
|
|
|
|
daemon_id: result.daemon_id.clone(),
|
|
|
|
|
severity: "operator-action-required".to_string(),
|
|
|
|
|
reason: format!(
|
|
|
|
|
"restart command failed for request {}: {}",
|
|
|
|
|
result.request_id, result.detail
|
|
|
|
|
),
|
|
|
|
|
escalation_target: "bifrost.operator-notification".to_string(),
|
|
|
|
|
raised_at: raised_at.to_string(),
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-09 15:32:20 +01:00
|
|
|
fn deployment_failure_alarm(
|
|
|
|
|
result: &IdunnDeploymentResultRecord,
|
|
|
|
|
raised_at: &str,
|
|
|
|
|
) -> IdunnOperatorAlarmRecord {
|
|
|
|
|
IdunnOperatorAlarmRecord {
|
|
|
|
|
alarm_id: format!("alarm:deployment-failed:{}:{}", result.daemon_id, raised_at),
|
|
|
|
|
daemon_id: result.daemon_id.clone(),
|
|
|
|
|
severity: "operator-action-required".to_string(),
|
|
|
|
|
reason: format!(
|
|
|
|
|
"deployment command failed for request {}: {}",
|
|
|
|
|
result.request_id, result.detail
|
|
|
|
|
),
|
|
|
|
|
escalation_target: "bifrost.operator-notification".to_string(),
|
|
|
|
|
raised_at: raised_at.to_string(),
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-04 21:07:42 +01:00
|
|
|
impl Options {
|
|
|
|
|
fn parse(args: impl Iterator<Item = String>) -> Result<Self> {
|
2026-06-11 16:41:21 +01:00
|
|
|
let mut store_path = PathBuf::from("scratch/idunn/idunn.keepalive.cc");
|
|
|
|
|
let mut operator_alarm_command = None;
|
|
|
|
|
let mut execute = false;
|
|
|
|
|
let mut command_timeout_seconds = 30;
|
|
|
|
|
let mut daemon_id = None;
|
|
|
|
|
let mut verse_id = "local".to_string();
|
|
|
|
|
let mut name = None;
|
|
|
|
|
let mut health_command = None;
|
|
|
|
|
let mut deploy_command = None;
|
|
|
|
|
let mut restart_command = None;
|
|
|
|
|
let mut enabled = true;
|
|
|
|
|
let mut interval_seconds = None;
|
|
|
|
|
let mut swarm_profile = None;
|
|
|
|
|
let mut repo_root = env::current_dir().context("determining current directory")?;
|
2026-06-04 21:07:42 +01:00
|
|
|
|
|
|
|
|
let mut args = args.peekable();
|
|
|
|
|
while let Some(arg) = args.next() {
|
|
|
|
|
match arg.as_str() {
|
2026-06-11 16:41:21 +01:00
|
|
|
"--store" => store_path = PathBuf::from(take_value(&mut args, "--store")?),
|
|
|
|
|
"--daemon" => daemon_id = Some(take_value(&mut args, "--daemon")?),
|
|
|
|
|
"--verse" => verse_id = take_value(&mut args, "--verse")?,
|
|
|
|
|
"--name" => name = Some(take_value(&mut args, "--name")?),
|
2026-06-04 21:07:42 +01:00
|
|
|
"--health-command" => {
|
2026-06-11 16:41:21 +01:00
|
|
|
health_command = Some(take_value(&mut args, "--health-command")?)
|
2026-06-04 21:07:42 +01:00
|
|
|
}
|
2026-06-09 15:32:20 +01:00
|
|
|
"--deploy-command" => {
|
2026-06-11 16:41:21 +01:00
|
|
|
deploy_command = Some(take_value(&mut args, "--deploy-command")?)
|
2026-06-09 15:32:20 +01:00
|
|
|
}
|
2026-06-04 21:07:42 +01:00
|
|
|
"--restart-command" => {
|
2026-06-11 16:41:21 +01:00
|
|
|
restart_command = Some(take_value(&mut args, "--restart-command")?)
|
2026-06-04 21:07:42 +01:00
|
|
|
}
|
2026-06-07 04:21:41 +01:00
|
|
|
"--operator-alarm-command" => {
|
2026-06-11 16:41:21 +01:00
|
|
|
operator_alarm_command =
|
2026-06-07 04:21:41 +01:00
|
|
|
Some(take_value(&mut args, "--operator-alarm-command")?)
|
|
|
|
|
}
|
2026-06-11 16:41:21 +01:00
|
|
|
"--disabled" => enabled = false,
|
|
|
|
|
"--execute" => execute = true,
|
2026-06-04 21:07:42 +01:00
|
|
|
"--interval-seconds" => {
|
2026-06-11 16:41:21 +01:00
|
|
|
interval_seconds = Some(
|
2026-06-04 21:07:42 +01:00
|
|
|
take_value(&mut args, "--interval-seconds")?
|
|
|
|
|
.parse()
|
|
|
|
|
.context("--interval-seconds must be a positive integer")?,
|
|
|
|
|
)
|
|
|
|
|
}
|
2026-06-11 16:41:21 +01:00
|
|
|
"--command-timeout-seconds" => {
|
|
|
|
|
command_timeout_seconds = take_value(&mut args, "--command-timeout-seconds")?
|
|
|
|
|
.parse()
|
|
|
|
|
.context("--command-timeout-seconds must be a positive integer")?
|
|
|
|
|
}
|
|
|
|
|
"--swarm-profile" => {
|
|
|
|
|
swarm_profile = Some(take_value(&mut args, "--swarm-profile")?)
|
|
|
|
|
}
|
|
|
|
|
"--repo-root" => repo_root = PathBuf::from(take_value(&mut args, "--repo-root")?),
|
2026-06-04 21:07:42 +01:00
|
|
|
"--help" | "-h" => return Err(anyhow!(help_text())),
|
|
|
|
|
other => {
|
|
|
|
|
return Err(anyhow!(
|
|
|
|
|
"unknown Idunn argument: {other}\n\n{}",
|
|
|
|
|
help_text()
|
|
|
|
|
));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-11 16:41:21 +01:00
|
|
|
if command_timeout_seconds == 0 {
|
|
|
|
|
return Err(anyhow!(
|
|
|
|
|
"--command-timeout-seconds must be greater than zero"
|
|
|
|
|
));
|
2026-06-04 21:07:42 +01:00
|
|
|
}
|
|
|
|
|
|
2026-06-11 16:41:21 +01:00
|
|
|
let common = CommonOptions {
|
|
|
|
|
store_path,
|
|
|
|
|
operator_alarm_command,
|
|
|
|
|
execute,
|
|
|
|
|
command_timeout_seconds,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
let mode = match (swarm_profile, daemon_id) {
|
|
|
|
|
(Some(profile), None) => Mode::Swarm(SwarmOptions { profile, repo_root }),
|
|
|
|
|
(Some(_), Some(_)) => {
|
|
|
|
|
return Err(anyhow!(
|
|
|
|
|
"use either --swarm-profile or --daemon, not both\n\n{}",
|
|
|
|
|
help_text()
|
|
|
|
|
));
|
|
|
|
|
}
|
|
|
|
|
(None, Some(daemon_id)) => {
|
|
|
|
|
let interval_seconds = interval_seconds.unwrap_or(30);
|
|
|
|
|
if interval_seconds == 0 {
|
|
|
|
|
return Err(anyhow!("--interval-seconds must be greater than zero"));
|
|
|
|
|
}
|
|
|
|
|
Mode::Single(DaemonTarget {
|
|
|
|
|
daemon_id: daemon_id.clone(),
|
|
|
|
|
verse_id,
|
|
|
|
|
name: name.unwrap_or(daemon_id),
|
2026-06-15 20:50:57 +01:00
|
|
|
health_contract: health_contract("manual.command-health", "failed"),
|
2026-06-11 16:41:21 +01:00
|
|
|
health_command,
|
|
|
|
|
deploy_command,
|
|
|
|
|
restart_command,
|
|
|
|
|
enabled,
|
|
|
|
|
interval_seconds,
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
(None, None) => {
|
|
|
|
|
return Err(anyhow!(
|
|
|
|
|
"either --daemon or --swarm-profile is required\n\n{}",
|
|
|
|
|
help_text()
|
|
|
|
|
));
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
Ok(Self { common, mode })
|
2026-06-04 21:07:42 +01:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn take_value(args: &mut impl Iterator<Item = String>, name: &str) -> Result<String> {
|
|
|
|
|
args.next()
|
|
|
|
|
.ok_or_else(|| anyhow!("{name} requires a value"))
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-11 16:41:21 +01:00
|
|
|
fn probe_health(
|
|
|
|
|
target: &DaemonTarget,
|
|
|
|
|
command_timeout_seconds: u64,
|
|
|
|
|
observed_at: &str,
|
|
|
|
|
) -> IdunnDaemonHealthRecord {
|
|
|
|
|
match &target.health_command {
|
|
|
|
|
Some(command) => match run_shell(command, command_timeout_seconds) {
|
2026-06-04 21:07:42 +01:00
|
|
|
Ok(output) if output.status.success() => IdunnDaemonHealthRecord {
|
2026-06-11 16:41:21 +01:00
|
|
|
daemon_id: target.daemon_id.clone(),
|
2026-06-04 21:07:42 +01:00
|
|
|
state: "active".to_string(),
|
2026-06-15 17:06:32 +01:00
|
|
|
detail: command_output_detail("health command exited successfully", &output),
|
2026-06-15 20:50:57 +01:00
|
|
|
health_contract: target.health_contract.id.clone(),
|
2026-06-04 21:07:42 +01:00
|
|
|
observed_at: observed_at.to_string(),
|
|
|
|
|
},
|
2026-06-15 19:44:48 +01:00
|
|
|
Ok(output) => {
|
|
|
|
|
let detail = command_output_detail(
|
2026-06-15 17:06:32 +01:00
|
|
|
&format!("health command exited with {}", output.status),
|
|
|
|
|
&output,
|
2026-06-15 19:44:48 +01:00
|
|
|
);
|
|
|
|
|
IdunnDaemonHealthRecord {
|
|
|
|
|
daemon_id: target.daemon_id.clone(),
|
2026-06-15 20:50:57 +01:00
|
|
|
state: health_state_from_detail(&detail, target).to_string(),
|
2026-06-15 19:44:48 +01:00
|
|
|
detail,
|
2026-06-15 20:50:57 +01:00
|
|
|
health_contract: target.health_contract.id.clone(),
|
2026-06-15 19:44:48 +01:00
|
|
|
observed_at: observed_at.to_string(),
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-06-04 21:07:42 +01:00
|
|
|
Err(error) => IdunnDaemonHealthRecord {
|
2026-06-11 16:41:21 +01:00
|
|
|
daemon_id: target.daemon_id.clone(),
|
2026-06-04 21:07:42 +01:00
|
|
|
state: "failed".to_string(),
|
|
|
|
|
detail: format!("health command could not run: {error}"),
|
2026-06-15 20:50:57 +01:00
|
|
|
health_contract: target.health_contract.id.clone(),
|
2026-06-04 21:07:42 +01:00
|
|
|
observed_at: observed_at.to_string(),
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
None => IdunnDaemonHealthRecord {
|
2026-06-11 16:41:21 +01:00
|
|
|
daemon_id: target.daemon_id.clone(),
|
2026-06-04 21:07:42 +01:00
|
|
|
state: "unknown".to_string(),
|
|
|
|
|
detail: "no health command was provided".to_string(),
|
2026-06-15 20:50:57 +01:00
|
|
|
health_contract: target.health_contract.id.clone(),
|
2026-06-04 21:07:42 +01:00
|
|
|
observed_at: observed_at.to_string(),
|
|
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-15 20:50:57 +01:00
|
|
|
fn health_state_from_detail<'a>(detail: &'a str, target: &'a DaemonTarget) -> &'a str {
|
2026-06-15 19:44:48 +01:00
|
|
|
for state in [
|
|
|
|
|
"stale-deployment",
|
|
|
|
|
"dependency-unavailable",
|
|
|
|
|
"degraded",
|
|
|
|
|
"failed",
|
|
|
|
|
] {
|
|
|
|
|
let marker = format!("idunn.health.state={state}");
|
|
|
|
|
if detail.contains(&marker) {
|
|
|
|
|
return state;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-15 20:50:57 +01:00
|
|
|
&target.health_contract.default_failure_state
|
2026-06-15 19:44:48 +01:00
|
|
|
}
|
|
|
|
|
|
2026-06-04 21:07:42 +01:00
|
|
|
fn run_restart(
|
|
|
|
|
request: &odin_core::IdunnRestartRequestRecord,
|
|
|
|
|
requested_at: &str,
|
2026-06-11 16:41:21 +01:00
|
|
|
command_timeout_seconds: u64,
|
2026-06-04 21:07:42 +01:00
|
|
|
) -> IdunnRestartResultRecord {
|
|
|
|
|
let result_id = format!("result:{}", request.request_id);
|
2026-06-11 16:41:21 +01:00
|
|
|
match run_shell(&request.command, command_timeout_seconds) {
|
2026-06-04 21:07:42 +01:00
|
|
|
Ok(output) if output.status.success() => IdunnRestartResultRecord {
|
|
|
|
|
result_id,
|
|
|
|
|
request_id: request.request_id.clone(),
|
|
|
|
|
daemon_id: request.daemon_id.clone(),
|
|
|
|
|
state: "succeeded".to_string(),
|
2026-06-15 17:06:32 +01:00
|
|
|
detail: command_output_detail("restart command exited successfully", &output),
|
2026-06-04 21:07:42 +01:00
|
|
|
completed_at: requested_at.to_string(),
|
|
|
|
|
},
|
|
|
|
|
Ok(output) => IdunnRestartResultRecord {
|
|
|
|
|
result_id,
|
|
|
|
|
request_id: request.request_id.clone(),
|
|
|
|
|
daemon_id: request.daemon_id.clone(),
|
|
|
|
|
state: "failed".to_string(),
|
2026-06-15 17:06:32 +01:00
|
|
|
detail: command_output_detail(
|
|
|
|
|
&format!("restart command exited with {}", output.status),
|
|
|
|
|
&output,
|
|
|
|
|
),
|
2026-06-04 21:07:42 +01:00
|
|
|
completed_at: requested_at.to_string(),
|
|
|
|
|
},
|
|
|
|
|
Err(error) => IdunnRestartResultRecord {
|
|
|
|
|
result_id,
|
|
|
|
|
request_id: request.request_id.clone(),
|
|
|
|
|
daemon_id: request.daemon_id.clone(),
|
|
|
|
|
state: "failed".to_string(),
|
|
|
|
|
detail: format!("restart command could not run: {error}"),
|
|
|
|
|
completed_at: requested_at.to_string(),
|
|
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-09 15:32:20 +01:00
|
|
|
fn run_deployment(
|
|
|
|
|
request: &odin_core::IdunnDeploymentRequestRecord,
|
|
|
|
|
requested_at: &str,
|
2026-06-11 16:41:21 +01:00
|
|
|
command_timeout_seconds: u64,
|
2026-06-09 15:32:20 +01:00
|
|
|
) -> IdunnDeploymentResultRecord {
|
|
|
|
|
let result_id = format!("result:{}", request.request_id);
|
2026-06-11 16:41:21 +01:00
|
|
|
match run_shell(&request.command, command_timeout_seconds) {
|
2026-06-09 15:32:20 +01:00
|
|
|
Ok(output) if output.status.success() => IdunnDeploymentResultRecord {
|
|
|
|
|
result_id,
|
|
|
|
|
request_id: request.request_id.clone(),
|
|
|
|
|
daemon_id: request.daemon_id.clone(),
|
|
|
|
|
state: "succeeded".to_string(),
|
2026-06-15 17:06:32 +01:00
|
|
|
detail: command_output_detail("deployment command exited successfully", &output),
|
2026-06-09 15:32:20 +01:00
|
|
|
completed_at: requested_at.to_string(),
|
|
|
|
|
},
|
|
|
|
|
Ok(output) => IdunnDeploymentResultRecord {
|
|
|
|
|
result_id,
|
|
|
|
|
request_id: request.request_id.clone(),
|
|
|
|
|
daemon_id: request.daemon_id.clone(),
|
|
|
|
|
state: "failed".to_string(),
|
2026-06-15 17:06:32 +01:00
|
|
|
detail: command_output_detail(
|
|
|
|
|
&format!("deployment command exited with {}", output.status),
|
|
|
|
|
&output,
|
|
|
|
|
),
|
2026-06-09 15:32:20 +01:00
|
|
|
completed_at: requested_at.to_string(),
|
|
|
|
|
},
|
|
|
|
|
Err(error) => IdunnDeploymentResultRecord {
|
|
|
|
|
result_id,
|
|
|
|
|
request_id: request.request_id.clone(),
|
|
|
|
|
daemon_id: request.daemon_id.clone(),
|
|
|
|
|
state: "failed".to_string(),
|
|
|
|
|
detail: format!("deployment command could not run: {error}"),
|
|
|
|
|
completed_at: requested_at.to_string(),
|
|
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-15 17:06:32 +01:00
|
|
|
fn command_output_detail(prefix: &str, output: &std::process::Output) -> String {
|
|
|
|
|
let mut detail = prefix.to_string();
|
|
|
|
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
|
|
|
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
|
|
|
|
let stdout = stdout.trim();
|
|
|
|
|
let stderr = stderr.trim();
|
|
|
|
|
if !stdout.is_empty() {
|
|
|
|
|
detail.push_str("; stdout: ");
|
|
|
|
|
detail.push_str(&truncate_detail(stdout, 600));
|
|
|
|
|
}
|
|
|
|
|
if !stderr.is_empty() {
|
|
|
|
|
detail.push_str("; stderr: ");
|
|
|
|
|
detail.push_str(&truncate_detail(stderr, 600));
|
|
|
|
|
}
|
|
|
|
|
detail
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn truncate_detail(value: &str, max_chars: usize) -> String {
|
|
|
|
|
let mut chars = value.chars();
|
|
|
|
|
let truncated: String = chars.by_ref().take(max_chars).collect();
|
|
|
|
|
if chars.next().is_some() {
|
|
|
|
|
format!("{truncated}...")
|
|
|
|
|
} else {
|
|
|
|
|
truncated
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-11 16:41:21 +01:00
|
|
|
fn run_shell(command: &str, timeout_seconds: u64) -> Result<std::process::Output> {
|
|
|
|
|
let mut process = if cfg!(windows) {
|
2026-06-15 17:06:32 +01:00
|
|
|
let path = windows_command_path();
|
|
|
|
|
let command = format!(r#"set "PATH={path}" && set "Path={path}" && {command}"#);
|
2026-06-11 16:41:21 +01:00
|
|
|
let mut process = Command::new("cmd");
|
2026-06-15 17:06:32 +01:00
|
|
|
process.arg("/D").arg("/S").arg("/C").arg(command);
|
|
|
|
|
apply_windows_command_environment(&mut process, &path);
|
2026-06-11 16:41:21 +01:00
|
|
|
process
|
2026-06-04 21:07:42 +01:00
|
|
|
} else {
|
2026-06-11 16:41:21 +01:00
|
|
|
let mut process = Command::new("sh");
|
|
|
|
|
process.arg("-c").arg(command);
|
|
|
|
|
process
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
let child = process
|
|
|
|
|
.stdout(Stdio::piped())
|
|
|
|
|
.stderr(Stdio::piped())
|
|
|
|
|
.spawn()
|
|
|
|
|
.with_context(|| format!("running command {command:?}"))?;
|
|
|
|
|
wait_for_child_with_timeout(child, Duration::from_secs(timeout_seconds), command)
|
2026-06-04 21:07:42 +01:00
|
|
|
}
|
|
|
|
|
|
2026-06-11 16:41:21 +01:00
|
|
|
fn run_operator_alarm_command(options: &CommonOptions, alarm: &IdunnOperatorAlarmRecord) {
|
2026-06-07 04:21:41 +01:00
|
|
|
let Some(command) = options.operator_alarm_command.as_deref() else {
|
|
|
|
|
return;
|
|
|
|
|
};
|
|
|
|
|
if command.trim().is_empty() {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let mut process = if cfg!(windows) {
|
2026-06-15 17:06:32 +01:00
|
|
|
let path = windows_command_path();
|
|
|
|
|
let command = format!(r#"set "PATH={path}" && set "Path={path}" && {command}"#);
|
2026-06-07 04:21:41 +01:00
|
|
|
let mut process = Command::new("cmd");
|
2026-06-15 17:06:32 +01:00
|
|
|
process.arg("/D").arg("/S").arg("/C").arg(command);
|
|
|
|
|
apply_windows_command_environment(&mut process, &path);
|
2026-06-07 04:21:41 +01:00
|
|
|
process
|
|
|
|
|
} else {
|
|
|
|
|
let mut process = Command::new("sh");
|
|
|
|
|
process.arg("-c").arg(command);
|
|
|
|
|
process
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
let output = process
|
|
|
|
|
.env("IDUNN_ALARM_ID", &alarm.alarm_id)
|
|
|
|
|
.env("IDUNN_ALARM_DAEMON_ID", &alarm.daemon_id)
|
|
|
|
|
.env("IDUNN_ALARM_SEVERITY", &alarm.severity)
|
|
|
|
|
.env("IDUNN_ALARM_REASON", &alarm.reason)
|
|
|
|
|
.env("IDUNN_ALARM_ESCALATION_TARGET", &alarm.escalation_target)
|
|
|
|
|
.env("IDUNN_ALARM_RAISED_AT", &alarm.raised_at)
|
2026-06-11 16:41:21 +01:00
|
|
|
.stdout(Stdio::piped())
|
|
|
|
|
.stderr(Stdio::piped())
|
|
|
|
|
.spawn();
|
2026-06-07 04:21:41 +01:00
|
|
|
|
|
|
|
|
match output {
|
2026-06-11 16:41:21 +01:00
|
|
|
Ok(child) => match wait_for_child_with_timeout(
|
|
|
|
|
child,
|
|
|
|
|
Duration::from_secs(options.command_timeout_seconds),
|
|
|
|
|
command,
|
|
|
|
|
) {
|
|
|
|
|
Ok(output) if output.status.success() => {
|
|
|
|
|
println!(
|
|
|
|
|
"Idunn operator alarm command completed for {}.",
|
|
|
|
|
alarm.daemon_id
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
Ok(output) => {
|
|
|
|
|
eprintln!(
|
|
|
|
|
"Idunn operator alarm command failed for {} with {}.",
|
|
|
|
|
alarm.daemon_id, output.status
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
Err(error) => {
|
|
|
|
|
eprintln!(
|
|
|
|
|
"Idunn operator alarm command could not run for {}: {}",
|
|
|
|
|
alarm.daemon_id, error
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
},
|
2026-06-07 04:21:41 +01:00
|
|
|
Err(error) => {
|
|
|
|
|
eprintln!(
|
|
|
|
|
"Idunn operator alarm command could not run for {}: {}",
|
|
|
|
|
alarm.daemon_id, error
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-15 17:06:32 +01:00
|
|
|
fn windows_command_path() -> String {
|
|
|
|
|
[
|
|
|
|
|
r"C:\WINDOWS\system32",
|
|
|
|
|
r"C:\WINDOWS",
|
|
|
|
|
r"C:\WINDOWS\System32\Wbem",
|
|
|
|
|
r"C:\WINDOWS\System32\WindowsPowerShell\v1.0",
|
|
|
|
|
r"C:\WINDOWS\System32\OpenSSH",
|
|
|
|
|
r"C:\Program Files\Git\cmd",
|
|
|
|
|
r"C:\Program Files\nodejs",
|
|
|
|
|
r"C:\Program Files\Docker\Docker\resources\bin",
|
|
|
|
|
r"C:\Users\Meta\AppData\Local\Programs\Ollama",
|
|
|
|
|
r"C:\Program Files\dotnet",
|
|
|
|
|
r"C:\Users\Meta\.cargo\bin",
|
|
|
|
|
]
|
|
|
|
|
.join(";")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn apply_windows_command_environment(process: &mut Command, path: &str) {
|
|
|
|
|
if !cfg!(windows) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
process.env("PATH", path);
|
|
|
|
|
process.env("Path", path);
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-11 16:41:21 +01:00
|
|
|
fn wait_for_child_with_timeout(
|
|
|
|
|
mut child: Child,
|
|
|
|
|
timeout: Duration,
|
|
|
|
|
command: &str,
|
|
|
|
|
) -> Result<std::process::Output> {
|
|
|
|
|
let started_at = Instant::now();
|
|
|
|
|
|
|
|
|
|
loop {
|
|
|
|
|
if child
|
|
|
|
|
.try_wait()
|
|
|
|
|
.with_context(|| format!("waiting on command {command:?}"))?
|
|
|
|
|
.is_some()
|
|
|
|
|
{
|
|
|
|
|
return child
|
|
|
|
|
.wait_with_output()
|
|
|
|
|
.with_context(|| format!("collecting output for command {command:?}"));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if started_at.elapsed() >= timeout {
|
|
|
|
|
let _ = child.kill();
|
|
|
|
|
let _ = child.wait_with_output();
|
|
|
|
|
return Err(anyhow!(
|
|
|
|
|
"command timed out after {} seconds: {command:?}",
|
|
|
|
|
timeout.as_secs()
|
|
|
|
|
));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
thread::sleep(Duration::from_millis(100));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-04 21:07:42 +01:00
|
|
|
fn timestamp() -> Result<String> {
|
|
|
|
|
let seconds = SystemTime::now()
|
|
|
|
|
.duration_since(UNIX_EPOCH)
|
|
|
|
|
.context("system clock is before Unix epoch")?
|
|
|
|
|
.as_secs();
|
|
|
|
|
Ok(format!("unix:{seconds}"))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn help_text() -> &'static str {
|
2026-06-11 16:41:21 +01:00
|
|
|
"Usage: idunn --daemon <id> [--name <name>] [--verse <verse>] [--store <path>] [--health-command <command>] [--deploy-command <command>] [--restart-command <command>] [--operator-alarm-command <command>] [--execute] [--interval-seconds <seconds>] [--command-timeout-seconds <seconds>] [--repo-root <path>] [--swarm-profile <profile>]\n\nIdunn can run one manual daemon probe lane with --daemon, or one built-in swarm supervisor with --swarm-profile starfire-local."
|
2026-06-04 21:07:42 +01:00
|
|
|
}
|
2026-06-15 20:50:57 +01:00
|
|
|
|
|
|
|
|
#[cfg(test)]
|
|
|
|
|
mod tests {
|
|
|
|
|
use super::*;
|
|
|
|
|
|
|
|
|
|
fn target(default_failure_state: &str, deploy_command: Option<&str>) -> DaemonTarget {
|
|
|
|
|
DaemonTarget {
|
|
|
|
|
daemon_id: "test-daemon".to_string(),
|
|
|
|
|
verse_id: "test.local".to_string(),
|
|
|
|
|
name: "Test daemon".to_string(),
|
|
|
|
|
health_contract: health_contract("test.contract", default_failure_state),
|
|
|
|
|
health_command: Some("exit 0".to_string()),
|
|
|
|
|
deploy_command: deploy_command.map(ToString::to_string),
|
|
|
|
|
restart_command: Some("restart test".to_string()),
|
|
|
|
|
enabled: true,
|
|
|
|
|
interval_seconds: 30,
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn target_catalog_requires_health_contract() {
|
|
|
|
|
let mut target = target("failed", None);
|
|
|
|
|
target.health_contract.id = String::new();
|
|
|
|
|
|
|
|
|
|
let error = validate_targets(&[target]).unwrap_err().to_string();
|
|
|
|
|
|
|
|
|
|
assert!(error.contains("has no health contract"));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn stale_deployment_contract_requires_deploy_authority() {
|
|
|
|
|
let error = validate_targets(&[target("stale-deployment", None)])
|
|
|
|
|
.unwrap_err()
|
|
|
|
|
.to_string();
|
|
|
|
|
|
|
|
|
|
assert!(error.contains("has no deploy command"));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn health_state_uses_contract_default_when_probe_has_no_marker() {
|
|
|
|
|
let target = target("dependency-unavailable", None);
|
|
|
|
|
|
|
|
|
|
assert_eq!(
|
|
|
|
|
health_state_from_detail("plain command failure", &target),
|
|
|
|
|
"dependency-unavailable"
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|