2026-06-04 21:07:42 +01:00
use anyhow ::{ Context , Result , anyhow } ;
2026-06-15 17:06:32 +01:00
use cultmesh_rs ::{ CultMesh , CultMeshNode , CultMeshNodeOptions } ;
2026-06-15 22:35:00 +01:00
use cultnet_rs ::{
2026-06-15 22:50:43 +01:00
CULTNET_RUDP_PROTOCOL_ID , CultNetMessage , CultNetRawPayloadEncoding , CultNetRudpOptions ,
recv_cultnet_message_rudp , send_cultnet_message_rudp ,
2026-06-15 22:35:00 +01:00
} ;
2026-06-04 21:07:42 +01:00
use odin_core ::{
2026-06-15 22:04:53 +01:00
IdunnCommandBoundaryRecord , IdunnDaemonHealthRecord , IdunnDaemonSurgeryPlanRecord ,
IdunnDaemonTransportProfileRecord , IdunnDeploymentResultRecord , IdunnDesiredDaemonRecord ,
2026-06-15 22:50:43 +01:00
IdunnOperatorAlarmRecord , IdunnRestartResultRecord , IdunnRudpHealthIngressRecord ,
2026-06-15 23:36:52 +01:00
IdunnRuntimeTransportCheckRecord , IdunnSwarmSurgeryPlanRecord , OdinDocuments , plan_keepalive ,
2026-06-04 21:07:42 +01:00
} ;
use std ::env ;
2026-06-15 22:50:43 +01:00
use std ::net ::{ SocketAddr , UdpSocket } ;
2026-06-04 21:07:42 +01:00
use std ::path ::PathBuf ;
2026-06-11 16:41:21 +01:00
use std ::process ::{ Child , Command , Stdio } ;
2026-06-15 17:06:32 +01:00
use std ::sync ::{ Arc , Mutex } ;
2026-06-04 21:07:42 +01:00
use std ::thread ;
use std ::time ::Duration ;
2026-06-11 16:41:21 +01:00
use std ::time ::{ Instant , SystemTime , UNIX_EPOCH } ;
2026-06-04 21:07:42 +01:00
#[ derive(Clone, Debug) ]
2026-06-11 16:41:21 +01:00
struct DaemonTarget {
2026-06-04 21:07:42 +01:00
daemon_id : String ,
verse_id : String ,
name : String ,
2026-06-15 20:50:57 +01:00
health_contract : HealthContract ,
2026-06-04 21:07:42 +01:00
health_command : Option < String > ,
2026-06-09 15:32:20 +01:00
deploy_command : Option < String > ,
2026-06-04 21:07:42 +01:00
restart_command : Option < String > ,
enabled : bool ,
2026-06-11 16:41:21 +01:00
interval_seconds : u64 ,
}
2026-06-15 20:50:57 +01:00
#[ derive(Clone, Debug) ]
struct HealthContract {
id : String ,
default_failure_state : String ,
}
fn health_contract ( id : & str , default_failure_state : & str ) -> HealthContract {
HealthContract {
id : id . to_string ( ) ,
default_failure_state : default_failure_state . to_string ( ) ,
}
}
2026-06-11 16:41:21 +01:00
#[ derive(Clone, Debug) ]
struct CommonOptions {
store_path : PathBuf ,
operator_alarm_command : Option < String > ,
2026-06-15 22:50:43 +01:00
rudp_health_bind : Option < SocketAddr > ,
2026-06-04 21:07:42 +01:00
execute : bool ,
2026-06-11 16:41:21 +01:00
command_timeout_seconds : u64 ,
}
#[ derive(Clone, Debug) ]
enum Mode {
Single ( DaemonTarget ) ,
Swarm ( SwarmOptions ) ,
}
#[ derive(Clone, Debug) ]
struct SwarmOptions {
profile : String ,
repo_root : PathBuf ,
}
#[ derive(Clone, Debug) ]
struct Options {
common : CommonOptions ,
mode : Mode ,
2026-06-04 21:07:42 +01:00
}
fn main ( ) -> Result < ( ) > {
let options = Options ::parse ( env ::args ( ) . skip ( 1 ) ) ? ;
2026-06-11 16:41:21 +01:00
if let Some ( parent ) = options . common . store_path . parent ( ) {
2026-06-04 21:07:42 +01:00
std ::fs ::create_dir_all ( parent )
. with_context ( | | format! ( " creating {} " , parent . display ( ) ) ) ? ;
}
2026-06-11 16:41:21 +01:00
match & options . mode {
2026-06-15 17:06:32 +01:00
Mode ::Single ( target ) = > {
let store_lock = Arc ::new ( Mutex ::new ( ( ) ) ) ;
2026-06-15 22:35:00 +01:00
let now = timestamp ( ) ? ;
publish_runtime_transport_check ( & options . common , & store_lock , & now ) ? ;
2026-06-15 17:06:32 +01:00
run_target_cycle ( target , & options . common , & store_lock )
}
2026-06-11 16:41:21 +01:00
Mode ::Swarm ( swarm ) = > run_swarm ( swarm , & options . common ) ,
}
}
fn run_swarm ( options : & SwarmOptions , common : & CommonOptions ) -> Result < ( ) > {
let targets = swarm_targets ( options ) ? ;
if targets . is_empty ( ) {
return Err ( anyhow! (
" Idunn swarm profile {} resolved to no targets " ,
options . profile
) ) ;
}
2026-06-15 20:50:57 +01:00
validate_targets ( & targets ) ? ;
2026-06-11 16:41:21 +01:00
println! (
" Idunn swarm profile {} starting with {} targets. " ,
options . profile ,
targets . len ( )
) ;
println! ( " CultMesh store: {} " , common . store_path . display ( ) ) ;
2026-06-15 17:06:32 +01:00
let store_lock = Arc ::new ( Mutex ::new ( ( ) ) ) ;
2026-06-15 21:47:53 +01:00
let now = timestamp ( ) ? ;
2026-06-15 22:35:00 +01:00
publish_runtime_transport_check ( common , & store_lock , & now ) ? ;
2026-06-15 22:50:43 +01:00
start_rudp_health_ingress ( common , & store_lock , & now ) ? ;
2026-06-15 23:36:52 +01:00
publish_surgery_plans ( & options . profile , & targets , common , & store_lock , & now ) ? ;
2026-06-15 21:47:53 +01:00
2026-06-11 16:41:21 +01:00
let mut workers = Vec ::with_capacity ( targets . len ( ) ) ;
for target in targets {
let worker_common = common . clone ( ) ;
2026-06-15 17:06:32 +01:00
let worker_store_lock = Arc ::clone ( & store_lock ) ;
2026-06-11 16:41:21 +01:00
workers . push ( thread ::spawn ( move | | {
2026-06-15 17:06:32 +01:00
run_target_loop ( target , worker_common , worker_store_lock )
2026-06-11 16:41:21 +01:00
} ) ) ;
}
for worker in workers {
match worker . join ( ) {
Ok ( Ok ( ( ) ) ) = > { }
Ok ( Err ( error ) ) = > return Err ( error ) ,
Err ( _ ) = > return Err ( anyhow! ( " Idunn swarm worker thread panicked " ) ) ,
2026-06-04 21:07:42 +01:00
}
}
2026-06-11 16:41:21 +01:00
Ok ( ( ) )
}
2026-06-15 20:50:57 +01:00
fn validate_targets ( targets : & [ DaemonTarget ] ) -> Result < ( ) > {
let mut issues = Vec ::new ( ) ;
for target in targets {
if target . health_contract . id . trim ( ) . is_empty ( ) {
issues . push ( format! ( " {} has no health contract " , target . daemon_id ) ) ;
}
if target . health_command . is_none ( ) {
issues . push ( format! ( " {} has no health command " , target . daemon_id ) ) ;
}
if target . health_contract . default_failure_state = = " stale-deployment "
& & target . deploy_command . is_none ( )
{
issues . push ( format! (
" {} treats probe failure as stale deployment but has no deploy command " ,
target . daemon_id
) ) ;
}
}
if issues . is_empty ( ) {
Ok ( ( ) )
} else {
Err ( anyhow! (
" Idunn target catalog is incoherent: {} " ,
issues . join ( " ; " )
) )
}
}
2026-06-15 17:06:32 +01:00
fn run_target_loop (
target : DaemonTarget ,
options : CommonOptions ,
store_lock : Arc < Mutex < ( ) > > ,
) -> Result < ( ) > {
2026-06-11 16:41:21 +01:00
loop {
2026-06-15 17:06:32 +01:00
if let Err ( error ) = run_target_cycle ( & target , & options , & store_lock ) {
2026-06-11 16:41:21 +01:00
eprintln! (
" Idunn swarm target {} cycle failed: {} " ,
target . daemon_id , error
) ;
}
thread ::sleep ( Duration ::from_secs ( target . interval_seconds ) ) ;
}
2026-06-04 21:07:42 +01:00
}
2026-06-15 17:06:32 +01:00
fn run_target_cycle (
target : & DaemonTarget ,
options : & CommonOptions ,
store_lock : & Arc < Mutex < ( ) > > ,
) -> Result < ( ) > {
2026-06-04 21:07:42 +01:00
let now = timestamp ( ) ? ;
let desired = IdunnDesiredDaemonRecord {
2026-06-11 16:41:21 +01:00
daemon_id : target . daemon_id . clone ( ) ,
verse_id : target . verse_id . clone ( ) ,
name : target . name . clone ( ) ,
enabled : target . enabled ,
health_command : target . health_command . clone ( ) ,
restart_command : target . restart_command . clone ( ) ,
deploy_command : target . deploy_command . clone ( ) ,
2026-06-15 20:50:57 +01:00
health_contract : target . health_contract . id . clone ( ) ,
2026-06-15 22:04:53 +01:00
transport_profile_id : transport_profile_id ( target ) ,
command_boundary_id : command_boundary_id ( target ) ,
2026-06-04 21:07:42 +01:00
authority : " idunn.local-command " . to_string ( ) ,
max_silence_seconds : 60 ,
observed_at : now . clone ( ) ,
} ;
2026-06-15 23:12:39 +01:00
let health = read_fresh_daemon_published_health ( options , store_lock , & desired , & now ) ?
. unwrap_or_else ( | | probe_health ( target , options . command_timeout_seconds , & now ) ) ;
2026-06-04 21:07:42 +01:00
let plan = plan_keepalive ( & desired , & health , now . clone ( ) ) ;
2026-06-15 17:06:32 +01:00
with_store_node ( options , store_lock , | node | {
2026-06-15 22:04:53 +01:00
let transport_profile = daemon_transport_profile ( target , & now ) ;
let command_boundary = command_boundary ( target , & now ) ;
node . put ( & transport_profile . profile_id , & transport_profile ) ? ;
node . put ( & command_boundary . boundary_id , & command_boundary ) ? ;
2026-06-15 17:06:32 +01:00
node . put ( & desired . daemon_id , & desired ) ? ;
node . put ( & health . daemon_id , & health ) ? ;
node . put ( & plan . decision . decision_id , & plan . decision ) ? ;
if let Some ( request ) = & plan . deployment_request {
node . put ( & request . request_id , request ) ? ;
}
if let Some ( request ) = & plan . restart_request {
node . put ( & request . request_id , request ) ? ;
}
if let Some ( alarm ) = & plan . operator_alarm {
node . put ( & alarm . alarm_id , alarm ) ? ;
}
Ok ( ( ) )
} ) ? ;
2026-06-04 21:07:42 +01:00
2026-06-09 15:32:20 +01:00
if let Some ( request ) = & plan . deployment_request {
if options . execute {
2026-06-11 16:41:21 +01:00
let result = run_deployment ( request , & now , options . command_timeout_seconds ) ;
2026-06-15 17:06:32 +01:00
let alarm = if result . state ! = " succeeded " {
Some ( deployment_failure_alarm ( & result , & now ) )
} else {
None
} ;
with_store_node ( options , store_lock , | node | {
node . put ( & result . result_id , & result ) ? ;
if let Some ( alarm ) = & alarm {
node . put ( & alarm . alarm_id , alarm ) ? ;
}
Ok ( ( ) )
} ) ? ;
2026-06-09 15:32:20 +01:00
println! (
" Idunn deployment {} for {}: {} " ,
result . state , result . daemon_id , result . detail
) ;
2026-06-15 17:06:32 +01:00
if let Some ( alarm ) = alarm {
2026-06-09 15:32:20 +01:00
println! (
" Idunn raised operator alarm for {} through {}: {} " ,
alarm . daemon_id , alarm . escalation_target , alarm . reason
) ;
2026-06-11 16:41:21 +01:00
run_operator_alarm_command ( options , & alarm ) ;
2026-06-09 15:32:20 +01:00
}
} else {
println! (
" Idunn requested deployment for {} but did not execute it. Pass --execute to actuate. " ,
request . daemon_id
) ;
}
}
2026-06-04 21:07:42 +01:00
if let Some ( request ) = & plan . restart_request {
if options . execute {
2026-06-11 16:41:21 +01:00
let result = run_restart ( request , & now , options . command_timeout_seconds ) ;
2026-06-15 17:06:32 +01:00
let alarm = if result . state ! = " succeeded " {
Some ( restart_failure_alarm ( & result , & now ) )
} else {
None
} ;
with_store_node ( options , store_lock , | node | {
node . put ( & result . result_id , & result ) ? ;
if let Some ( alarm ) = & alarm {
node . put ( & alarm . alarm_id , alarm ) ? ;
}
Ok ( ( ) )
} ) ? ;
2026-06-04 21:07:42 +01:00
println! (
" Idunn restart {} for {}: {} " ,
result . state , result . daemon_id , result . detail
) ;
2026-06-15 17:06:32 +01:00
if let Some ( alarm ) = alarm {
2026-06-07 02:43:24 +01:00
println! (
" Idunn raised operator alarm for {} through {}: {} " ,
alarm . daemon_id , alarm . escalation_target , alarm . reason
) ;
2026-06-11 16:41:21 +01:00
run_operator_alarm_command ( options , & alarm ) ;
2026-06-07 02:43:24 +01:00
}
2026-06-04 21:07:42 +01:00
} else {
println! (
" Idunn requested restart for {} but did not execute it. Pass --execute to actuate. " ,
request . daemon_id
) ;
}
}
if let Some ( alarm ) = & plan . operator_alarm {
println! (
" Idunn raised operator alarm for {} through {}: {} " ,
alarm . daemon_id , alarm . escalation_target , alarm . reason
) ;
2026-06-07 04:21:41 +01:00
run_operator_alarm_command ( options , alarm ) ;
2026-06-04 21:07:42 +01:00
}
println! (
" Idunn decision for {}: {} ({}) " ,
plan . decision . daemon_id , plan . decision . action , plan . decision . reason
) ;
Ok ( ( ) )
}
2026-06-15 23:12:39 +01:00
fn with_store_node < T , F > (
options : & CommonOptions ,
store_lock : & Arc < Mutex < ( ) > > ,
write : F ,
) -> Result < T >
2026-06-15 17:06:32 +01:00
where
2026-06-15 23:12:39 +01:00
F : FnOnce ( & mut CultMeshNode ) -> Result < T > ,
2026-06-15 17:06:32 +01:00
{
let _store_guard = store_lock
. lock ( )
. map_err ( | _ | anyhow! ( " Idunn store lock is poisoned " ) ) ? ;
let mut node = CultMesh ::create_node (
& options . store_path ,
OdinDocuments ,
CultMeshNodeOptions {
runtime_id : " idunn-daemon " . to_string ( ) ,
pull_on_start : true ,
} ,
) ? ;
write ( & mut node )
}
2026-06-15 23:12:39 +01:00
fn read_fresh_daemon_published_health (
options : & CommonOptions ,
store_lock : & Arc < Mutex < ( ) > > ,
desired : & IdunnDesiredDaemonRecord ,
now : & str ,
) -> Result < Option < IdunnDaemonHealthRecord > > {
with_store_node ( options , store_lock , | node | {
let Some ( health ) = node . get ::< IdunnDaemonHealthRecord > ( & desired . daemon_id ) ? else {
return Ok ( None ) ;
} ;
if is_fresh_daemon_published_health ( & health , desired , now ) {
Ok ( Some ( health ) )
} else {
Ok ( None )
}
} )
}
fn is_fresh_daemon_published_health (
health : & IdunnDaemonHealthRecord ,
desired : & IdunnDesiredDaemonRecord ,
now : & str ,
) -> bool {
if health . daemon_id ! = desired . daemon_id {
return false ;
}
if health . health_contract ! = desired . health_contract {
return false ;
}
if health . publication_source ! = " daemon-published " {
return false ;
}
if health . transport ! = CULTNET_RUDP_PROTOCOL_ID {
return false ;
}
let Some ( now_seconds ) = parse_unix_timestamp ( now ) else {
return false ;
} ;
let Some ( observed_seconds ) = parse_unix_timestamp ( & health . observed_at ) else {
return false ;
} ;
if observed_seconds > now_seconds {
return false ;
}
now_seconds . saturating_sub ( observed_seconds ) < = u64 ::from ( desired . max_silence_seconds )
}
fn parse_unix_timestamp ( value : & str ) -> Option < u64 > {
value . strip_prefix ( " unix: " ) ? . parse ( ) . ok ( )
}
2026-06-15 22:35:00 +01:00
fn publish_runtime_transport_check (
options : & CommonOptions ,
store_lock : & Arc < Mutex < ( ) > > ,
observed_at : & str ,
) -> Result < ( ) > {
let check = runtime_transport_check ( observed_at ) ;
with_store_node ( options , store_lock , | node | {
node . put ( & check . check_id , & check ) ? ;
Ok ( ( ) )
} )
}
fn runtime_transport_check ( observed_at : & str ) -> IdunnRuntimeTransportCheckRecord {
match verify_rudp_loopback ( ) {
Ok ( transfer_id ) = > IdunnRuntimeTransportCheckRecord {
check_id : " idunn-runtime-rudp-loopback " . to_string ( ) ,
runtime_id : " idunn-daemon " . to_string ( ) ,
transport : CULTNET_RUDP_PROTOCOL_ID . to_string ( ) ,
state : " available " . to_string ( ) ,
detail : format ! ( " loopback CultNet RUDP message acknowledged as transfer {transfer_id} " ) ,
observed_at : observed_at . to_string ( ) ,
} ,
Err ( error ) = > IdunnRuntimeTransportCheckRecord {
check_id : " idunn-runtime-rudp-loopback " . to_string ( ) ,
runtime_id : " idunn-daemon " . to_string ( ) ,
transport : CULTNET_RUDP_PROTOCOL_ID . to_string ( ) ,
state : " failed " . to_string ( ) ,
detail : format ! ( " loopback CultNet RUDP check failed: {error} " ) ,
observed_at : observed_at . to_string ( ) ,
} ,
}
}
fn verify_rudp_loopback ( ) -> Result < String > {
let sender = UdpSocket ::bind ( " 127.0.0.1:0 " ) ? ;
let receiver = UdpSocket ::bind ( " 127.0.0.1:0 " ) ? ;
let receiver_addr = receiver . local_addr ( ) ? ;
let options = CultNetRudpOptions {
ack_timeout : Duration ::from_millis ( 100 ) ,
retries : 3 ,
.. CultNetRudpOptions ::default ( )
} ;
let receiver_options = options . clone ( ) ;
let handle = thread ::spawn ( move | | -> Result < ( ) > {
let ( message , _source , _transfer_id ) =
recv_cultnet_message_rudp ( & receiver , & receiver_options ) ? ;
match message {
CultNetMessage ::Hello { runtime_id , .. } if runtime_id = = " idunn-daemon " = > Ok ( ( ) ) ,
other = > Err ( anyhow! ( " unexpected loopback CultNet message: {other:?} " ) ) ,
}
} ) ;
let transfer_id = send_cultnet_message_rudp (
& sender ,
receiver_addr ,
& CultNetMessage ::Hello {
runtime_id : " idunn-daemon " . to_string ( ) ,
runtime_kind : " keepalive " . to_string ( ) ,
agent_id : None ,
role : Some ( " idunn.runtime-transport-self-check " . to_string ( ) ) ,
display_name : Some ( " Idunn RUDP self-check " . to_string ( ) ) ,
supported_document_types : Some ( vec! [ " idunn.runtime_transport_check " . to_string ( ) ] ) ,
supported_mutation_contracts : None ,
supported_message_versions : Some ( vec! [ " cultnet.hello.v0 " . to_string ( ) ] ) ,
supports_schema_catalog : Some ( false ) ,
} ,
& options ,
) ? ;
handle
. join ( )
. map_err ( | _ | anyhow! ( " Idunn RUDP loopback receiver thread panicked " ) ) ? ? ;
Ok ( transfer_id )
}
2026-06-15 22:50:43 +01:00
fn start_rudp_health_ingress (
options : & CommonOptions ,
store_lock : & Arc < Mutex < ( ) > > ,
observed_at : & str ,
) -> Result < ( ) > {
let Some ( bind_address ) = options . rudp_health_bind else {
return Ok ( ( ) ) ;
} ;
let ingress_id = " idunn-rudp-health-ingress " . to_string ( ) ;
let socket = match UdpSocket ::bind ( bind_address ) {
Ok ( socket ) = > socket ,
Err ( error ) = > {
let failed = rudp_health_ingress_record (
& ingress_id ,
bind_address ,
" failed " ,
& format! ( " failed to bind RUDP health ingress: {error} " ) ,
observed_at ,
) ;
with_store_node ( options , store_lock , | node | {
node . put ( & failed . ingress_id , & failed ) ? ;
Ok ( ( ) )
} ) ? ;
return Err ( error . into ( ) ) ;
}
} ;
let local_addr = socket . local_addr ( ) ? ;
socket . set_read_timeout ( Some ( Duration ::from_millis ( 500 ) ) ) ? ;
let active = rudp_health_ingress_record (
& ingress_id ,
local_addr ,
" active " ,
" listening for raw idunn.daemon_health documents over CultNet RUDP " ,
observed_at ,
) ;
with_store_node ( options , store_lock , | node | {
node . put ( & active . ingress_id , & active ) ? ;
Ok ( ( ) )
} ) ? ;
let worker_options = options . clone ( ) ;
let worker_store_lock = Arc ::clone ( store_lock ) ;
thread ::spawn ( move | | {
if let Err ( error ) = run_rudp_health_ingress_loop ( socket , worker_options , worker_store_lock )
{
eprintln! ( " Idunn RUDP health ingress stopped: {error} " ) ;
}
} ) ;
Ok ( ( ) )
}
fn rudp_health_ingress_record (
ingress_id : & str ,
bind_address : SocketAddr ,
state : & str ,
detail : & str ,
observed_at : & str ,
) -> IdunnRudpHealthIngressRecord {
IdunnRudpHealthIngressRecord {
ingress_id : ingress_id . to_string ( ) ,
bind_address : bind_address . to_string ( ) ,
transport : CULTNET_RUDP_PROTOCOL_ID . to_string ( ) ,
accepted_schema : " idunn.daemon_health " . to_string ( ) ,
state : state . to_string ( ) ,
detail : detail . to_string ( ) ,
observed_at : observed_at . to_string ( ) ,
}
}
fn run_rudp_health_ingress_loop (
socket : UdpSocket ,
options : CommonOptions ,
store_lock : Arc < Mutex < ( ) > > ,
) -> Result < ( ) > {
let rudp_options = CultNetRudpOptions {
ack_timeout : Duration ::from_millis ( 100 ) ,
retries : 3 ,
.. CultNetRudpOptions ::default ( )
} ;
loop {
match recv_cultnet_message_rudp ( & socket , & rudp_options ) {
Ok ( ( message , source , transfer_id ) ) = > {
let observed_at = timestamp ( ) ? ;
match health_from_rudp_message ( & message ) {
2026-06-15 23:12:39 +01:00
Ok ( mut health ) = > {
health . publication_source = " daemon-published " . to_string ( ) ;
health . transport = CULTNET_RUDP_PROTOCOL_ID . to_string ( ) ;
2026-06-15 22:50:43 +01:00
with_store_node ( & options , & store_lock , | node | {
node . put ( & health . daemon_id , & health ) ? ;
Ok ( ( ) )
} ) ? ;
println! (
" Idunn accepted RUDP health for {} from {} transfer {}. " ,
health . daemon_id , source , transfer_id
) ;
}
Err ( error ) = > {
let failed = rudp_health_ingress_record (
" idunn-rudp-health-ingress " ,
socket . local_addr ( ) ? ,
" degraded " ,
& format! ( " rejected RUDP health message from {source} : {error} " ) ,
& observed_at ,
) ;
with_store_node ( & options , & store_lock , | node | {
node . put ( & failed . ingress_id , & failed ) ? ;
Ok ( ( ) )
} ) ? ;
}
}
}
Err ( error )
if error . downcast_ref ::< std ::io ::Error > ( ) . is_some_and ( | io | {
matches! (
io . kind ( ) ,
std ::io ::ErrorKind ::WouldBlock | std ::io ::ErrorKind ::TimedOut
)
} ) = > { }
Err ( error ) = > return Err ( error ) ,
}
}
}
fn health_from_rudp_message ( message : & CultNetMessage ) -> Result < IdunnDaemonHealthRecord > {
let CultNetMessage ::DocumentPutRaw { document , .. } = message else {
return Err ( anyhow! ( " expected cultnet.document_put_raw.v0 " ) ) ;
} ;
if document . schema_id ! = " idunn.daemon_health " {
return Err ( anyhow! (
" expected idunn.daemon_health schema, received {} " ,
document . schema_id
) ) ;
}
if document . payload_encoding ! = CultNetRawPayloadEncoding ::Messagepack {
return Err ( anyhow! ( " expected MessagePack raw payload encoding " ) ) ;
}
let health : IdunnDaemonHealthRecord = rmp_serde ::from_slice ( & document . payload ) ? ;
if document . record_key ! = health . daemon_id {
return Err ( anyhow! (
" record key {} does not match health daemon_id {} " ,
document . record_key ,
health . daemon_id
) ) ;
}
Ok ( health )
}
2026-06-15 23:36:52 +01:00
fn publish_surgery_plans (
profile : & str ,
2026-06-15 21:47:53 +01:00
targets : & [ DaemonTarget ] ,
options : & CommonOptions ,
store_lock : & Arc < Mutex < ( ) > > ,
updated_at : & str ,
) -> Result < ( ) > {
2026-06-15 23:36:52 +01:00
let swarm_plan = swarm_surgery_plan ( profile , targets , updated_at ) ;
2026-06-15 21:47:53 +01:00
let plans = daemon_surgery_plans ( targets , updated_at ) ;
with_store_node ( options , store_lock , | node | {
2026-06-15 23:36:52 +01:00
node . put ( & swarm_plan . plan_id , & swarm_plan ) ? ;
2026-06-15 22:04:53 +01:00
for target in targets {
let transport_profile = daemon_transport_profile ( target , updated_at ) ;
let command_boundary = command_boundary ( target , updated_at ) ;
node . put ( & transport_profile . profile_id , & transport_profile ) ? ;
node . put ( & command_boundary . boundary_id , & command_boundary ) ? ;
}
2026-06-15 21:47:53 +01:00
for plan in & plans {
node . put ( & plan . plan_id , plan ) ? ;
}
Ok ( ( ) )
} )
}
2026-06-15 23:36:52 +01:00
fn swarm_surgery_plan (
profile : & str ,
targets : & [ DaemonTarget ] ,
updated_at : & str ,
) -> IdunnSwarmSurgeryPlanRecord {
2026-06-16 01:27:45 +01:00
let has_odin = targets . iter ( ) . any ( | target | target . daemon_id = = " odin " ) ;
let next_target = if has_odin {
" odin "
2026-06-15 23:36:52 +01:00
} else if let Some ( target ) = targets . iter ( ) . find ( | target | target . enabled ) {
target . daemon_id . as_str ( )
} else {
" none "
} ;
IdunnSwarmSurgeryPlanRecord {
plan_id : format ! ( " swarm-surgery:{profile} " ) ,
profile : profile . to_string ( ) ,
status : " active-transport-migration " . to_string ( ) ,
owner : " Idunn swarm supervisor " . to_string ( ) ,
objective :
" Move daemon awareness from compatibility probes to daemon-published typed CultNet/RUDP state. "
. to_string ( ) ,
current_mechanism :
" Idunn publishes per-daemon desired state, surgery plans, transport profiles, command boundaries, runtime RUDP self-checks, and RUDP health ingress; compatibility command probes remain fallback evidence until each daemon publishes its own health. "
. to_string ( ) ,
invariants : vec ! [
" Daemon truth is typed CultCache/CultMesh state carried over cultnet.transport.rudp.v0. " . to_string ( ) ,
" Compatibility commands, HTTP, WebSocket, SSH, and systemd probes are evidence only and must not own daemon health. " . to_string ( ) ,
" Idunn consumes daemon-published RUDP health before compatibility probes and actuates only advertised lifecycle authority. " . to_string ( ) ,
" Each migrated daemon must publish the same health contract that Idunn expects for its target. " . to_string ( ) ,
] ,
phases : vec ! [
" 1. Publish Idunn's own RUDP substrate and ingress state. " . to_string ( ) ,
" 2. Install daemon-published RUDP health in one Rust daemon and prove Idunn consumes it live. " . to_string ( ) ,
" 3. Extend CultLib RUDP publication support across TypeScript, C#, and remaining daemon runtimes. " . to_string ( ) ,
" 4. Promote provider advertisements, command boundaries, and transport profiles to daemon-owned CultNet/RUDP records. " . to_string ( ) ,
" 5. Delete or demote compatibility probes once every target has daemon-owned publication and advertised lifecycle authority. " . to_string ( ) ,
] ,
current_phase :
2026-06-16 01:27:45 +01:00
" Phase 5: move Odin's own provider health from compatibility command evidence to daemon-published RUDP state. "
2026-06-15 23:36:52 +01:00
. to_string ( ) ,
next_target : next_target . to_string ( ) ,
cut_line :
2026-06-16 01:27:45 +01:00
" All Muninn lanes now publish daemon-owned RUDP health. Odin remains the next critical provider-health cut before its local command probe can be demoted. "
2026-06-15 23:36:52 +01:00
. to_string ( ) ,
verification_layer :
" CultMesh keepalive store records plus live Idunn decision cycles, not process exit codes or chat summaries. "
. to_string ( ) ,
updated_at : updated_at . to_string ( ) ,
}
}
2026-06-15 22:04:53 +01:00
fn transport_profile_id ( target : & DaemonTarget ) -> String {
format! ( " transport: {} " , target . daemon_id )
}
fn command_boundary_id ( target : & DaemonTarget ) -> String {
format! ( " command-boundary: {} " , target . daemon_id )
}
fn daemon_transport_profile (
target : & DaemonTarget ,
observed_at : & str ,
) -> IdunnDaemonTransportProfileRecord {
IdunnDaemonTransportProfileRecord {
profile_id : transport_profile_id ( target ) ,
daemon_id : target . daemon_id . clone ( ) ,
target_transport : " cultnet.transport.rudp.v0 " . to_string ( ) ,
current_transport : " compatibility.local-command " . to_string ( ) ,
state : " migration-required " . to_string ( ) ,
health_contract : target . health_contract . id . clone ( ) ,
publication_schema : " idunn.daemon_health.v1 " . to_string ( ) ,
compatibility_mechanism : target
. health_command
. clone ( )
. unwrap_or_else ( | | " none " . to_string ( ) ) ,
cut_line : " Compatibility command probes are evidence only; daemon truth moves to CultNet/RUDP health publication and advertised command boundaries. "
. to_string ( ) ,
observed_at : observed_at . to_string ( ) ,
}
}
fn command_boundary ( target : & DaemonTarget , observed_at : & str ) -> IdunnCommandBoundaryRecord {
let restart_authority = target
. restart_command
. as_ref ( )
. map ( | _ | " idunn.local-command.restart " )
. unwrap_or ( " none " )
. to_string ( ) ;
let deploy_authority = target
. deploy_command
. as_ref ( )
. map ( | _ | " idunn.local-command.deploy " )
. unwrap_or ( " none " )
. to_string ( ) ;
let compatibility_commands = [
target . health_command . as_ref ( ) ,
target . restart_command . as_ref ( ) ,
target . deploy_command . as_ref ( ) ,
]
. into_iter ( )
. flatten ( )
. cloned ( )
. collect ( ) ;
IdunnCommandBoundaryRecord {
boundary_id : command_boundary_id ( target ) ,
daemon_id : target . daemon_id . clone ( ) ,
owner : " idunn.local-command-compatibility " . to_string ( ) ,
restart_authority ,
deploy_authority ,
health_authority : " compatibility.probe-only " . to_string ( ) ,
alarm_authority : " bifrost.operator-notification " . to_string ( ) ,
compatibility_commands ,
forbidden_authority :
" Health commands, HTTP endpoints, WebSocket decks, SSH probes, and systemd status do not own daemon truth. "
. to_string ( ) ,
observed_at : observed_at . to_string ( ) ,
}
}
2026-06-15 21:47:53 +01:00
fn daemon_surgery_plans (
targets : & [ DaemonTarget ] ,
updated_at : & str ,
) -> Vec < IdunnDaemonSurgeryPlanRecord > {
targets
. iter ( )
. map ( | target | daemon_surgery_plan ( target , updated_at ) )
. collect ( )
}
fn daemon_surgery_plan ( target : & DaemonTarget , updated_at : & str ) -> IdunnDaemonSurgeryPlanRecord {
let mut severity = " high " ;
let mut status = " transport-surgery-required " ;
let mut owner = " daemon-owned CultLib update " ;
let mut objective = format! (
" {} publishes daemon-owned health, provider state, command boundary, and transport profile through CultNet/RUDP. " ,
target . name
) ;
let mut current_mechanism = format! (
" Idunn target {} currently uses local compatibility health command {:?} under contract {}. " ,
target . daemon_id , target . health_command , target . health_contract . id
) ;
let mut intended_authority = " Daemon publishes typed CultMesh/CultNet documents over cultnet.transport.rudp.v0; Idunn consumes those records and only actuates advertised lifecycle commands. " . to_string ( ) ;
let mut cut_line = " Cut HTTP, WebSocket, SSH, systemd, and command-exit probes as sources of daemon truth once the daemon's CultLib can publish the RUDP health contract. " . to_string ( ) ;
let mut steps = vec! [
" Update the daemon's runtime CultLib dependency to a build that can speak CultNet over RUDP. " . to_string ( ) ,
" Publish daemon_health, provider_advertisement, command_boundary, and transport_profile typed records over cultnet.transport.rudp.v0. " . to_string ( ) ,
" Teach Odin to accept the daemon's RUDP provider records into the service/catalog surface. " . to_string ( ) ,
" Switch Idunn from the compatibility health command to the daemon-owned RUDP health record. " . to_string ( ) ,
" Delete or demote the old probe to a xenos-boundary compatibility check with no lifecycle authority. " . to_string ( ) ,
] ;
let mut blockers = Vec ::new ( ) ;
match target . daemon_id . as_str ( ) {
" odin " = > {
severity = " critical " ;
owner = " Odin core " ;
current_mechanism =
" Odin still exposes local health and Eve deck compatibility surfaces over HTTP/WebSocket while Idunn verifies liveness through a command probe. "
. to_string ( ) ;
intended_authority =
" Odin owns accepted Verse discovery and provider catalog truth as typed CultMesh/CultNet records over cultnet.transport.rudp.v0. "
. to_string ( ) ;
cut_line =
" HTTP/WebSocket deck and health endpoints become browser/lowering bridges only; they no longer decide daemon health, discovery, or provider truth. "
. to_string ( ) ;
steps [ 2 ] =
" Make Odin ingest RUDP provider advertisements directly into its accepted service/catalog surface. "
. to_string ( ) ;
}
" nightwing-gjallar " = > {
severity = " critical " ;
owner = " Gjallar C# runtime plus Odin provider feed " ;
current_mechanism =
" Gjallar currently proves visible composition through service checks and /var/log/gjallar.status, fed by Odin's WebSocket deck compatibility bridge. "
. to_string ( ) ;
intended_authority =
" Gjallar subscribes to Odin/provider deck state over CultNet/RUDP and publishes framebuffer composition health as typed CultMesh/CultNet state. "
. to_string ( ) ;
cut_line =
" Cut Odin WebSocket deck consumption as the compositor truth path; keep it only as a browser/debug lowering if needed. "
. to_string ( ) ;
blockers
. push ( " Requires native C# CultNet/RUDP subscription path for Gjallar. " . to_string ( ) ) ;
}
" stonks " = > {
owner = " Stonks TypeScript runtime " ;
current_mechanism =
" Stonks has a repaired CultCache startup path, but health and market projection are still checked through local HTTP/WebSocket compatibility surfaces. "
. to_string ( ) ;
blockers . push ( " Requires TypeScript CultLib RUDP publication support. " . to_string ( ) ) ;
}
" mimir-eve-dashboard " = > {
severity = " high " ;
owner = " Mimir dashboard runtime " ;
current_mechanism =
" Mimir dashboard health is currently a local compatibility probe with no Idunn restart authority. "
. to_string ( ) ;
blockers . push (
" No restart/deploy command authority is currently advertised to Idunn. " . to_string ( ) ,
) ;
}
" weksa " = > {
severity = " medium-high " ;
owner = " Weksa provider runtime " ;
current_mechanism =
" Weksa has provider advertisement bones, but Idunn still relies on compatibility process/health probes. "
. to_string ( ) ;
}
" voidbot " = > {
severity = " medium-high " ;
owner = " VoidBot internal provider stack " ;
current_mechanism =
" VoidBot has CultMesh/CultCache swarm state, but Idunn still treats stack health through a compatibility command; Discord remains an external xenos boundary. "
. to_string ( ) ;
intended_authority =
" VoidBot publishes internal swarm, repo-face, and provider health over CultNet/RUDP; Discord delivery remains a boundary adapter, never daemon truth. "
. to_string ( ) ;
}
" starfire-muninn " | " muninn " | " nightwing-muninn " = > {
owner = " Muninn Rust runtime " ;
current_mechanism =
" Muninn has typed telemetry state, but Idunn still validates continuity through local or remote script probes. "
. to_string ( ) ;
intended_authority =
" Muninn publishes telemetry, Quest/Move access, and daemon health over CultNet/RUDP; activation commands remain separate from keepalive. "
. to_string ( ) ;
}
" vili " = > {
owner = " Vili animation runtime " ;
current_mechanism =
" Vili is watched through compatibility health/deck checks instead of daemon-owned CultNet/RUDP animation health. "
. to_string ( ) ;
}
" nightwing-eve-dashboard " | " nightwing-eve-browser-reference " = > {
severity = " medium " ;
owner = " Eve lowering/runtime owner " ;
current_mechanism =
" Nightwing Eve runtime services are watched through local service probes; browser-facing transports are lowering bridges. "
. to_string ( ) ;
intended_authority =
" Eve runtimes subscribe to provider-owned CultMesh/CultNet state; any browser/WebSocket bridge is display-only and does not own daemon health. "
. to_string ( ) ;
cut_line =
" Cut display/runtime liveness from provider truth. Browser-compatible bridges may remain only as lowering targets. "
. to_string ( ) ;
}
" yggdrasil-heimdall " | " yggdrasil-repixelizer " | " yggdrasil-streampixels " = > {
severity = " medium " ;
owner = " Yggdrasil service owner plus gamecult-ops deploy lane " ;
current_mechanism =
" Yggdrasil source apps are currently checked and deployed through SSH/systemd/source-artifact compatibility scripts. "
. to_string ( ) ;
intended_authority =
" Public HTTP remains the product boundary, but deployment freshness, daemon health, and lifecycle commands publish as internal CultNet/RUDP state. "
. to_string ( ) ;
cut_line =
" SSH/systemd probes stop deciding freshness once hosted services publish internal RUDP health and deployment manifests. "
. to_string ( ) ;
}
" idunn-swarm-deployment-coverage " = > {
severity = " medium " ;
status = " catalog-coherence-probe " ;
owner = " Idunn " ;
objective =
" Keep Idunn's deployment target catalog honest while daemon-owned RUDP publication is being installed. "
. to_string ( ) ;
current_mechanism =
" A local coverage command verifies deployment authority categories for the swarm target catalog. "
. to_string ( ) ;
intended_authority =
" Daemon provider advertisements and command_boundary records make deploy/restart ownership inspectable without a local coverage shim. "
. to_string ( ) ;
cut_line =
" Delete this probe after every deployable daemon publishes command_boundary and transport_profile over CultNet/RUDP. "
. to_string ( ) ;
}
_ = > { }
}
IdunnDaemonSurgeryPlanRecord {
plan_id : format ! ( " surgery:{} " , target . daemon_id ) ,
daemon_id : target . daemon_id . clone ( ) ,
severity : severity . to_string ( ) ,
status : status . to_string ( ) ,
owner : owner . to_string ( ) ,
objective ,
current_mechanism ,
intended_authority ,
cut_line ,
steps ,
blockers ,
updated_at : updated_at . to_string ( ) ,
}
}
2026-06-11 16:41:21 +01:00
fn swarm_targets ( options : & SwarmOptions ) -> Result < Vec < DaemonTarget > > {
let repo_root = options . repo_root . display ( ) . to_string ( ) ;
let script = | name : & str | format! ( r "{}\scripts\{}" , repo_root , name ) ;
match options . profile . as_str ( ) {
" starfire-local " = > Ok ( vec! [
DaemonTarget {
daemon_id : " odin " . to_string ( ) ,
verse_id : " starfire.local " . to_string ( ) ,
name : " Odin all-seer " . to_string ( ) ,
2026-06-15 21:09:37 +01:00
health_contract : health_contract ( " odin.cultnet-rudp-provider-health " , " failed " ) ,
2026-06-11 16:41:21 +01:00
health_command : Some ( script ( " health-odin.cmd " ) ) ,
deploy_command : None ,
restart_command : Some ( script ( " restart-odin.cmd " ) ) ,
enabled : true ,
interval_seconds : 30 ,
} ,
DaemonTarget {
daemon_id : " mimir-eve-dashboard " . to_string ( ) ,
verse_id : " starfire.local " . to_string ( ) ,
name : " Mimir Eve dashboard " . to_string ( ) ,
2026-06-15 21:09:37 +01:00
health_contract : health_contract ( " mimir.cultnet-rudp-provider-health " , " failed " ) ,
2026-06-11 16:41:21 +01:00
health_command : Some ( script ( " health-mimir-eve-dashboard.cmd " ) ) ,
deploy_command : None ,
restart_command : None ,
enabled : true ,
interval_seconds : 30 ,
} ,
DaemonTarget {
daemon_id : " stonks " . to_string ( ) ,
verse_id : " starfire.local " . to_string ( ) ,
name : " Stonks market pulse " . to_string ( ) ,
2026-06-15 21:09:37 +01:00
health_contract : health_contract ( " stonks.cultnet-rudp-market-health " , " failed " ) ,
2026-06-11 16:41:21 +01:00
health_command : Some ( script ( " health-stonks.cmd " ) ) ,
deploy_command : None ,
restart_command : Some ( script ( " restart-stonks.cmd " ) ) ,
enabled : true ,
interval_seconds : 30 ,
} ,
DaemonTarget {
daemon_id : " voidbot " . to_string ( ) ,
verse_id : " starfire.local " . to_string ( ) ,
name : " VoidBot local stack " . to_string ( ) ,
2026-06-15 21:09:37 +01:00
health_contract : health_contract ( " voidbot.cultnet-rudp-stack-health " , " failed " ) ,
2026-06-11 16:41:21 +01:00
health_command : Some ( script ( " health-voidbot.cmd " ) ) ,
deploy_command : None ,
restart_command : Some ( script ( " restart-voidbot.cmd " ) ) ,
enabled : true ,
interval_seconds : 300 ,
} ,
2026-06-11 20:27:58 +01:00
DaemonTarget {
daemon_id : " weksa " . to_string ( ) ,
verse_id : " starfire.local " . to_string ( ) ,
name : " Weksa intent and utterance lowering service " . to_string ( ) ,
2026-06-15 21:09:37 +01:00
health_contract : health_contract ( " weksa.cultnet-rudp-provider-health " , " failed " ) ,
2026-06-11 20:27:58 +01:00
health_command : Some ( script ( " health-weksa.cmd " ) ) ,
deploy_command : None ,
restart_command : Some ( script ( " restart-weksa.cmd " ) ) ,
enabled : true ,
interval_seconds : 60 ,
} ,
2026-06-12 16:09:03 +01:00
DaemonTarget {
daemon_id : " starfire-muninn " . to_string ( ) ,
verse_id : " starfire.local " . to_string ( ) ,
name : " Starfire Muninn telemetry and Quest access daemon " . to_string ( ) ,
2026-06-15 20:50:57 +01:00
health_contract : health_contract (
2026-06-15 21:09:37 +01:00
" muninn.cultnet-rudp-local-telemetry-and-quest-access " ,
2026-06-15 20:50:57 +01:00
" failed " ,
) ,
2026-06-12 16:09:03 +01:00
health_command : Some ( script ( " health-starfire-muninn.cmd " ) ) ,
deploy_command : None ,
restart_command : Some ( script ( " restart-starfire-muninn.cmd " ) ) ,
enabled : true ,
interval_seconds : 30 ,
} ,
2026-06-11 16:41:21 +01:00
DaemonTarget {
daemon_id : " muninn " . to_string ( ) ,
verse_id : " raven.local " . to_string ( ) ,
name : " Muninn telemetry Verse assembler " . to_string ( ) ,
2026-06-15 21:47:53 +01:00
health_contract : health_contract (
" muninn.cultnet-rudp-remote-telemetry-health " ,
" failed " ,
) ,
2026-06-11 16:41:21 +01:00
health_command : Some ( script ( " health-muninn.cmd " ) ) ,
deploy_command : None ,
restart_command : Some ( script ( " restart-muninn.cmd " ) ) ,
enabled : true ,
interval_seconds : 30 ,
} ,
2026-06-12 12:56:40 +01:00
DaemonTarget {
daemon_id : " vili " . to_string ( ) ,
verse_id : " raven.local " . to_string ( ) ,
name : " Vili Persona animation daemon " . to_string ( ) ,
2026-06-15 21:09:37 +01:00
health_contract : health_contract ( " vili.cultnet-rudp-animation-health " , " failed " ) ,
2026-06-12 12:56:40 +01:00
health_command : Some ( script ( " health-vili.cmd " ) ) ,
deploy_command : None ,
restart_command : Some ( script ( " restart-vili.cmd " ) ) ,
enabled : true ,
interval_seconds : 60 ,
} ,
2026-06-11 16:41:21 +01:00
DaemonTarget {
daemon_id : " idunn-swarm-deployment-coverage " . to_string ( ) ,
verse_id : " starfire.local " . to_string ( ) ,
name : " Idunn swarm deployment coverage " . to_string ( ) ,
2026-06-15 20:50:57 +01:00
health_contract : health_contract ( " idunn.deployment-catalog-coherence " , " degraded " ) ,
2026-06-11 16:41:21 +01:00
health_command : Some ( script ( " health-idunn-swarm-deployment-coverage.cmd " ) ) ,
deploy_command : None ,
restart_command : None ,
enabled : true ,
interval_seconds : 30 ,
} ,
DaemonTarget {
daemon_id : " yggdrasil-heimdall " . to_string ( ) ,
verse_id : " yggdrasil.local " . to_string ( ) ,
name : " Yggdrasil Heimdall " . to_string ( ) ,
2026-06-15 20:50:57 +01:00
health_contract : health_contract (
" yggdrasil.source-deployment-freshness " ,
" stale-deployment " ,
) ,
2026-06-11 16:41:21 +01:00
health_command : Some ( script ( " health-yggdrasil-heimdall.cmd " ) ) ,
deploy_command : Some ( script ( " deploy-yggdrasil-heimdall.cmd " ) ) ,
restart_command : None ,
enabled : true ,
interval_seconds : 300 ,
} ,
DaemonTarget {
daemon_id : " yggdrasil-repixelizer " . to_string ( ) ,
verse_id : " yggdrasil.local " . to_string ( ) ,
name : " Yggdrasil Repixelizer " . to_string ( ) ,
2026-06-15 20:50:57 +01:00
health_contract : health_contract (
" yggdrasil.source-deployment-freshness " ,
" stale-deployment " ,
) ,
2026-06-11 16:41:21 +01:00
health_command : Some ( script ( " health-yggdrasil-repixelizer.cmd " ) ) ,
deploy_command : Some ( script ( " deploy-yggdrasil-repixelizer.cmd " ) ) ,
restart_command : None ,
enabled : true ,
interval_seconds : 300 ,
} ,
DaemonTarget {
daemon_id : " yggdrasil-streampixels " . to_string ( ) ,
verse_id : " yggdrasil.local " . to_string ( ) ,
name : " Yggdrasil StreamPixels " . to_string ( ) ,
2026-06-15 20:50:57 +01:00
health_contract : health_contract (
" yggdrasil.source-deployment-freshness " ,
" stale-deployment " ,
) ,
2026-06-11 16:41:21 +01:00
health_command : Some ( script ( " health-yggdrasil-streampixels.cmd " ) ) ,
deploy_command : Some ( script ( " deploy-yggdrasil-streampixels.cmd " ) ) ,
restart_command : None ,
enabled : true ,
interval_seconds : 300 ,
} ,
DaemonTarget {
daemon_id : " nightwing-gjallar " . to_string ( ) ,
verse_id : " nightwing.local " . to_string ( ) ,
name : " Nightwing Gjallar framebuffer compositor " . to_string ( ) ,
2026-06-15 20:50:57 +01:00
health_contract : health_contract (
2026-06-15 21:09:37 +01:00
" gjallar.cultnet-rudp-framebuffer-composition-health " ,
2026-06-15 20:50:57 +01:00
" dependency-unavailable " ,
) ,
2026-06-11 16:41:21 +01:00
health_command : Some ( script ( " health-nightwing-gjallar.cmd " ) ) ,
deploy_command : Some ( script ( " deploy-nightwing-gjallar.cmd " ) ) ,
restart_command : Some ( script ( " restart-nightwing-gjallar.cmd " ) ) ,
enabled : true ,
interval_seconds : 30 ,
} ,
DaemonTarget {
daemon_id : " nightwing-muninn " . to_string ( ) ,
verse_id : " nightwing.local " . to_string ( ) ,
name : " Nightwing Muninn telemetry and Move HID daemon " . to_string ( ) ,
2026-06-15 21:09:37 +01:00
health_contract : health_contract (
" muninn.cultnet-rudp-remote-telemetry-and-move-hid " ,
" failed " ,
) ,
2026-06-11 16:41:21 +01:00
health_command : Some ( script ( " health-nightwing-muninn.cmd " ) ) ,
deploy_command : None ,
restart_command : Some ( script ( " restart-nightwing-muninn.cmd " ) ) ,
enabled : true ,
interval_seconds : 30 ,
} ,
DaemonTarget {
daemon_id : " nightwing-eve-dashboard " . to_string ( ) ,
verse_id : " nightwing.local " . to_string ( ) ,
name : " Nightwing Eve dashboard broker " . to_string ( ) ,
2026-06-15 21:47:53 +01:00
health_contract : health_contract (
" nightwing.cultnet-rudp-eve-dashboard-health " ,
" failed " ,
) ,
2026-06-11 16:41:21 +01:00
health_command : Some ( script ( " health-nightwing-eve-dashboard.cmd " ) ) ,
deploy_command : None ,
restart_command : Some ( script ( " restart-nightwing-eve-dashboard.cmd " ) ) ,
enabled : true ,
interval_seconds : 30 ,
} ,
DaemonTarget {
daemon_id : " nightwing-eve-browser-reference " . to_string ( ) ,
verse_id : " nightwing.local " . to_string ( ) ,
name : " Nightwing Eve browser reference " . to_string ( ) ,
2026-06-15 21:47:53 +01:00
health_contract : health_contract (
" nightwing.cultnet-rudp-browser-reference-health " ,
" failed " ,
) ,
2026-06-11 16:41:21 +01:00
health_command : Some ( script ( " health-nightwing-eve-browser-reference.cmd " ) ) ,
deploy_command : None ,
restart_command : Some ( script ( " restart-nightwing-eve-browser-reference.cmd " ) ) ,
enabled : true ,
interval_seconds : 30 ,
} ,
] ) ,
other = > Err ( anyhow! ( " unknown Idunn swarm profile: {other} " ) ) ,
}
}
2026-06-07 02:43:24 +01:00
fn restart_failure_alarm (
result : & IdunnRestartResultRecord ,
raised_at : & str ,
) -> IdunnOperatorAlarmRecord {
IdunnOperatorAlarmRecord {
alarm_id : format ! ( " alarm:restart-failed:{}:{} " , result . daemon_id , raised_at ) ,
daemon_id : result . daemon_id . clone ( ) ,
severity : " operator-action-required " . to_string ( ) ,
reason : format ! (
" restart command failed for request {}: {} " ,
result . request_id , result . detail
) ,
escalation_target : " bifrost.operator-notification " . to_string ( ) ,
raised_at : raised_at . to_string ( ) ,
}
}
2026-06-09 15:32:20 +01:00
fn deployment_failure_alarm (
result : & IdunnDeploymentResultRecord ,
raised_at : & str ,
) -> IdunnOperatorAlarmRecord {
IdunnOperatorAlarmRecord {
alarm_id : format ! ( " alarm:deployment-failed:{}:{} " , result . daemon_id , raised_at ) ,
daemon_id : result . daemon_id . clone ( ) ,
severity : " operator-action-required " . to_string ( ) ,
reason : format ! (
" deployment command failed for request {}: {} " ,
result . request_id , result . detail
) ,
escalation_target : " bifrost.operator-notification " . to_string ( ) ,
raised_at : raised_at . to_string ( ) ,
}
}
2026-06-04 21:07:42 +01:00
impl Options {
fn parse ( args : impl Iterator < Item = String > ) -> Result < Self > {
2026-06-11 16:41:21 +01:00
let mut store_path = PathBuf ::from ( " scratch/idunn/idunn.keepalive.cc " ) ;
let mut operator_alarm_command = None ;
2026-06-15 22:50:43 +01:00
let mut rudp_health_bind = Some ( " 127.0.0.1:17870 " . parse ::< SocketAddr > ( ) ? ) ;
2026-06-11 16:41:21 +01:00
let mut execute = false ;
let mut command_timeout_seconds = 30 ;
let mut daemon_id = None ;
let mut verse_id = " local " . to_string ( ) ;
let mut name = None ;
let mut health_command = None ;
let mut deploy_command = None ;
let mut restart_command = None ;
let mut enabled = true ;
let mut interval_seconds = None ;
let mut swarm_profile = None ;
let mut repo_root = env ::current_dir ( ) . context ( " determining current directory " ) ? ;
2026-06-04 21:07:42 +01:00
let mut args = args . peekable ( ) ;
while let Some ( arg ) = args . next ( ) {
match arg . as_str ( ) {
2026-06-11 16:41:21 +01:00
" --store " = > store_path = PathBuf ::from ( take_value ( & mut args , " --store " ) ? ) ,
" --daemon " = > daemon_id = Some ( take_value ( & mut args , " --daemon " ) ? ) ,
" --verse " = > verse_id = take_value ( & mut args , " --verse " ) ? ,
" --name " = > name = Some ( take_value ( & mut args , " --name " ) ? ) ,
2026-06-04 21:07:42 +01:00
" --health-command " = > {
2026-06-11 16:41:21 +01:00
health_command = Some ( take_value ( & mut args , " --health-command " ) ? )
2026-06-04 21:07:42 +01:00
}
2026-06-09 15:32:20 +01:00
" --deploy-command " = > {
2026-06-11 16:41:21 +01:00
deploy_command = Some ( take_value ( & mut args , " --deploy-command " ) ? )
2026-06-09 15:32:20 +01:00
}
2026-06-04 21:07:42 +01:00
" --restart-command " = > {
2026-06-11 16:41:21 +01:00
restart_command = Some ( take_value ( & mut args , " --restart-command " ) ? )
2026-06-04 21:07:42 +01:00
}
2026-06-07 04:21:41 +01:00
" --operator-alarm-command " = > {
2026-06-11 16:41:21 +01:00
operator_alarm_command =
2026-06-07 04:21:41 +01:00
Some ( take_value ( & mut args , " --operator-alarm-command " ) ? )
}
2026-06-15 22:50:43 +01:00
" --rudp-health-bind " = > {
let value = take_value ( & mut args , " --rudp-health-bind " ) ? ;
rudp_health_bind = if value . eq_ignore_ascii_case ( " none " ) {
None
} else {
Some (
value
. parse ( )
. with_context ( | | " --rudp-health-bind must be a socket address " ) ? ,
)
} ;
}
2026-06-11 16:41:21 +01:00
" --disabled " = > enabled = false ,
" --execute " = > execute = true ,
2026-06-04 21:07:42 +01:00
" --interval-seconds " = > {
2026-06-11 16:41:21 +01:00
interval_seconds = Some (
2026-06-04 21:07:42 +01:00
take_value ( & mut args , " --interval-seconds " ) ?
. parse ( )
. context ( " --interval-seconds must be a positive integer " ) ? ,
)
}
2026-06-11 16:41:21 +01:00
" --command-timeout-seconds " = > {
command_timeout_seconds = take_value ( & mut args , " --command-timeout-seconds " ) ?
. parse ( )
. context ( " --command-timeout-seconds must be a positive integer " ) ?
}
" --swarm-profile " = > {
swarm_profile = Some ( take_value ( & mut args , " --swarm-profile " ) ? )
}
" --repo-root " = > repo_root = PathBuf ::from ( take_value ( & mut args , " --repo-root " ) ? ) ,
2026-06-04 21:07:42 +01:00
" --help " | " -h " = > return Err ( anyhow! ( help_text ( ) ) ) ,
other = > {
return Err ( anyhow! (
" unknown Idunn argument: {other} \n \n {} " ,
help_text ( )
) ) ;
}
}
}
2026-06-11 16:41:21 +01:00
if command_timeout_seconds = = 0 {
return Err ( anyhow! (
" --command-timeout-seconds must be greater than zero "
) ) ;
2026-06-04 21:07:42 +01:00
}
2026-06-11 16:41:21 +01:00
let common = CommonOptions {
store_path ,
operator_alarm_command ,
2026-06-15 22:50:43 +01:00
rudp_health_bind ,
2026-06-11 16:41:21 +01:00
execute ,
command_timeout_seconds ,
} ;
let mode = match ( swarm_profile , daemon_id ) {
( Some ( profile ) , None ) = > Mode ::Swarm ( SwarmOptions { profile , repo_root } ) ,
( Some ( _ ) , Some ( _ ) ) = > {
return Err ( anyhow! (
" use either --swarm-profile or --daemon, not both \n \n {} " ,
help_text ( )
) ) ;
}
( None , Some ( daemon_id ) ) = > {
let interval_seconds = interval_seconds . unwrap_or ( 30 ) ;
if interval_seconds = = 0 {
return Err ( anyhow! ( " --interval-seconds must be greater than zero " ) ) ;
}
Mode ::Single ( DaemonTarget {
daemon_id : daemon_id . clone ( ) ,
verse_id ,
name : name . unwrap_or ( daemon_id ) ,
2026-06-15 20:50:57 +01:00
health_contract : health_contract ( " manual.command-health " , " failed " ) ,
2026-06-11 16:41:21 +01:00
health_command ,
deploy_command ,
restart_command ,
enabled ,
interval_seconds ,
} )
}
( None , None ) = > {
return Err ( anyhow! (
" either --daemon or --swarm-profile is required \n \n {} " ,
help_text ( )
) ) ;
}
} ;
Ok ( Self { common , mode } )
2026-06-04 21:07:42 +01:00
}
}
fn take_value ( args : & mut impl Iterator < Item = String > , name : & str ) -> Result < String > {
args . next ( )
. ok_or_else ( | | anyhow! ( " {name} requires a value " ) )
}
2026-06-11 16:41:21 +01:00
fn probe_health (
target : & DaemonTarget ,
command_timeout_seconds : u64 ,
observed_at : & str ,
) -> IdunnDaemonHealthRecord {
match & target . health_command {
Some ( command ) = > match run_shell ( command , command_timeout_seconds ) {
2026-06-04 21:07:42 +01:00
Ok ( output ) if output . status . success ( ) = > IdunnDaemonHealthRecord {
2026-06-11 16:41:21 +01:00
daemon_id : target . daemon_id . clone ( ) ,
2026-06-04 21:07:42 +01:00
state : " active " . to_string ( ) ,
2026-06-15 17:06:32 +01:00
detail : command_output_detail ( " health command exited successfully " , & output ) ,
2026-06-15 20:50:57 +01:00
health_contract : target . health_contract . id . clone ( ) ,
2026-06-15 23:12:39 +01:00
publication_source : " compatibility-command " . to_string ( ) ,
transport : " compatibility.local-command " . to_string ( ) ,
2026-06-04 21:07:42 +01:00
observed_at : observed_at . to_string ( ) ,
} ,
2026-06-15 19:44:48 +01:00
Ok ( output ) = > {
let detail = command_output_detail (
2026-06-15 17:06:32 +01:00
& format! ( " health command exited with {} " , output . status ) ,
& output ,
2026-06-15 19:44:48 +01:00
) ;
IdunnDaemonHealthRecord {
daemon_id : target . daemon_id . clone ( ) ,
2026-06-15 20:50:57 +01:00
state : health_state_from_detail ( & detail , target ) . to_string ( ) ,
2026-06-15 19:44:48 +01:00
detail ,
2026-06-15 20:50:57 +01:00
health_contract : target . health_contract . id . clone ( ) ,
2026-06-15 23:12:39 +01:00
publication_source : " compatibility-command " . to_string ( ) ,
transport : " compatibility.local-command " . to_string ( ) ,
2026-06-15 19:44:48 +01:00
observed_at : observed_at . to_string ( ) ,
}
}
2026-06-04 21:07:42 +01:00
Err ( error ) = > IdunnDaemonHealthRecord {
2026-06-11 16:41:21 +01:00
daemon_id : target . daemon_id . clone ( ) ,
2026-06-04 21:07:42 +01:00
state : " failed " . to_string ( ) ,
detail : format ! ( " health command could not run: {error} " ) ,
2026-06-15 20:50:57 +01:00
health_contract : target . health_contract . id . clone ( ) ,
2026-06-15 23:12:39 +01:00
publication_source : " compatibility-command " . to_string ( ) ,
transport : " compatibility.local-command " . to_string ( ) ,
2026-06-04 21:07:42 +01:00
observed_at : observed_at . to_string ( ) ,
} ,
} ,
None = > IdunnDaemonHealthRecord {
2026-06-11 16:41:21 +01:00
daemon_id : target . daemon_id . clone ( ) ,
2026-06-04 21:07:42 +01:00
state : " unknown " . to_string ( ) ,
detail : " no health command was provided " . to_string ( ) ,
2026-06-15 20:50:57 +01:00
health_contract : target . health_contract . id . clone ( ) ,
2026-06-15 23:12:39 +01:00
publication_source : " compatibility-command " . to_string ( ) ,
transport : " compatibility.local-command " . to_string ( ) ,
2026-06-04 21:07:42 +01:00
observed_at : observed_at . to_string ( ) ,
} ,
}
}
2026-06-15 20:50:57 +01:00
fn health_state_from_detail < ' a > ( detail : & ' a str , target : & ' a DaemonTarget ) -> & ' a str {
2026-06-15 19:44:48 +01:00
for state in [
" stale-deployment " ,
" dependency-unavailable " ,
" degraded " ,
" failed " ,
] {
let marker = format! ( " idunn.health.state= {state} " ) ;
if detail . contains ( & marker ) {
return state ;
}
}
2026-06-15 20:50:57 +01:00
& target . health_contract . default_failure_state
2026-06-15 19:44:48 +01:00
}
2026-06-04 21:07:42 +01:00
fn run_restart (
request : & odin_core ::IdunnRestartRequestRecord ,
requested_at : & str ,
2026-06-11 16:41:21 +01:00
command_timeout_seconds : u64 ,
2026-06-04 21:07:42 +01:00
) -> IdunnRestartResultRecord {
let result_id = format! ( " result: {} " , request . request_id ) ;
2026-06-11 16:41:21 +01:00
match run_shell ( & request . command , command_timeout_seconds ) {
2026-06-04 21:07:42 +01:00
Ok ( output ) if output . status . success ( ) = > IdunnRestartResultRecord {
result_id ,
request_id : request . request_id . clone ( ) ,
daemon_id : request . daemon_id . clone ( ) ,
state : " succeeded " . to_string ( ) ,
2026-06-15 17:06:32 +01:00
detail : command_output_detail ( " restart command exited successfully " , & output ) ,
2026-06-04 21:07:42 +01:00
completed_at : requested_at . to_string ( ) ,
} ,
Ok ( output ) = > IdunnRestartResultRecord {
result_id ,
request_id : request . request_id . clone ( ) ,
daemon_id : request . daemon_id . clone ( ) ,
state : " failed " . to_string ( ) ,
2026-06-15 17:06:32 +01:00
detail : command_output_detail (
& format! ( " restart command exited with {} " , output . status ) ,
& output ,
) ,
2026-06-04 21:07:42 +01:00
completed_at : requested_at . to_string ( ) ,
} ,
Err ( error ) = > IdunnRestartResultRecord {
result_id ,
request_id : request . request_id . clone ( ) ,
daemon_id : request . daemon_id . clone ( ) ,
state : " failed " . to_string ( ) ,
detail : format ! ( " restart command could not run: {error} " ) ,
completed_at : requested_at . to_string ( ) ,
} ,
}
}
2026-06-09 15:32:20 +01:00
fn run_deployment (
request : & odin_core ::IdunnDeploymentRequestRecord ,
requested_at : & str ,
2026-06-11 16:41:21 +01:00
command_timeout_seconds : u64 ,
2026-06-09 15:32:20 +01:00
) -> IdunnDeploymentResultRecord {
let result_id = format! ( " result: {} " , request . request_id ) ;
2026-06-11 16:41:21 +01:00
match run_shell ( & request . command , command_timeout_seconds ) {
2026-06-09 15:32:20 +01:00
Ok ( output ) if output . status . success ( ) = > IdunnDeploymentResultRecord {
result_id ,
request_id : request . request_id . clone ( ) ,
daemon_id : request . daemon_id . clone ( ) ,
state : " succeeded " . to_string ( ) ,
2026-06-15 17:06:32 +01:00
detail : command_output_detail ( " deployment command exited successfully " , & output ) ,
2026-06-09 15:32:20 +01:00
completed_at : requested_at . to_string ( ) ,
} ,
Ok ( output ) = > IdunnDeploymentResultRecord {
result_id ,
request_id : request . request_id . clone ( ) ,
daemon_id : request . daemon_id . clone ( ) ,
state : " failed " . to_string ( ) ,
2026-06-15 17:06:32 +01:00
detail : command_output_detail (
& format! ( " deployment command exited with {} " , output . status ) ,
& output ,
) ,
2026-06-09 15:32:20 +01:00
completed_at : requested_at . to_string ( ) ,
} ,
Err ( error ) = > IdunnDeploymentResultRecord {
result_id ,
request_id : request . request_id . clone ( ) ,
daemon_id : request . daemon_id . clone ( ) ,
state : " failed " . to_string ( ) ,
detail : format ! ( " deployment command could not run: {error} " ) ,
completed_at : requested_at . to_string ( ) ,
} ,
}
}
2026-06-15 17:06:32 +01:00
fn command_output_detail ( prefix : & str , output : & std ::process ::Output ) -> String {
let mut detail = prefix . to_string ( ) ;
let stdout = String ::from_utf8_lossy ( & output . stdout ) ;
let stderr = String ::from_utf8_lossy ( & output . stderr ) ;
let stdout = stdout . trim ( ) ;
let stderr = stderr . trim ( ) ;
if ! stdout . is_empty ( ) {
detail . push_str ( " ; stdout: " ) ;
detail . push_str ( & truncate_detail ( stdout , 600 ) ) ;
}
if ! stderr . is_empty ( ) {
detail . push_str ( " ; stderr: " ) ;
detail . push_str ( & truncate_detail ( stderr , 600 ) ) ;
}
detail
}
fn truncate_detail ( value : & str , max_chars : usize ) -> String {
let mut chars = value . chars ( ) ;
let truncated : String = chars . by_ref ( ) . take ( max_chars ) . collect ( ) ;
if chars . next ( ) . is_some ( ) {
format! ( " {truncated} ... " )
} else {
truncated
}
}
2026-06-11 16:41:21 +01:00
fn run_shell ( command : & str , timeout_seconds : u64 ) -> Result < std ::process ::Output > {
let mut process = if cfg! ( windows ) {
2026-06-15 17:06:32 +01:00
let path = windows_command_path ( ) ;
let command = format! ( r # "set "PATH={path}" && set "Path={path}" && {command}"# ) ;
2026-06-11 16:41:21 +01:00
let mut process = Command ::new ( " cmd " ) ;
2026-06-15 17:06:32 +01:00
process . arg ( " /D " ) . arg ( " /S " ) . arg ( " /C " ) . arg ( command ) ;
apply_windows_command_environment ( & mut process , & path ) ;
2026-06-11 16:41:21 +01:00
process
2026-06-04 21:07:42 +01:00
} else {
2026-06-11 16:41:21 +01:00
let mut process = Command ::new ( " sh " ) ;
process . arg ( " -c " ) . arg ( command ) ;
process
} ;
let child = process
. stdout ( Stdio ::piped ( ) )
. stderr ( Stdio ::piped ( ) )
. spawn ( )
. with_context ( | | format! ( " running command {command:?} " ) ) ? ;
wait_for_child_with_timeout ( child , Duration ::from_secs ( timeout_seconds ) , command )
2026-06-04 21:07:42 +01:00
}
2026-06-11 16:41:21 +01:00
fn run_operator_alarm_command ( options : & CommonOptions , alarm : & IdunnOperatorAlarmRecord ) {
2026-06-07 04:21:41 +01:00
let Some ( command ) = options . operator_alarm_command . as_deref ( ) else {
return ;
} ;
if command . trim ( ) . is_empty ( ) {
return ;
}
let mut process = if cfg! ( windows ) {
2026-06-15 17:06:32 +01:00
let path = windows_command_path ( ) ;
let command = format! ( r # "set "PATH={path}" && set "Path={path}" && {command}"# ) ;
2026-06-07 04:21:41 +01:00
let mut process = Command ::new ( " cmd " ) ;
2026-06-15 17:06:32 +01:00
process . arg ( " /D " ) . arg ( " /S " ) . arg ( " /C " ) . arg ( command ) ;
apply_windows_command_environment ( & mut process , & path ) ;
2026-06-07 04:21:41 +01:00
process
} else {
let mut process = Command ::new ( " sh " ) ;
process . arg ( " -c " ) . arg ( command ) ;
process
} ;
let output = process
. env ( " IDUNN_ALARM_ID " , & alarm . alarm_id )
. env ( " IDUNN_ALARM_DAEMON_ID " , & alarm . daemon_id )
. env ( " IDUNN_ALARM_SEVERITY " , & alarm . severity )
. env ( " IDUNN_ALARM_REASON " , & alarm . reason )
. env ( " IDUNN_ALARM_ESCALATION_TARGET " , & alarm . escalation_target )
. env ( " IDUNN_ALARM_RAISED_AT " , & alarm . raised_at )
2026-06-11 16:41:21 +01:00
. stdout ( Stdio ::piped ( ) )
. stderr ( Stdio ::piped ( ) )
. spawn ( ) ;
2026-06-07 04:21:41 +01:00
match output {
2026-06-11 16:41:21 +01:00
Ok ( child ) = > match wait_for_child_with_timeout (
child ,
Duration ::from_secs ( options . command_timeout_seconds ) ,
command ,
) {
Ok ( output ) if output . status . success ( ) = > {
println! (
" Idunn operator alarm command completed for {}. " ,
alarm . daemon_id
) ;
}
Ok ( output ) = > {
eprintln! (
" Idunn operator alarm command failed for {} with {}. " ,
alarm . daemon_id , output . status
) ;
}
Err ( error ) = > {
eprintln! (
" Idunn operator alarm command could not run for {}: {} " ,
alarm . daemon_id , error
) ;
}
} ,
2026-06-07 04:21:41 +01:00
Err ( error ) = > {
eprintln! (
" Idunn operator alarm command could not run for {}: {} " ,
alarm . daemon_id , error
) ;
}
}
}
2026-06-15 17:06:32 +01:00
fn windows_command_path ( ) -> String {
[
r "C:\WINDOWS\system32" ,
r "C:\WINDOWS" ,
r "C:\WINDOWS\System32\Wbem" ,
r "C:\WINDOWS\System32\WindowsPowerShell\v1.0" ,
r "C:\WINDOWS\System32\OpenSSH" ,
r "C:\Program Files\Git\cmd" ,
r "C:\Program Files\nodejs" ,
r "C:\Program Files\Docker\Docker\resources\bin" ,
r "C:\Users\Meta\AppData\Local\Programs\Ollama" ,
r "C:\Program Files\dotnet" ,
r "C:\Users\Meta\.cargo\bin" ,
]
. join ( " ; " )
}
fn apply_windows_command_environment ( process : & mut Command , path : & str ) {
if ! cfg! ( windows ) {
return ;
}
process . env ( " PATH " , path ) ;
process . env ( " Path " , path ) ;
}
2026-06-11 16:41:21 +01:00
fn wait_for_child_with_timeout (
mut child : Child ,
timeout : Duration ,
command : & str ,
) -> Result < std ::process ::Output > {
let started_at = Instant ::now ( ) ;
loop {
if child
. try_wait ( )
. with_context ( | | format! ( " waiting on command {command:?} " ) ) ?
. is_some ( )
{
return child
. wait_with_output ( )
. with_context ( | | format! ( " collecting output for command {command:?} " ) ) ;
}
if started_at . elapsed ( ) > = timeout {
let _ = child . kill ( ) ;
let _ = child . wait_with_output ( ) ;
return Err ( anyhow! (
" command timed out after {} seconds: {command:?} " ,
timeout . as_secs ( )
) ) ;
}
thread ::sleep ( Duration ::from_millis ( 100 ) ) ;
}
}
2026-06-04 21:07:42 +01:00
fn timestamp ( ) -> Result < String > {
let seconds = SystemTime ::now ( )
. duration_since ( UNIX_EPOCH )
. context ( " system clock is before Unix epoch " ) ?
. as_secs ( ) ;
Ok ( format! ( " unix: {seconds} " ) )
}
fn help_text ( ) -> & 'static str {
2026-06-15 22:50:43 +01:00
" Usage: idunn --daemon <id> [--name <name>] [--verse <verse>] [--store <path>] [--health-command <command>] [--deploy-command <command>] [--restart-command <command>] [--operator-alarm-command <command>] [--rudp-health-bind <addr|none>] [--execute] [--interval-seconds <seconds>] [--command-timeout-seconds <seconds>] [--repo-root <path>] [--swarm-profile <profile>] \n \n Idunn can run one manual daemon probe lane with --daemon, or one built-in swarm supervisor with --swarm-profile starfire-local. "
2026-06-04 21:07:42 +01:00
}
2026-06-15 20:50:57 +01:00
#[ cfg(test) ]
mod tests {
use super ::* ;
2026-06-15 22:50:43 +01:00
use cultnet_rs ::{ CultNetRawDocumentRecord , CultNetRawPayloadEncoding } ;
2026-06-15 20:50:57 +01:00
fn target ( default_failure_state : & str , deploy_command : Option < & str > ) -> DaemonTarget {
DaemonTarget {
daemon_id : " test-daemon " . to_string ( ) ,
verse_id : " test.local " . to_string ( ) ,
name : " Test daemon " . to_string ( ) ,
health_contract : health_contract ( " test.contract " , default_failure_state ) ,
health_command : Some ( " exit 0 " . to_string ( ) ) ,
deploy_command : deploy_command . map ( ToString ::to_string ) ,
restart_command : Some ( " restart test " . to_string ( ) ) ,
enabled : true ,
interval_seconds : 30 ,
}
}
#[ test ]
fn target_catalog_requires_health_contract ( ) {
let mut target = target ( " failed " , None ) ;
target . health_contract . id = String ::new ( ) ;
let error = validate_targets ( & [ target ] ) . unwrap_err ( ) . to_string ( ) ;
assert! ( error . contains ( " has no health contract " ) ) ;
}
#[ test ]
fn stale_deployment_contract_requires_deploy_authority ( ) {
let error = validate_targets ( & [ target ( " stale-deployment " , None ) ] )
. unwrap_err ( )
. to_string ( ) ;
assert! ( error . contains ( " has no deploy command " ) ) ;
}
#[ test ]
fn health_state_uses_contract_default_when_probe_has_no_marker ( ) {
let target = target ( " dependency-unavailable " , None ) ;
assert_eq! (
health_state_from_detail ( " plain command failure " , & target ) ,
" dependency-unavailable "
) ;
}
2026-06-15 22:35:00 +01:00
2026-06-15 23:36:52 +01:00
#[ test ]
2026-06-16 01:27:45 +01:00
fn swarm_surgery_plan_names_odin_after_muninn_cuts ( ) {
2026-06-16 00:45:48 +01:00
let starfire_muninn = DaemonTarget {
2026-06-15 23:36:52 +01:00
daemon_id : " starfire-muninn " . to_string ( ) ,
verse_id : " starfire.local " . to_string ( ) ,
name : " Starfire Muninn " . to_string ( ) ,
health_contract : health_contract (
" muninn.cultnet-rudp-local-telemetry-and-quest-access " ,
" degraded " ,
) ,
health_command : Some ( " health-starfire-muninn.cmd " . to_string ( ) ) ,
deploy_command : None ,
restart_command : Some ( " restart-starfire-muninn.cmd " . to_string ( ) ) ,
enabled : true ,
interval_seconds : 30 ,
} ;
2026-06-16 00:45:48 +01:00
let nightwing_muninn = DaemonTarget {
daemon_id : " nightwing-muninn " . to_string ( ) ,
verse_id : " nightwing.local " . to_string ( ) ,
name : " Nightwing Muninn " . to_string ( ) ,
health_contract : health_contract (
" muninn.cultnet-rudp-remote-telemetry-and-move-hid " ,
" failed " ,
) ,
health_command : Some ( " health-nightwing-muninn.cmd " . to_string ( ) ) ,
deploy_command : None ,
restart_command : Some ( " restart-nightwing-muninn.cmd " . to_string ( ) ) ,
enabled : true ,
interval_seconds : 30 ,
} ;
let raven_muninn = DaemonTarget {
daemon_id : " muninn " . to_string ( ) ,
verse_id : " raven.local " . to_string ( ) ,
name : " Raven Muninn " . to_string ( ) ,
health_contract : health_contract (
" muninn.cultnet-rudp-remote-telemetry-health " ,
" failed " ,
) ,
health_command : Some ( " health-muninn.cmd " . to_string ( ) ) ,
deploy_command : None ,
restart_command : Some ( " restart-muninn.cmd " . to_string ( ) ) ,
enabled : true ,
interval_seconds : 30 ,
} ;
2026-06-15 23:36:52 +01:00
let odin = DaemonTarget {
daemon_id : " odin " . to_string ( ) ,
verse_id : " starfire.local " . to_string ( ) ,
name : " Odin " . to_string ( ) ,
health_contract : health_contract ( " odin.cultnet-rudp-provider-health " , " failed " ) ,
health_command : Some ( " health-odin.cmd " . to_string ( ) ) ,
deploy_command : None ,
restart_command : Some ( " restart-odin.cmd " . to_string ( ) ) ,
enabled : true ,
interval_seconds : 30 ,
} ;
2026-06-16 00:45:48 +01:00
let plan = swarm_surgery_plan (
" starfire-local " ,
& [ odin , starfire_muninn , nightwing_muninn , raven_muninn ] ,
" unix:100 " ,
) ;
2026-06-15 23:36:52 +01:00
assert_eq! ( plan . plan_id , " swarm-surgery:starfire-local " ) ;
assert_eq! ( plan . status , " active-transport-migration " ) ;
2026-06-16 01:27:45 +01:00
assert_eq! ( plan . next_target , " odin " ) ;
assert! ( plan . current_phase . contains ( " Odin " ) ) ;
assert! ( plan . cut_line . contains ( " All Muninn lanes " ) ) ;
2026-06-15 23:36:52 +01:00
assert! ( plan . verification_layer . contains ( " CultMesh keepalive store " ) ) ;
assert! (
plan . invariants
. iter ( )
. any ( | invariant | invariant . contains ( " cultnet.transport.rudp.v0 " ) )
) ;
}
2026-06-15 23:12:39 +01:00
#[ test ]
fn fresh_daemon_published_rudp_health_can_replace_probe_health ( ) {
let desired = IdunnDesiredDaemonRecord {
daemon_id : " test-daemon " . to_string ( ) ,
verse_id : " test.local " . to_string ( ) ,
name : " Test daemon " . to_string ( ) ,
enabled : true ,
health_command : Some ( " exit 1 " . to_string ( ) ) ,
restart_command : Some ( " restart test " . to_string ( ) ) ,
authority : " idunn.local-command " . to_string ( ) ,
max_silence_seconds : 60 ,
observed_at : " unix:100 " . to_string ( ) ,
deploy_command : None ,
health_contract : " test.cultnet-rudp-health " . to_string ( ) ,
transport_profile_id : " transport:test-daemon " . to_string ( ) ,
command_boundary_id : " command-boundary:test-daemon " . to_string ( ) ,
} ;
let mut health = IdunnDaemonHealthRecord {
daemon_id : " test-daemon " . to_string ( ) ,
state : " active " . to_string ( ) ,
detail : " daemon published " . to_string ( ) ,
observed_at : " unix:95 " . to_string ( ) ,
health_contract : " test.cultnet-rudp-health " . to_string ( ) ,
publication_source : " daemon-published " . to_string ( ) ,
transport : CULTNET_RUDP_PROTOCOL_ID . to_string ( ) ,
} ;
assert! ( is_fresh_daemon_published_health (
& health , & desired , " unix:100 "
) ) ;
health . observed_at = " unix:1 " . to_string ( ) ;
assert! ( ! is_fresh_daemon_published_health (
& health , & desired , " unix:100 "
) ) ;
health . observed_at = " unix:95 " . to_string ( ) ;
health . publication_source = " compatibility-command " . to_string ( ) ;
assert! ( ! is_fresh_daemon_published_health (
& health , & desired , " unix:100 "
) ) ;
}
2026-06-15 22:35:00 +01:00
#[ test ]
fn runtime_transport_check_proves_rudp_loopback ( ) {
let check = runtime_transport_check ( " 2026-06-15T00:00:00Z " ) ;
assert_eq! ( check . check_id , " idunn-runtime-rudp-loopback " ) ;
assert_eq! ( check . transport , CULTNET_RUDP_PROTOCOL_ID ) ;
assert_eq! ( check . state , " available " ) ;
}
2026-06-15 22:50:43 +01:00
#[ test ]
fn rudp_health_ingress_accepts_raw_daemon_health_document ( ) {
let health = IdunnDaemonHealthRecord {
daemon_id : " test-daemon " . to_string ( ) ,
state : " active " . to_string ( ) ,
detail : " published by daemon " . to_string ( ) ,
observed_at : " 2026-06-15T00:00:00Z " . to_string ( ) ,
health_contract : " test.cultnet-rudp-health " . to_string ( ) ,
2026-06-15 23:12:39 +01:00
publication_source : " daemon-published " . to_string ( ) ,
transport : CULTNET_RUDP_PROTOCOL_ID . to_string ( ) ,
2026-06-15 22:50:43 +01:00
} ;
let message = CultNetMessage ::DocumentPutRaw {
message_id : " test-health " . to_string ( ) ,
document : CultNetRawDocumentRecord {
schema_id : " idunn.daemon_health " . to_string ( ) ,
record_key : health . daemon_id . clone ( ) ,
stored_at : " 2026-06-15T00:00:00Z " . to_string ( ) ,
payload_encoding : CultNetRawPayloadEncoding ::Messagepack ,
payload : rmp_serde ::to_vec ( & health ) . unwrap ( ) ,
source_runtime_id : Some ( " test-daemon " . to_string ( ) ) ,
source_agent_id : None ,
source_role : Some ( " daemon-health-publisher " . to_string ( ) ) ,
tags : Some ( vec! [ " cultnet.transport.rudp.v0 " . to_string ( ) ] ) ,
} ,
} ;
assert_eq! ( health_from_rudp_message ( & message ) . unwrap ( ) , health ) ;
}
2026-06-15 20:50:57 +01:00
}