From 98dd33899418b3e7c22b3183d4fdfc5a57f9c0e4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 15 Jul 2026 11:47:54 +0100 Subject: [PATCH] Replace false SCM authority with durable task ownership --- .../src/bin/epiphany-daemon-supervisor.rs | 2227 ++++------------- .../src/bin/epiphany-hands-action.rs | 109 +- .../src/bin/epiphany-mvp-coordinator.rs | 71 + epiphany-core/src/bin/epiphany-verse-query.rs | 67 +- notes/fresh-workspace-handoff.md | 39 + state/ledgers.msgpack | Bin 501332 -> 501870 bytes state/map.yaml | 7 +- tools/epiphany_local_run.ps1 | 1042 +------- 8 files changed, 821 insertions(+), 2741 deletions(-) diff --git a/epiphany-core/src/bin/epiphany-daemon-supervisor.rs b/epiphany-core/src/bin/epiphany-daemon-supervisor.rs index 31408ca0..24b28093 100644 --- a/epiphany-core/src/bin/epiphany-daemon-supervisor.rs +++ b/epiphany-core/src/bin/epiphany-daemon-supervisor.rs @@ -15,13 +15,10 @@ use epiphany_core::EpiphanyCultMeshManagedServicePolicyEntry; use epiphany_core::EpiphanyCultMeshSwarmBrakeEntry; use epiphany_core::EpiphanyLocalVerseContext; use epiphany_core::EpiphanyProcessObservation as ProcessObservation; -use epiphany_core::EpiphanyServiceExecutionAuditReport; use epiphany_core::MemorySemanticProjectionInput; use epiphany_core::agent_memory_semantic_projection_input; -use epiphany_core::epiphany_cluster_service_execution_audit_report; use epiphany_core::epiphany_cultmesh_daemon_poke_intent_from_status; use epiphany_core::epiphany_cultmesh_daemon_poke_receipt_for_intent; -use epiphany_core::epiphany_service_execution_audit_report; use epiphany_core::idunn_recover_memory_semantic_projection_from_cultmesh; use epiphany_core::load_epiphany_cultmesh_cluster_topology; use epiphany_core::load_epiphany_cultmesh_daemon_restart_policy; @@ -41,7 +38,6 @@ use epiphany_core::retire_memory_semantic_projection_claims_v0; use epiphany_core::retire_orphaned_memory_semantic_projection_attempts_v0; use epiphany_core::retire_unowned_memory_semantic_index_receipts; use epiphany_core::runtime_modeling_semantic_projection_input; -use epiphany_core::seed_epiphany_local_verse_context; use epiphany_core::write_epiphany_cultmesh_daemon_poke_intent; use epiphany_core::write_epiphany_cultmesh_daemon_poke_receipt; use epiphany_core::write_epiphany_cultmesh_daemon_restart_policy; @@ -68,11 +64,28 @@ use std::os::windows::process::CommandExt; fn main() -> Result<()> { let args = Args::parse()?; + let fatal_log = args.fatal_log.clone(); + let result = dispatch(args); + if let Err(error) = &result + && let Some(path) = fatal_log + { + let _ = append_fatal_log(&path, error); + } + result +} + +fn dispatch(args: Args) -> Result<()> { match args.command.as_str() { "reconcile" | "poke" | "restart" => reconcile(args), "tick" | "schedule" | "reconcile-all" => tick(args), "serve" | "loop" | "daemon" => serve(args), "managed-service-serve" | "service-desired-state-serve" => managed_service_serve(args), + "managed-service-task-plan" => managed_service_task_install(args, false), + "managed-service-task-install" => managed_service_task_install(args, true), + "managed-service-task-status" => managed_service_task_status(args), + "managed-service-task-start" => managed_service_task_control(args, "start"), + "managed-service-task-stop" => managed_service_task_control(args, "stop"), + "managed-service-task-uninstall" => managed_service_task_control(args, "uninstall"), "migrate-retired-operator-status" => migrate_retired_operator_status(args), "migrate-semantic-attempts-v0" => migrate_semantic_attempts_v0(args), "semantic-projector-service-status" => semantic_projector_service_status(args), @@ -89,74 +102,387 @@ fn main() -> Result<()> { | "repo-work-service-readiness" | "repo-work-queue-runner-audit" => repo_work_service_audit(args), "cluster-service-runbook" | "cluster-daemon-runbook" => cluster_daemon_runbook(args), - "cluster-service-install-plan" | "cluster-daemon-install-plan" => { - let mut args = args; - args.execute_install = false; - cluster_windows_service_install(args) - } - "cluster-service-install-execute" => { - let mut args = args; - args.execute_install = true; - cluster_windows_service_install(args) - } - "cluster-windows-service-audit" | "cluster-service-audit" | "cluster-service-readiness" => { - cluster_windows_service_audit(args) - } - "cluster-windows-service-start" | "cluster-service-start" => { - cluster_windows_service_control(args, "start") - } - "cluster-windows-service-stop" | "cluster-service-stop" => { - cluster_windows_service_control(args, "stop") - } - "cluster-windows-service-execution-readiness" | "cluster-service-execution-readiness" => { - cluster_windows_service_execution_readiness(args) - } - "cluster-windows-service-execution-runbook" | "cluster-service-execution-runbook" => { - cluster_windows_service_execution_runbook(args) - } - "cluster-windows-service-execution-audit" | "cluster-service-execution-audit" => { - cluster_windows_service_execution_audit(args) - } + "cluster-service-install-plan" + | "cluster-daemon-install-plan" + | "cluster-service-install-execute" => refuse_false_windows_scm_authority(), + "cluster-windows-service-audit" + | "cluster-service-audit" + | "cluster-service-readiness" + | "cluster-windows-service-start" + | "cluster-service-start" + | "cluster-windows-service-stop" + | "cluster-service-stop" => refuse_false_windows_scm_authority(), + "cluster-windows-service-execution-readiness" + | "cluster-service-execution-readiness" + | "cluster-windows-service-execution-runbook" + | "cluster-service-execution-runbook" + | "cluster-windows-service-execution-audit" + | "cluster-service-execution-audit" => refuse_false_windows_scm_authority(), "cluster-windows-service-execution-audit-smoke" - | "cluster-service-execution-audit-smoke" => { - cluster_windows_service_execution_audit_smoke(args) - } - "service-install-plan" => { - let mut args = args; - args.execute_install = false; - windows_service_install(args) - } - "service-install-execute" => { - let mut args = args; - args.execute_install = true; - windows_service_install(args) - } + | "cluster-service-execution-audit-smoke" => refuse_false_windows_scm_authority(), + "service-install-plan" | "service-install-execute" => refuse_false_windows_scm_authority(), "windows-service-execution-readiness" | "service-execution-readiness" - | "service-elevation-status" => windows_service_execution_readiness(args), + | "service-elevation-status" => refuse_false_windows_scm_authority(), "windows-service-execution-runbook" | "service-execution-runbook" => { - windows_service_execution_runbook(args) + refuse_false_windows_scm_authority() } "windows-service-reconcile" | "service-reconcile" | "service-policy-reconcile" => { - windows_service_reconcile(args) + refuse_false_windows_scm_authority() } "windows-service-execution-audit" | "service-execution-audit" => { - windows_service_execution_audit(args) + refuse_false_windows_scm_authority() } "service-execution-audit-smoke" | "windows-service-execution-audit-smoke" => { - windows_service_execution_audit_smoke(args) + refuse_false_windows_scm_authority() } - "windows-service-status" | "service-status" => windows_service_status(args), - "windows-service-start" | "service-start" => windows_service_control(args, "start"), - "windows-service-stop" | "service-stop" => windows_service_control(args, "stop"), + "windows-service-status" + | "service-status" + | "windows-service-start" + | "service-start" + | "windows-service-stop" + | "service-stop" => refuse_false_windows_scm_authority(), "policy" | "write-policy" => write_policy(args), "semantic-recover" => semantic_recover(args), other => anyhow::bail!( - "unknown command {other:?}; use reconcile, tick, serve, service-plan, service-launch, service-runbook, managed-service-policy, semantic-projector-service-policy, repo-work-service-audit, cluster-service-runbook, cluster-service-install-plan, cluster-service-audit, cluster-service-start, cluster-service-stop, cluster-service-execution-readiness, cluster-service-execution-runbook, cluster-service-execution-audit, windows-service-install, windows-service-execution-readiness, windows-service-execution-runbook, windows-service-execution-audit, service-execution-audit-smoke, windows-service-reconcile, windows-service-status, windows-service-start, windows-service-stop, policy, or semantic-recover" + "unknown command {other:?}; use reconcile, tick, serve, managed-service-serve, managed-service-task-plan/install/status/start/stop/uninstall, service-plan, service-launch, service-runbook, managed-service-policy, semantic-projector-service-policy, repo-work-service-audit, cluster-service-runbook, policy, or semantic-recover" ), } } +fn append_fatal_log(path: &Path, error: &anyhow::Error) -> Result<()> { + use std::io::Write; + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let mut file = fs::OpenOptions::new() + .create(true) + .append(true) + .open(path)?; + writeln!(file, "{} {}", Utc::now().to_rfc3339(), error)?; + Ok(()) +} + +fn refuse_false_windows_scm_authority() -> Result<()> { + anyhow::bail!( + "Windows SCM deployment is unsupported: this executable is a foreground console process, not a Windows service host; use managed-service-task-plan/install/status/start/stop/uninstall" + ) +} + +fn managed_service_task_name(args: &Args) -> String { + args.service_name + .clone() + .unwrap_or_else(|| "Epiphany-Idunn-Managed-Service-Reconciler".to_string()) +} + +fn managed_service_task_action(args: &Args) -> Result<(PathBuf, PathBuf, Vec)> { + let command = fs::canonicalize(service_command_path(args)?) + .context("managed-service task command must be an existing absolute executable")?; + let cwd = fs::canonicalize( + args.cwd + .clone() + .unwrap_or(env::current_dir().context("failed to resolve task working directory")?), + ) + .context("managed-service task working directory must exist")?; + if !command.is_absolute() || !cwd.is_absolute() { + anyhow::bail!("managed-service task command and working directory must be absolute"); + } + let mut action_args = vec![ + "managed-service-serve".to_string(), + "--store".to_string(), + absolutize_from(&cwd, &args.store).display().to_string(), + "--runtime-id".to_string(), + args.runtime_id.clone(), + "--loop-interval-seconds".to_string(), + args.loop_interval_seconds.to_string(), + "--fatal-log".to_string(), + absolutize_from( + &cwd, + args.fatal_log.as_deref().unwrap_or_else(|| { + Path::new(".epiphany-run/services/managed-service-scheduler.fatal.log") + }), + ) + .display() + .to_string(), + ]; + if args.max_iterations != 0 { + action_args.extend([ + "--max-iterations".to_string(), + args.max_iterations.to_string(), + ]); + } + Ok((command, cwd, action_args)) +} + +fn absolutize_from(cwd: &Path, path: &Path) -> PathBuf { + if path.is_absolute() { + path.to_path_buf() + } else { + cwd.join(path) + } +} + +fn managed_service_task_install(args: Args, execute: bool) -> Result<()> { + require_supervisor_bootstrap(&args)?; + let context = query_epiphany_local_verse_context(&args.store, args.runtime_id.clone())?; + assert_swarm_brake_allows_service_lifecycle(&context)?; + let started_at = Utc::now(); + let task_name = managed_service_task_name(&args); + let (command, cwd, action_args) = managed_service_task_action(&args)?; + let argument_line = windows_command_line(&action_args); + let delay = format!("PT{}S", args.task_logon_delay_seconds); + let register_script = format!( + "$ErrorActionPreference='Stop'; $user=[System.Security.Principal.WindowsIdentity]::GetCurrent().Name; \ +$action=New-ScheduledTaskAction -Execute {} -Argument {} -WorkingDirectory {}; \ +$trigger=New-ScheduledTaskTrigger -AtLogOn -User $user; $trigger.Delay={}; \ +$recoveryTrigger=New-ScheduledTaskTrigger -Once -At (Get-Date).AddMinutes(1) -RepetitionInterval ([TimeSpan]::FromSeconds({})); \ +$principal=New-ScheduledTaskPrincipal -UserId $user -LogonType Interactive -RunLevel Limited; \ +$settings=New-ScheduledTaskSettingsSet -MultipleInstances IgnoreNew -RestartCount {} -RestartInterval ([TimeSpan]::FromSeconds({})) -ExecutionTimeLimit ([TimeSpan]::Zero) -StartWhenAvailable -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries; \ +Register-ScheduledTask -TaskName {} -Action $action -Trigger @($trigger,$recoveryTrigger) -Principal $principal -Settings $settings -Description 'Idunn foreground managed-service reconciler; typed local Verse policy remains authority.' -Force | Out-Null", + quote_powershell(&command.display().to_string()), + quote_powershell(&argument_line), + quote_powershell(&cwd.display().to_string()), + quote_powershell(&delay), + args.task_restart_interval_seconds, + args.task_restart_count, + args.task_restart_interval_seconds, + quote_powershell(&task_name), + ); + let readback_script = managed_service_task_readback_script( + &task_name, + &command, + &argument_line, + &cwd, + &delay, + args.task_restart_count, + args.task_restart_interval_seconds, + ); + let script = format!("{register_script}; {readback_script}"); + let (status, exit_code, stdout, stderr) = if execute { + let output = run_powershell(&script)?; + let drifted = output.status.success() && task_readback_has_drift(&output.stdout)?; + let status = if !output.status.success() { + "install-failed" + } else if drifted { + "install-drift" + } else { + "installed" + }; + ( + status, + output.status.code(), + String::from_utf8_lossy(&output.stdout).trim().to_string(), + String::from_utf8_lossy(&output.stderr).trim().to_string(), + ) + } else { + ("planned", None, String::new(), String::new()) + }; + let receipt = service_lifecycle_receipt( + &args, + "managed-service-task-install", + status, + command.display().to_string(), + action_args.clone(), + None, + exit_code, + started_at, + Some(Utc::now()), + Some(format!("task-scheduler://windows/{task_name}")), + ); + let written = write_epiphany_cultmesh_daemon_service_lifecycle_receipt( + &args.store, + args.runtime_id.clone(), + receipt, + )?; + println!( + "{}", + serde_json::to_string_pretty(&json!({ + "schemaVersion":"epiphany.windows.managed_service_task.v0", "status":status, + "taskName":task_name, "receiptId":written.receipt_id, "executeRequested":execute, + "principal":{"user":"current-user","logonType":"InteractiveToken","runLevel":"Limited"}, + "trigger":{"kind":"AtLogOn","delaySeconds":args.task_logon_delay_seconds}, + "settings":{"multipleInstances":"IgnoreNew","restartCount":args.task_restart_count, + "restartIntervalSeconds":args.task_restart_interval_seconds,"executionTimeLimitSeconds":0, + "startWhenAvailable":true,"allowStartIfOnBatteries":true,"stopIfGoingOnBatteries":false}, + "action":{"command":command,"args":action_args,"workingDirectory":cwd}, + "exitCode":exit_code,"stdout":stdout,"stderr":stderr,"privateStateExposed":false + }))? + ); + if execute && status != "installed" { + anyhow::bail!( + "scheduled task installation did not verify: stdout={stdout}; stderr={stderr}" + ); + } + Ok(()) +} + +fn managed_service_task_status(args: Args) -> Result<()> { + managed_service_task_operation(args, "status") +} + +fn managed_service_task_control(args: Args, operation: &str) -> Result<()> { + managed_service_task_operation(args, operation) +} + +fn managed_service_task_operation(args: Args, operation: &str) -> Result<()> { + require_supervisor_bootstrap(&args)?; + let context = query_epiphany_local_verse_context(&args.store, args.runtime_id.clone())?; + assert_swarm_brake_allows_service_lifecycle(&context)?; + let started_at = Utc::now(); + let task_name = managed_service_task_name(&args); + let quoted = quote_powershell(&task_name); + let script = match operation { + "status" => { + let (command, cwd, action_args) = managed_service_task_action(&args)?; + managed_service_task_readback_script( + &task_name, + &command, + &windows_command_line(&action_args), + &cwd, + &format!("PT{}S", args.task_logon_delay_seconds), + args.task_restart_count, + args.task_restart_interval_seconds, + ) + } + "start" => format!("$ErrorActionPreference='Stop'; Start-ScheduledTask -TaskName {quoted}"), + "stop" => format!("$ErrorActionPreference='Stop'; Stop-ScheduledTask -TaskName {quoted}"), + "uninstall" => format!( + "$ErrorActionPreference='Stop'; Unregister-ScheduledTask -TaskName {quoted} -Confirm:$false" + ), + _ => anyhow::bail!("unsupported managed-service task operation {operation}"), + }; + let output = run_powershell(&script)?; + let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string(); + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + let status = if operation == "status" && output.status.code() == Some(3) { + "missing".to_string() + } else if operation == "status" + && output.status.success() + && task_readback_has_drift(&output.stdout)? + { + "drift".to_string() + } else if output.status.success() { + if operation == "status" { + "in-sync".to_string() + } else { + format!("{operation}-requested") + } + } else { + format!("{operation}-failed") + }; + let receipt = service_lifecycle_receipt( + &args, + &format!("managed-service-task-{operation}"), + &status, + "powershell.exe".to_string(), + vec!["-NoProfile".to_string(), "-Command".to_string(), script], + None, + output.status.code(), + started_at, + Some(Utc::now()), + Some(format!("task-scheduler://windows/{task_name}/{operation}")), + ); + let written = write_epiphany_cultmesh_daemon_service_lifecycle_receipt( + &args.store, + args.runtime_id.clone(), + receipt, + )?; + println!( + "{}", + serde_json::to_string_pretty( + &json!({"schemaVersion":"epiphany.windows.managed_service_task.v0", + "status":status,"taskName":task_name,"operation":operation,"receiptId":written.receipt_id, + "exitCode":output.status.code(),"stdout":stdout,"stderr":stderr,"privateStateExposed":false}) + )? + ); + if !output.status.success() && !(operation == "status" && output.status.code() == Some(3)) { + anyhow::bail!("scheduled task {operation} failed: {stderr}"); + } + Ok(()) +} + +fn managed_service_task_readback_script( + task_name: &str, + command: &Path, + arguments: &str, + cwd: &Path, + delay: &str, + restart_count: u32, + restart_interval_seconds: u64, +) -> String { + format!( + "$ErrorActionPreference='Stop'; $expectedSid=[System.Security.Principal.WindowsIdentity]::GetCurrent().User.Value; function Resolve-TaskSid($id){{try{{(New-Object System.Security.Principal.NTAccount($id)).Translate([System.Security.Principal.SecurityIdentifier]).Value}}catch{{''}}}}; $t=Get-ScheduledTask -TaskName {} -ErrorAction SilentlyContinue; if($null -eq $t){{'missing';exit 3}}; $i=Get-ScheduledTaskInfo -TaskName {}; $d=@(); \ +if($t.Actions[0].Execute -ne {}){{$d+='action.execute'}}; if($t.Actions[0].Arguments -ne {}){{$d+='action.arguments'}}; if($t.Actions[0].WorkingDirectory -ne {}){{$d+='action.workingDirectory'}}; \ +if((Resolve-TaskSid $t.Principal.UserId) -ne $expectedSid){{$d+='principal.userId'}}; if([string]$t.Principal.LogonType -notin @('Interactive','InteractiveToken')){{$d+='principal.logonType'}}; if([string]$t.Principal.RunLevel -ne 'Limited'){{$d+='principal.runLevel'}}; \ +if($t.Triggers.Count -ne 2){{$d+='trigger.count'}}; if($t.Triggers[0].CimClass.CimClassName -ne 'MSFT_TaskLogonTrigger'){{$d+='trigger.kind'}}; if((Resolve-TaskSid $t.Triggers[0].UserId) -ne $expectedSid){{$d+='trigger.userId'}}; if([Xml.XmlConvert]::ToTimeSpan([string]$t.Triggers[0].Delay).TotalSeconds -ne [Xml.XmlConvert]::ToTimeSpan({}).TotalSeconds){{$d+='trigger.delay'}}; if($t.Triggers[1].CimClass.CimClassName -ne 'MSFT_TaskTimeTrigger'){{$d+='recoveryTrigger.kind'}}; if([Xml.XmlConvert]::ToTimeSpan([string]$t.Triggers[1].Repetition.Interval).TotalSeconds -ne {}){{$d+='recoveryTrigger.interval'}}; \ +if([string]$t.Settings.MultipleInstances -ne 'IgnoreNew'){{$d+='settings.multipleInstances'}}; if($t.Settings.RestartCount -ne {}){{$d+='settings.restartCount'}}; if([Xml.XmlConvert]::ToTimeSpan([string]$t.Settings.RestartInterval).TotalSeconds -ne {}){{$d+='settings.restartInterval'}}; \ +if([Xml.XmlConvert]::ToTimeSpan([string]$t.Settings.ExecutionTimeLimit).TotalSeconds -ne 0){{$d+='settings.executionTimeLimit'}}; if($t.Settings.DisallowStartIfOnBatteries){{$d+='settings.allowStartIfOnBatteries'}}; if($t.Settings.StopIfGoingOnBatteries){{$d+='settings.stopIfGoingOnBatteries'}}; \ +[pscustomobject]@{{TaskName=$t.TaskName;State=[string]$t.State;LastRunTime=$i.LastRunTime;LastTaskResult=$i.LastTaskResult;NextRunTime=$i.NextRunTime;Execute=$t.Actions[0].Execute;Arguments=$t.Actions[0].Arguments;WorkingDirectory=$t.Actions[0].WorkingDirectory;UserId=$t.Principal.UserId;LogonType=[string]$t.Principal.LogonType;RunLevel=[string]$t.Principal.RunLevel;DriftReasons=$d}}|ConvertTo-Json -Compress", + quote_powershell(task_name), + quote_powershell(task_name), + quote_powershell(&command.display().to_string()), + quote_powershell(arguments), + quote_powershell(&cwd.display().to_string()), + quote_powershell(delay), + restart_interval_seconds, + restart_count, + restart_interval_seconds, + ) +} + +fn task_readback_has_drift(stdout: &[u8]) -> Result { + let value: Value = + serde_json::from_slice(stdout).context("failed to parse scheduled-task readback")?; + Ok(value + .get("DriftReasons") + .and_then(Value::as_array) + .is_none_or(|reasons| !reasons.is_empty())) +} + +fn run_powershell(script: &str) -> Result { + Command::new("powershell.exe") + .arg("-NoProfile") + .arg("-NonInteractive") + .arg("-Command") + .arg(script) + .output() + .context("failed to invoke Windows Task Scheduler through PowerShell") +} + +fn windows_command_line(args: &[String]) -> String { + args.iter() + .map(|arg| windows_quote_argv(arg)) + .collect::>() + .join(" ") +} + +fn windows_quote_argv(arg: &str) -> String { + if arg.is_empty() { + return "\"\"".to_string(); + } + if !arg.chars().any(|ch| ch.is_whitespace() || ch == '"') { + return arg.to_string(); + } + let mut quoted = String::from("\""); + let mut backslashes = 0_usize; + for ch in arg.chars() { + if ch == '\\' { + backslashes += 1; + } else if ch == '"' { + quoted.push_str(&"\\".repeat(backslashes * 2 + 1)); + quoted.push('"'); + backslashes = 0; + } else { + quoted.push_str(&"\\".repeat(backslashes)); + backslashes = 0; + quoted.push(ch); + } + } + quoted.push_str(&"\\".repeat(backslashes * 2)); + quoted.push('"'); + quoted +} + fn migrate_retired_operator_status(args: Args) -> Result<()> { let removed_keys = retire_epiphany_cultmesh_operator_status_documents(&args.store)?; println!( @@ -304,31 +630,6 @@ fn require_supervisor_bootstrap(args: &Args) -> Result<()> { Ok(()) } -fn seed_supervisor_smoke_fixture(args: &Args) -> Result<()> { - if args - .store - .components() - .any(|component| matches!(component, std::path::Component::ParentDir)) - { - anyhow::bail!("daemon-supervisor smoke store may not contain parent traversal"); - } - let workspace = env::current_dir()?.canonicalize()?; - let smoke_root = workspace.join(".epiphany-smoke"); - let store = if args.store.is_absolute() { - args.store.clone() - } else { - workspace.join(&args.store) - }; - if !store.starts_with(&smoke_root) { - anyhow::bail!("daemon-supervisor smoke fixtures may write only beneath .epiphany-smoke"); - } - seed_epiphany_local_verse_context( - &args.store, - args.runtime_id.clone(), - Utc::now().to_rfc3339(), - ) -} - fn reconcile(args: Args) -> Result<()> { require_supervisor_bootstrap(&args)?; @@ -1262,1080 +1563,6 @@ fn cluster_daemon_runbook(args: Args) -> Result<()> { Ok(()) } -fn cluster_windows_service_install(args: Args) -> Result<()> { - require_supervisor_bootstrap(&args)?; - - let context = query_epiphany_local_verse_context(&args.store, args.runtime_id.clone())?; - assert_swarm_brake_allows_service_lifecycle(&context)?; - let started_at = Utc::now(); - let command_path = cluster_daemon_command_path(&args)?; - let daemon_rows = cluster_daemon_runbook_rows(&args, &context)?; - let install_script_path = windows_service_install_script_path(&args); - if let Some(parent) = install_script_path.parent() { - fs::create_dir_all(parent) - .with_context(|| format!("failed to create {}", parent.display()))?; - } - let service_specs = daemon_rows - .iter() - .map(|row| cluster_windows_service_spec(&args, row)) - .collect::>(); - fs::write( - &install_script_path, - cluster_windows_service_install_script_content( - &args, - &command_path, - &daemon_rows, - &service_specs, - ), - ) - .with_context(|| format!("failed to write {}", install_script_path.display()))?; - let artifact_ref = install_script_path.display().to_string(); - let mut status = "planned".to_string(); - let mut exit_code = None; - let mut executed = false; - let mut stdout = String::new(); - let mut stderr = String::new(); - let mut completed_at = Some(Utc::now()); - if args.execute_install { - if windows_has_elevated_service_authority()? { - executed = true; - let output = Command::new("powershell.exe") - .arg("-NoProfile") - .arg("-ExecutionPolicy") - .arg("Bypass") - .arg("-File") - .arg(&install_script_path) - .output() - .with_context(|| { - format!( - "failed to execute cluster Windows service install script {}", - install_script_path.display() - ) - })?; - exit_code = output.status.code(); - stdout = String::from_utf8_lossy(&output.stdout).to_string(); - stderr = String::from_utf8_lossy(&output.stderr).to_string(); - completed_at = Some(Utc::now()); - status = if output.status.success() { - "install-command-succeeded".to_string() - } else { - "failed".to_string() - }; - } else { - status = "execution-refused-not-elevated".to_string(); - } - } - let service_args = daemon_rows - .iter() - .flat_map(|row| { - let mut values = vec![format!("daemon={}", row.daemon_id)]; - values.extend(row.args.clone()); - values - }) - .collect::>(); - let receipt = service_lifecycle_receipt( - &args, - "cluster-windows-service-install", - &status, - "powershell.exe".to_string(), - vec![ - "-NoProfile".to_string(), - "-ExecutionPolicy".to_string(), - "Bypass".to_string(), - "-File".to_string(), - artifact_ref.clone(), - format!("cluster-daemon-count={}", daemon_rows.len()), - format!("command={}", command_path.display()), - format!("args={}", service_args.join(" ")), - ], - None, - exit_code, - started_at, - completed_at, - Some(artifact_ref.clone()), - ); - let written = write_epiphany_cultmesh_daemon_service_lifecycle_receipt( - &args.store, - args.runtime_id.clone(), - receipt, - )?; - println!( - "{}", - serde_json::to_string_pretty(&json!({ - "status": written.status, - "store": args.store, - "runtimeId": args.runtime_id, - "serviceId": written.service_id, - "receiptId": written.receipt_id, - "daemonSelector": written.daemon_selector, - "daemonCount": daemon_rows.len(), - "installScriptPath": artifact_ref, - "command": written.command, - "exitCode": written.exit_code, - "stdout": stdout, - "stderr": stderr, - "executeRequested": args.execute_install, - "executed": executed, - "services": daemon_rows.iter().zip(service_specs.iter()).map(|(row, spec)| json!({ - "serviceName": spec.service_name, - "displayName": spec.display_name, - "daemonId": row.daemon_id, - "clusterId": row.cluster_id, - "observedStatus": row.observed_status, - "startType": args.service_start_type, - "privateStateExposed": false, - })).collect::>(), - "privateStateExposed": written.private_state_exposed, - }))? - ); - Ok(()) -} - -fn cluster_windows_service_audit(args: Args) -> Result<()> { - require_supervisor_bootstrap(&args)?; - - let context = query_epiphany_local_verse_context(&args.store, args.runtime_id.clone())?; - assert_swarm_brake_allows_service_lifecycle(&context)?; - let started_at = Utc::now(); - let daemon_rows = cluster_daemon_runbook_rows(&args, &context)?; - let service_specs = daemon_rows - .iter() - .map(|row| cluster_windows_service_spec(&args, row)) - .collect::>(); - let mut rows = Vec::new(); - let mut missing_count = 0_usize; - let mut running_count = 0_usize; - let mut present_count = 0_usize; - let mut query_failed_count = 0_usize; - for (daemon, spec) in daemon_rows.iter().zip(service_specs.iter()) { - let script = format!( - "$s = Get-Service -Name {} -ErrorAction SilentlyContinue; if ($null -eq $s) {{ 'missing'; exit 3 }} else {{ \"status=$($s.Status); canStop=$($s.CanStop); serviceType=$($s.ServiceType)\" }}", - quote_powershell(&spec.service_name) - ); - let output = Command::new("powershell.exe") - .arg("-NoProfile") - .arg("-Command") - .arg(&script) - .output() - .with_context(|| format!("failed to query Windows service {}", spec.service_name))?; - let exit_code = output.status.code(); - let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); - let service_status = - windows_service_query_status(exit_code, &stdout, output.status.success()); - match service_status.as_str() { - "missing" => missing_count = missing_count.saturating_add(1), - "running" => running_count = running_count.saturating_add(1), - "present" | "stopped" => present_count = present_count.saturating_add(1), - _ => query_failed_count = query_failed_count.saturating_add(1), - } - rows.push(json!({ - "serviceName": spec.service_name, - "displayName": spec.display_name, - "daemonId": daemon.daemon_id, - "clusterId": daemon.cluster_id, - "status": service_status, - "exitCode": exit_code, - "stdout": stdout, - "stderr": stderr, - "privateStateExposed": false, - })); - } - let status = if query_failed_count > 0 { - "query-failed" - } else if missing_count > 0 { - "incomplete" - } else { - "complete" - }; - let receipt = service_lifecycle_receipt( - &args, - "cluster-windows-service-audit", - status, - "powershell.exe".to_string(), - vec![ - "-NoProfile".to_string(), - "-Command".to_string(), - format!("cluster-service-count={}", service_specs.len()), - format!("missing={missing_count}"), - format!("running={running_count}"), - format!("present={present_count}"), - format!("queryFailed={query_failed_count}"), - ], - None, - None, - started_at, - Some(Utc::now()), - Some(format!( - "service-manager://windows/{}/cluster-audit", - sanitize_id(&args.service_id) - )), - ); - let written = write_epiphany_cultmesh_daemon_service_lifecycle_receipt( - &args.store, - args.runtime_id.clone(), - receipt, - )?; - println!( - "{}", - serde_json::to_string_pretty(&json!({ - "status": written.status, - "store": args.store, - "runtimeId": args.runtime_id, - "serviceId": written.service_id, - "receiptId": written.receipt_id, - "daemonSelector": written.daemon_selector, - "daemonCount": daemon_rows.len(), - "serviceCount": service_specs.len(), - "missingCount": missing_count, - "runningCount": running_count, - "presentCount": present_count, - "queryFailedCount": query_failed_count, - "services": rows, - "privateStateExposed": written.private_state_exposed, - }))? - ); - Ok(()) -} - -fn cluster_windows_service_control(args: Args, control: &str) -> Result<()> { - require_supervisor_bootstrap(&args)?; - - let context = query_epiphany_local_verse_context(&args.store, args.runtime_id.clone())?; - assert_swarm_brake_allows_service_lifecycle(&context)?; - let started_at = Utc::now(); - let daemon_rows = cluster_daemon_runbook_rows(&args, &context)?; - let service_specs = daemon_rows - .iter() - .map(|row| cluster_windows_service_spec(&args, row)) - .collect::>(); - let mut rows = Vec::new(); - let mut requested_count = 0_usize; - let mut planned_count = 0_usize; - let mut refused_count = 0_usize; - let mut failed_count = 0_usize; - let mut executed_count = 0_usize; - let elevated = if args.execute_control { - Some(windows_has_elevated_service_authority()?) - } else { - None - }; - for (daemon, spec) in daemon_rows.iter().zip(service_specs.iter()) { - let (service_status, exit_code, stdout, stderr, executed) = if !args.execute_control { - planned_count = planned_count.saturating_add(1); - ( - format!("{control}-planned"), - None, - String::new(), - String::new(), - false, - ) - } else if elevated == Some(false) { - refused_count = refused_count.saturating_add(1); - ( - "execution-refused-not-elevated".to_string(), - None, - String::new(), - String::new(), - false, - ) - } else { - let output = Command::new("sc.exe") - .arg(control) - .arg(&spec.service_name) - .output() - .with_context(|| format!("failed to run sc.exe {control} {}", spec.service_name))?; - let service_status = if output.status.success() { - requested_count = requested_count.saturating_add(1); - format!("{control}-requested") - } else { - failed_count = failed_count.saturating_add(1); - format!("{control}-failed") - }; - executed_count = executed_count.saturating_add(1); - ( - service_status, - output.status.code(), - String::from_utf8_lossy(&output.stdout).trim().to_string(), - String::from_utf8_lossy(&output.stderr).trim().to_string(), - true, - ) - }; - rows.push(json!({ - "serviceName": spec.service_name, - "displayName": spec.display_name, - "daemonId": daemon.daemon_id, - "clusterId": daemon.cluster_id, - "status": service_status, - "exitCode": exit_code, - "stdout": stdout, - "stderr": stderr, - "executed": executed, - "privateStateExposed": false, - })); - } - let status = if failed_count > 0 { - format!("{control}-failed") - } else if refused_count > 0 { - "execution-refused-not-elevated".to_string() - } else if planned_count > 0 { - format!("{control}-planned") - } else { - format!("{control}-requested") - }; - let receipt = service_lifecycle_receipt( - &args, - &format!("cluster-windows-service-{control}"), - &status, - "sc.exe".to_string(), - vec![ - control.to_string(), - format!("cluster-service-count={}", service_specs.len()), - format!("requested={requested_count}"), - format!("planned={planned_count}"), - format!("refused={refused_count}"), - format!("failed={failed_count}"), - ], - None, - None, - started_at, - Some(Utc::now()), - Some(format!( - "service-manager://windows/{}/cluster-{control}", - sanitize_id(&args.service_id) - )), - ); - let written = write_epiphany_cultmesh_daemon_service_lifecycle_receipt( - &args.store, - args.runtime_id.clone(), - receipt, - )?; - println!( - "{}", - serde_json::to_string_pretty(&json!({ - "status": written.status, - "store": args.store, - "runtimeId": args.runtime_id, - "serviceId": written.service_id, - "receiptId": written.receipt_id, - "daemonSelector": written.daemon_selector, - "daemonCount": daemon_rows.len(), - "serviceCount": service_specs.len(), - "requestedCount": requested_count, - "plannedCount": planned_count, - "refusedCount": refused_count, - "failedCount": failed_count, - "executeRequested": args.execute_control, - "executed": executed_count > 0, - "executedCount": executed_count, - "elevated": elevated, - "services": rows, - "privateStateExposed": written.private_state_exposed, - }))? - ); - Ok(()) -} - -fn windows_service_install(args: Args) -> Result<()> { - require_supervisor_bootstrap(&args)?; - - let context = query_epiphany_local_verse_context(&args.store, args.runtime_id.clone())?; - assert_swarm_brake_allows_service_lifecycle(&context)?; - let started_at = Utc::now(); - let command_path = service_command_path(&args)?; - let service_args = service_serve_args(&args); - let service_name = windows_service_name(&args); - let install_script_path = windows_service_install_script_path(&args); - if let Some(parent) = install_script_path.parent() { - fs::create_dir_all(parent) - .with_context(|| format!("failed to create {}", parent.display()))?; - } - let content = - windows_service_install_script_content(&args, &service_name, &command_path, &service_args); - fs::write(&install_script_path, content) - .with_context(|| format!("failed to write {}", install_script_path.display()))?; - let artifact_ref = install_script_path.display().to_string(); - let mut status = "planned".to_string(); - let mut exit_code = None; - let mut executed = false; - let mut completed_at = Some(Utc::now()); - if args.execute_install { - if windows_has_elevated_service_authority()? { - executed = true; - let output = Command::new("powershell.exe") - .arg("-NoProfile") - .arg("-ExecutionPolicy") - .arg("Bypass") - .arg("-File") - .arg(&install_script_path) - .output() - .with_context(|| { - format!( - "failed to execute Windows service install script {}", - install_script_path.display() - ) - })?; - exit_code = output.status.code(); - completed_at = Some(Utc::now()); - status = if output.status.success() { - "install-command-succeeded".to_string() - } else { - "failed".to_string() - }; - } else { - status = "execution-refused-not-elevated".to_string(); - } - } - let receipt = service_lifecycle_receipt( - &args, - if args.execute_install { - "windows-service-install" - } else { - "windows-service-install-plan" - }, - &status, - "powershell.exe".to_string(), - vec![ - "-NoProfile".to_string(), - "-ExecutionPolicy".to_string(), - "Bypass".to_string(), - "-File".to_string(), - install_script_path.display().to_string(), - ], - None, - exit_code, - started_at, - completed_at, - Some(artifact_ref.clone()), - ); - let written = write_epiphany_cultmesh_daemon_service_lifecycle_receipt( - &args.store, - args.runtime_id.clone(), - receipt, - )?; - println!( - "{}", - serde_json::to_string_pretty(&json!({ - "status": written.status, - "store": args.store, - "runtimeId": args.runtime_id, - "serviceId": written.service_id, - "serviceName": service_name, - "receiptId": written.receipt_id, - "installScriptPath": artifact_ref, - "exitCode": written.exit_code, - "executed": executed, - "privateStateExposed": written.private_state_exposed, - }))? - ); - Ok(()) -} - -fn windows_service_status(args: Args) -> Result<()> { - require_supervisor_bootstrap(&args)?; - - let context = query_epiphany_local_verse_context(&args.store, args.runtime_id.clone())?; - assert_swarm_brake_allows_service_lifecycle(&context)?; - let started_at = Utc::now(); - let service_name = windows_service_name(&args); - let script = format!( - "$s = Get-Service -Name {} -ErrorAction SilentlyContinue; if ($null -eq $s) {{ 'missing'; exit 3 }} else {{ \"status=$($s.Status); canStop=$($s.CanStop); serviceType=$($s.ServiceType)\" }}", - quote_powershell(&service_name) - ); - let output = Command::new("powershell.exe") - .arg("-NoProfile") - .arg("-Command") - .arg(&script) - .output() - .with_context(|| format!("failed to query Windows service {service_name}"))?; - let exit_code = output.status.code(); - let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); - let status = windows_service_query_status(exit_code, &stdout, output.status.success()); - let receipt = service_lifecycle_receipt( - &args, - "windows-service-status", - &status, - "powershell.exe".to_string(), - vec!["-NoProfile".to_string(), "-Command".to_string(), script], - None, - exit_code, - started_at, - Some(Utc::now()), - Some(format!("service-manager://windows/{service_name}/status")), - ); - let written = write_epiphany_cultmesh_daemon_service_lifecycle_receipt( - &args.store, - args.runtime_id.clone(), - receipt, - )?; - println!( - "{}", - serde_json::to_string_pretty(&json!({ - "status": written.status, - "store": args.store, - "runtimeId": args.runtime_id, - "serviceId": written.service_id, - "serviceName": service_name, - "receiptId": written.receipt_id, - "exitCode": written.exit_code, - "stdout": stdout, - "stderr": stderr, - "privateStateExposed": written.private_state_exposed, - }))? - ); - Ok(()) -} - -fn windows_service_execution_readiness(args: Args) -> Result<()> { - require_supervisor_bootstrap(&args)?; - - let context = query_epiphany_local_verse_context(&args.store, args.runtime_id.clone())?; - assert_swarm_brake_allows_service_lifecycle(&context)?; - let started_at = Utc::now(); - let service_name = windows_service_name(&args); - let elevated = windows_has_elevated_service_authority()?; - let status = if elevated { - "elevated-ready" - } else { - "not-elevated" - }; - let receipt = service_lifecycle_receipt( - &args, - "windows-service-execution-readiness", - status, - "powershell.exe".to_string(), - vec![ - "-NoProfile".to_string(), - "-Command".to_string(), - "IsInRole(Administrator)".to_string(), - ], - None, - None, - started_at, - Some(Utc::now()), - Some(format!( - "service-manager://windows/{service_name}/execution-readiness" - )), - ); - let written = write_epiphany_cultmesh_daemon_service_lifecycle_receipt( - &args.store, - args.runtime_id.clone(), - receipt, - )?; - println!( - "{}", - serde_json::to_string_pretty(&json!({ - "status": written.status, - "store": args.store, - "runtimeId": args.runtime_id, - "serviceId": written.service_id, - "serviceName": service_name, - "receiptId": written.receipt_id, - "elevated": elevated, - "privateStateExposed": written.private_state_exposed, - }))? - ); - Ok(()) -} - -fn cluster_windows_service_execution_readiness(args: Args) -> Result<()> { - require_supervisor_bootstrap(&args)?; - - let context = query_epiphany_local_verse_context(&args.store, args.runtime_id.clone())?; - assert_swarm_brake_allows_service_lifecycle(&context)?; - let started_at = Utc::now(); - let daemon_rows = cluster_daemon_runbook_rows(&args, &context)?; - let service_specs = daemon_rows - .iter() - .map(|row| cluster_windows_service_spec(&args, row)) - .collect::>(); - let elevated = windows_has_elevated_service_authority()?; - let status = if elevated { - "elevated-ready" - } else { - "not-elevated" - }; - let receipt = service_lifecycle_receipt( - &args, - "cluster-windows-service-execution-readiness", - status, - "powershell.exe".to_string(), - vec![ - "-NoProfile".to_string(), - "-Command".to_string(), - "IsInRole(Administrator)".to_string(), - format!("cluster-service-count={}", service_specs.len()), - ], - None, - None, - started_at, - Some(Utc::now()), - Some(format!( - "service-manager://windows/{}/cluster-execution-readiness", - sanitize_id(&args.service_id) - )), - ); - let written = write_epiphany_cultmesh_daemon_service_lifecycle_receipt( - &args.store, - args.runtime_id.clone(), - receipt, - )?; - println!( - "{}", - serde_json::to_string_pretty(&json!({ - "status": written.status, - "store": args.store, - "runtimeId": args.runtime_id, - "serviceId": written.service_id, - "receiptId": written.receipt_id, - "daemonCount": daemon_rows.len(), - "serviceCount": service_specs.len(), - "elevated": elevated, - "services": daemon_rows.iter().zip(service_specs.iter()).map(|(row, spec)| json!({ - "serviceName": spec.service_name, - "displayName": spec.display_name, - "daemonId": row.daemon_id, - "clusterId": row.cluster_id, - "privateStateExposed": false, - })).collect::>(), - "privateStateExposed": written.private_state_exposed, - }))? - ); - Ok(()) -} - -fn windows_service_execution_audit(args: Args) -> Result<()> { - require_supervisor_bootstrap(&args)?; - - let context = query_epiphany_local_verse_context(&args.store, args.runtime_id.clone())?; - assert_swarm_brake_allows_service_lifecycle(&context)?; - let started_at = Utc::now(); - let receipts = load_epiphany_cultmesh_daemon_service_lifecycle_receipts( - &args.store, - args.runtime_id.clone(), - )? - .into_iter() - .filter(|receipt| receipt.service_id == args.service_id) - .collect::>(); - let report = epiphany_service_execution_audit_report(&receipts); - let runbook_witness = - service_execution_runbook_witness(&report, "windows-service-execution-runbook"); - let receipt = service_lifecycle_receipt( - &args, - "windows-service-execution-audit", - &report.status, - "epiphany-daemon-supervisor".to_string(), - vec!["windows-service-execution-audit".to_string()], - None, - None, - started_at, - Some(Utc::now()), - Some(format!( - "service-manager://windows/{}/execution-audit", - windows_service_name(&args) - )), - ); - let written = write_epiphany_cultmesh_daemon_service_lifecycle_receipt( - &args.store, - args.runtime_id.clone(), - receipt, - )?; - println!( - "{}", - serde_json::to_string_pretty(&json!({ - "status": report.status, - "store": args.store, - "runtimeId": args.runtime_id, - "serviceId": written.service_id, - "serviceName": windows_service_name(&args), - "receiptId": written.receipt_id, - "receiptCount": report.receipt_count, - "missingCount": report.missing_count, - "failedCount": report.failed_count, - "runbookPath": runbook_witness.runbook_path, - "runbookSha256": runbook_witness.runbook_sha256, - "elevatedCommand": runbook_witness.elevated_command, - "requiresElevatedAuthority": runbook_witness.requires_elevated_authority, - "aftercareMode": "service-execution-audit", - "aftercareCommand": "tools/epiphany_local_run.ps1 -Mode service-execution-audit", - "checks": report.checks, - "privateStateExposed": written.private_state_exposed || report.private_state_exposed, - }))? - ); - Ok(()) -} - -fn windows_service_execution_audit_smoke(args: Args) -> Result<()> { - seed_supervisor_smoke_fixture(&args)?; - - let started_at = Utc::now(); - for (action, status) in [ - ("windows-service-execution-runbook", "written"), - ("windows-service-execution-readiness", "elevated-ready"), - ("windows-service-install", "install-command-succeeded"), - ("windows-service-start", "start-requested"), - ("windows-service-status", "stopped"), - ("windows-service-reconcile", "in-sync"), - ("windows-service-stop", "stop-requested"), - ] { - let receipt = service_lifecycle_receipt( - &args, - action, - status, - "smoke".to_string(), - vec![action.to_string()], - None, - Some(0), - started_at, - Some(Utc::now()), - Some(format!("smoke://service-execution-audit/{action}")), - ); - write_epiphany_cultmesh_daemon_service_lifecycle_receipt( - &args.store, - args.runtime_id.clone(), - receipt, - )?; - } - let receipts = load_epiphany_cultmesh_daemon_service_lifecycle_receipts( - &args.store, - args.runtime_id.clone(), - )? - .into_iter() - .filter(|receipt| receipt.service_id == args.service_id) - .collect::>(); - let complete_report = epiphany_service_execution_audit_report(&receipts); - if complete_report.status != "complete" - || complete_report.missing_count != 0 - || complete_report.failed_count != 0 - || complete_report.private_state_exposed - { - anyhow::bail!("service execution audit smoke failed to accept complete sealed receipts"); - } - let drifted = service_lifecycle_receipt( - &args, - "windows-service-reconcile", - "drift", - "smoke".to_string(), - vec!["windows-service-reconcile".to_string()], - None, - Some(0), - started_at, - Some(Utc::now()), - Some("smoke://service-execution-audit/drift".to_string()), - ); - write_epiphany_cultmesh_daemon_service_lifecycle_receipt( - &args.store, - args.runtime_id.clone(), - drifted, - )?; - let receipts = load_epiphany_cultmesh_daemon_service_lifecycle_receipts( - &args.store, - args.runtime_id.clone(), - )? - .into_iter() - .filter(|receipt| receipt.service_id == args.service_id) - .collect::>(); - let incomplete_report = epiphany_service_execution_audit_report(&receipts); - if incomplete_report.status != "incomplete" || incomplete_report.failed_count == 0 { - anyhow::bail!("service execution audit smoke failed to reject drifted reconcile receipt"); - } - println!( - "{}", - serde_json::to_string_pretty(&json!({ - "status": "ok", - "store": args.store, - "runtimeId": args.runtime_id, - "serviceId": args.service_id, - "completeStatus": complete_report.status, - "incompleteStatus": incomplete_report.status, - "incompleteFailedCount": incomplete_report.failed_count, - "privateStateExposed": false, - }))? - ); - Ok(()) -} - -fn cluster_windows_service_execution_runbook(args: Args) -> Result<()> { - require_supervisor_bootstrap(&args)?; - - let context = query_epiphany_local_verse_context(&args.store, args.runtime_id.clone())?; - assert_swarm_brake_allows_service_lifecycle(&context)?; - let started_at = Utc::now(); - let runbook_path = args - .runbook_path - .clone() - .context("cluster-service-execution-runbook requires --runbook-path")?; - if !runbook_path.exists() { - anyhow::bail!( - "cluster service execution runbook artifact does not exist: {}", - runbook_path.display() - ); - } - let command = args - .service_command - .as_ref() - .map(|path| path.display().to_string()) - .unwrap_or_else(|| "tools/epiphany_local_run.ps1".to_string()); - let try_modes = [ - "cluster-service-execution-readiness", - "cluster-service-install-execute", - "cluster-service-start-execute", - "cluster-service-audit", - "cluster-service-stop-execute", - "cluster-service-audit", - ]; - let final_audit_mode = "cluster-service-execution-audit"; - let step_failure_policy = "continue-after-step-failure"; - let continue_after_step_failure = true; - let nonzero_exit_fails_step = true; - let exits_nonzero_after_final_audit = true; - let final_audit_runs_in_finally = true; - let mut service_args = vec![ - format!("runbookPath={}", runbook_path.display()), - format!("tryModes={}", try_modes.join(",")), - format!("finally={final_audit_mode}"), - format!("stepFailurePolicy={step_failure_policy}"), - format!("continueAfterStepFailure={continue_after_step_failure}"), - format!("nonzeroExitFailsStep={nonzero_exit_fails_step}"), - format!("exitsNonzeroAfterFinalAudit={exits_nonzero_after_final_audit}"), - ]; - if let Some(reason) = &args.reason { - service_args.push(format!("reason={reason}")); - } - let artifact_ref = args - .artifact_ref - .clone() - .unwrap_or_else(|| runbook_path.display().to_string()); - let receipt = service_lifecycle_receipt( - &args, - "cluster-windows-service-execution-runbook", - "written", - command, - service_args, - None, - None, - started_at, - Some(Utc::now()), - Some(artifact_ref.clone()), - ); - let written = write_epiphany_cultmesh_daemon_service_lifecycle_receipt( - &args.store, - args.runtime_id.clone(), - receipt, - )?; - println!( - "{}", - serde_json::to_string_pretty(&json!({ - "status": "written", - "store": args.store, - "runtimeId": args.runtime_id, - "serviceId": written.service_id, - "receiptId": written.receipt_id, - "runbookPath": runbook_path, - "runbookTryModes": try_modes, - "finalAuditMode": final_audit_mode, - "finalAuditRunsInFinally": final_audit_runs_in_finally, - "stepFailurePolicy": step_failure_policy, - "continueAfterStepFailure": continue_after_step_failure, - "nonzeroExitFailsStep": nonzero_exit_fails_step, - "exitsNonzeroAfterFinalAudit": exits_nonzero_after_final_audit, - "privateStateExposed": written.private_state_exposed, - }))? - ); - Ok(()) -} - -fn cluster_windows_service_execution_audit(args: Args) -> Result<()> { - require_supervisor_bootstrap(&args)?; - - let context = query_epiphany_local_verse_context(&args.store, args.runtime_id.clone())?; - assert_swarm_brake_allows_service_lifecycle(&context)?; - let started_at = Utc::now(); - let receipts = load_epiphany_cultmesh_daemon_service_lifecycle_receipts( - &args.store, - args.runtime_id.clone(), - )? - .into_iter() - .filter(|receipt| receipt.service_id == args.service_id) - .collect::>(); - let report = epiphany_cluster_service_execution_audit_report(&receipts); - let runbook_witness = - service_execution_runbook_witness(&report, "cluster-windows-service-execution-runbook"); - let receipt = service_lifecycle_receipt( - &args, - "cluster-windows-service-execution-audit", - &report.status, - "epiphany-daemon-supervisor".to_string(), - vec!["cluster-windows-service-execution-audit".to_string()], - None, - None, - started_at, - Some(Utc::now()), - Some(format!( - "service-manager://windows/{}/cluster-execution-audit", - sanitize_id(&args.service_id) - )), - ); - let written = write_epiphany_cultmesh_daemon_service_lifecycle_receipt( - &args.store, - args.runtime_id.clone(), - receipt, - )?; - println!( - "{}", - serde_json::to_string_pretty(&json!({ - "status": report.status, - "store": args.store, - "runtimeId": args.runtime_id, - "serviceId": written.service_id, - "receiptId": written.receipt_id, - "receiptCount": report.receipt_count, - "missingCount": report.missing_count, - "failedCount": report.failed_count, - "runbookPath": runbook_witness.runbook_path, - "runbookSha256": runbook_witness.runbook_sha256, - "elevatedCommand": runbook_witness.elevated_command, - "requiresElevatedAuthority": runbook_witness.requires_elevated_authority, - "aftercareMode": "cluster-service-execution-audit", - "aftercareCommand": "tools/epiphany_local_run.ps1 -Mode cluster-service-execution-audit", - "checks": report.checks, - "privateStateExposed": written.private_state_exposed || report.private_state_exposed, - }))? - ); - Ok(()) -} - -fn cluster_windows_service_execution_audit_smoke(args: Args) -> Result<()> { - seed_supervisor_smoke_fixture(&args)?; - - let started_at = Utc::now(); - for (action, status) in [ - ("cluster-windows-service-execution-runbook", "written"), - ( - "cluster-windows-service-execution-readiness", - "elevated-ready", - ), - ( - "cluster-windows-service-install", - "install-command-succeeded", - ), - ("cluster-windows-service-start", "start-requested"), - ("cluster-windows-service-audit", "complete"), - ("cluster-windows-service-stop", "stop-requested"), - ] { - let receipt = service_lifecycle_receipt( - &args, - action, - status, - "smoke".to_string(), - vec![action.to_string()], - None, - Some(0), - started_at, - Some(Utc::now()), - Some(format!("smoke://cluster-service-execution-audit/{action}")), - ); - write_epiphany_cultmesh_daemon_service_lifecycle_receipt( - &args.store, - args.runtime_id.clone(), - receipt, - )?; - } - let receipts = load_epiphany_cultmesh_daemon_service_lifecycle_receipts( - &args.store, - args.runtime_id.clone(), - )? - .into_iter() - .filter(|receipt| receipt.service_id == args.service_id) - .collect::>(); - let complete_report = epiphany_cluster_service_execution_audit_report(&receipts); - if complete_report.status != "complete" - || complete_report.missing_count != 0 - || complete_report.failed_count != 0 - || complete_report.private_state_exposed - { - anyhow::bail!( - "cluster service execution audit smoke failed to accept complete sealed receipts" - ); - } - let incomplete = service_lifecycle_receipt( - &args, - "cluster-windows-service-audit", - "incomplete", - "smoke".to_string(), - vec!["cluster-windows-service-audit".to_string()], - None, - Some(0), - started_at, - Some(Utc::now()), - Some("smoke://cluster-service-execution-audit/incomplete".to_string()), - ); - write_epiphany_cultmesh_daemon_service_lifecycle_receipt( - &args.store, - args.runtime_id.clone(), - incomplete, - )?; - let receipts = load_epiphany_cultmesh_daemon_service_lifecycle_receipts( - &args.store, - args.runtime_id.clone(), - )? - .into_iter() - .filter(|receipt| receipt.service_id == args.service_id) - .collect::>(); - let incomplete_report = epiphany_cluster_service_execution_audit_report(&receipts); - if incomplete_report.status != "incomplete" || incomplete_report.failed_count == 0 { - anyhow::bail!( - "cluster service execution audit smoke failed to reject incomplete audit receipt" - ); - } - println!( - "{}", - serde_json::to_string_pretty(&json!({ - "status": "ok", - "store": args.store, - "runtimeId": args.runtime_id, - "serviceId": args.service_id, - "completeStatus": complete_report.status, - "incompleteStatus": incomplete_report.status, - "incompleteFailedCount": incomplete_report.failed_count, - "privateStateExposed": false, - }))? - ); - Ok(()) -} - -struct ServiceExecutionRunbookWitness { - runbook_path: String, - runbook_sha256: String, - elevated_command: String, - requires_elevated_authority: bool, -} - -fn service_execution_runbook_witness( - report: &EpiphanyServiceExecutionAuditReport, - runbook_action: &str, -) -> ServiceExecutionRunbookWitness { - let runbook_path = report - .checks - .iter() - .find(|check| check.action == runbook_action) - .and_then(|check| check.operator_artifact_ref.clone()) - .unwrap_or_else(|| "none".to_string()); - let runbook_sha256 = local_file_sha256(&runbook_path).unwrap_or_else(|| "none".to_string()); - let elevated_command = if runbook_sha256 == "none" { - "none".to_string() - } else { - elevated_powershell_runbook_command(&runbook_path) - }; - ServiceExecutionRunbookWitness { - runbook_path, - runbook_sha256, - elevated_command, - requires_elevated_authority: true, - } -} - fn local_file_sha256(path: &str) -> Option { if path.trim().is_empty() || path == "none" { return None; @@ -2348,294 +1575,6 @@ fn local_file_sha256(path: &str) -> Option { let digest = Sha256::digest(&bytes); Some(format!("{digest:x}")) } - -fn elevated_powershell_runbook_command(path: &str) -> String { - let escaped = path.replace('\'', "''"); - format!( - "Start-Process PowerShell -Verb RunAs -Wait -ArgumentList @('-NoProfile','-ExecutionPolicy','Bypass','-File','{escaped}')" - ) -} - -fn windows_service_execution_runbook(args: Args) -> Result<()> { - require_supervisor_bootstrap(&args)?; - - let context = query_epiphany_local_verse_context(&args.store, args.runtime_id.clone())?; - assert_swarm_brake_allows_service_lifecycle(&context)?; - let started_at = Utc::now(); - let runbook_path = args - .runbook_path - .clone() - .context("service-execution-runbook requires --runbook-path")?; - if !runbook_path.exists() { - anyhow::bail!( - "service execution runbook artifact does not exist: {}", - runbook_path.display() - ); - } - let command = args - .service_command - .as_ref() - .map(|path| path.display().to_string()) - .unwrap_or_else(|| "tools/epiphany_local_run.ps1".to_string()); - let modes = [ - "service-execution-readiness", - "service-install-execute", - "service-start-execute", - "service-status", - "service-reconcile", - "service-stop-execute", - "service-status", - "service-execution-audit", - ]; - let final_audit_mode = "service-execution-audit"; - let step_failure_policy = "continue-after-step-failure"; - let continue_after_step_failure = true; - let nonzero_exit_fails_step = true; - let exits_nonzero_after_final_audit = true; - let final_audit_runs_in_finally = true; - let mut service_args = vec![ - format!("runbookPath={}", runbook_path.display()), - format!("modes={}", modes.join(",")), - format!("finally={final_audit_mode}"), - format!("stepFailurePolicy={step_failure_policy}"), - format!("continueAfterStepFailure={continue_after_step_failure}"), - format!("nonzeroExitFailsStep={nonzero_exit_fails_step}"), - format!("exitsNonzeroAfterFinalAudit={exits_nonzero_after_final_audit}"), - ]; - if let Some(reason) = &args.reason { - service_args.push(format!("reason={reason}")); - } - let artifact_ref = args - .artifact_ref - .clone() - .unwrap_or_else(|| runbook_path.display().to_string()); - let receipt = service_lifecycle_receipt( - &args, - "windows-service-execution-runbook", - "written", - command, - service_args, - None, - None, - started_at, - Some(Utc::now()), - Some(artifact_ref.clone()), - ); - let written = write_epiphany_cultmesh_daemon_service_lifecycle_receipt( - &args.store, - args.runtime_id.clone(), - receipt, - )?; - println!( - "{}", - serde_json::to_string_pretty(&json!({ - "status": "written", - "store": args.store, - "runtimeId": args.runtime_id, - "serviceId": written.service_id, - "serviceName": windows_service_name(&args), - "receiptId": written.receipt_id, - "runbookPath": runbook_path, - "runbookModes": modes, - "finalAuditMode": final_audit_mode, - "finalAuditRunsInFinally": final_audit_runs_in_finally, - "stepFailurePolicy": step_failure_policy, - "continueAfterStepFailure": continue_after_step_failure, - "nonzeroExitFailsStep": nonzero_exit_fails_step, - "exitsNonzeroAfterFinalAudit": exits_nonzero_after_final_audit, - "privateStateExposed": written.private_state_exposed, - }))? - ); - Ok(()) -} - -fn windows_service_reconcile(args: Args) -> Result<()> { - require_supervisor_bootstrap(&args)?; - - let context = query_epiphany_local_verse_context(&args.store, args.runtime_id.clone())?; - assert_swarm_brake_allows_service_lifecycle(&context)?; - let started_at = Utc::now(); - let command_path = service_command_path(&args)?; - let desired_args = service_serve_args(&args); - let desired_binary = windows_service_binary_path(&command_path, &desired_args); - let service_name = windows_service_name(&args); - let script = format!( - "$s = Get-CimInstance -ClassName Win32_Service -Filter {}; if ($null -eq $s) {{ 'missing'; exit 3 }} else {{ [pscustomobject]@{{Name=$s.Name; DisplayName=$s.DisplayName; State=$s.State; StartMode=$s.StartMode; PathName=$s.PathName; StartName=$s.StartName}} | ConvertTo-Json -Compress }}", - quote_powershell(&format!("Name='{}'", service_name.replace('\'', "''"))) - ); - let output = Command::new("powershell.exe") - .arg("-NoProfile") - .arg("-Command") - .arg(&script) - .output() - .with_context(|| format!("failed to reconcile Windows service {service_name}"))?; - let exit_code = output.status.code(); - let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); - let mut drift_reasons = Vec::new(); - let status = if exit_code == Some(3) || stdout.trim().eq_ignore_ascii_case("missing") { - "missing".to_string() - } else if !output.status.success() { - "query-failed".to_string() - } else { - let service = serde_json::from_str::(&stdout) - .with_context(|| format!("failed to parse Windows service query for {service_name}"))?; - let actual_start = service - .get("StartMode") - .and_then(Value::as_str) - .map(windows_service_start_mode_to_arg) - .unwrap_or_else(|| "unknown".to_string()); - if actual_start != args.service_start_type { - drift_reasons.push(format!( - "start-type expected {} but found {}", - args.service_start_type, actual_start - )); - } - let actual_binary = service - .get("PathName") - .and_then(Value::as_str) - .unwrap_or("") - .trim() - .to_string(); - if !windows_service_binary_matches(&actual_binary, &desired_binary) { - drift_reasons - .push("binary-path differs from desired supervisor serve command".to_string()); - } - if drift_reasons.is_empty() { - "in-sync".to_string() - } else { - "drift".to_string() - } - }; - let receipt = service_lifecycle_receipt( - &args, - "windows-service-reconcile", - &status, - "powershell.exe".to_string(), - vec!["-NoProfile".to_string(), "-Command".to_string(), script], - None, - exit_code, - started_at, - Some(Utc::now()), - Some(format!( - "service-manager://windows/{service_name}/reconcile" - )), - ); - let written = write_epiphany_cultmesh_daemon_service_lifecycle_receipt( - &args.store, - args.runtime_id.clone(), - receipt, - )?; - println!( - "{}", - serde_json::to_string_pretty(&json!({ - "status": written.status, - "store": args.store, - "runtimeId": args.runtime_id, - "serviceId": written.service_id, - "serviceName": service_name, - "receiptId": written.receipt_id, - "exitCode": written.exit_code, - "desiredStartType": args.service_start_type, - "desiredBinaryPath": desired_binary, - "driftReasons": drift_reasons, - "stdout": stdout, - "stderr": stderr, - "privateStateExposed": written.private_state_exposed, - }))? - ); - Ok(()) -} - -fn windows_service_control(args: Args, control: &str) -> Result<()> { - require_supervisor_bootstrap(&args)?; - - let context = query_epiphany_local_verse_context(&args.store, args.runtime_id.clone())?; - assert_swarm_brake_allows_service_lifecycle(&context)?; - let started_at = Utc::now(); - let service_name = windows_service_name(&args); - let (status, exit_code, stdout, stderr, completed_at, artifact_ref, executed) = if args - .execute_control - { - if windows_has_elevated_service_authority()? { - let output = Command::new("sc.exe") - .arg(control) - .arg(&service_name) - .output() - .with_context(|| format!("failed to run sc.exe {control} {service_name}"))?; - let status = if output.status.success() { - format!("{control}-requested") - } else { - format!("{control}-failed") - }; - ( - status, - output.status.code(), - String::from_utf8_lossy(&output.stdout).trim().to_string(), - String::from_utf8_lossy(&output.stderr).trim().to_string(), - Some(Utc::now()), - format!("service-manager://windows/{service_name}/{control}"), - true, - ) - } else { - ( - "execution-refused-not-elevated".to_string(), - None, - String::new(), - String::new(), - Some(Utc::now()), - format!("service-manager://windows/{service_name}/{control}-refused-not-elevated"), - false, - ) - } - } else { - ( - format!("{control}-planned"), - None, - String::new(), - String::new(), - Some(Utc::now()), - format!("service-manager://windows/{service_name}/{control}-plan"), - false, - ) - }; - let receipt = service_lifecycle_receipt( - &args, - &format!("windows-service-{control}"), - &status, - "sc.exe".to_string(), - vec![control.to_string(), service_name.clone()], - None, - exit_code, - started_at, - completed_at, - Some(artifact_ref), - ); - let written = write_epiphany_cultmesh_daemon_service_lifecycle_receipt( - &args.store, - args.runtime_id.clone(), - receipt, - )?; - println!( - "{}", - serde_json::to_string_pretty(&json!({ - "status": written.status, - "store": args.store, - "runtimeId": args.runtime_id, - "serviceId": written.service_id, - "serviceName": service_name, - "receiptId": written.receipt_id, - "exitCode": written.exit_code, - "stdout": stdout, - "stderr": stderr, - "executeRequested": args.execute_control, - "executed": executed, - "privateStateExposed": written.private_state_exposed, - }))? - ); - Ok(()) -} - fn run_tick(args: &Args, iteration: u64, next_wake_utc: Option) -> Result { let tick_started = Utc::now(); @@ -3288,38 +2227,6 @@ fn cluster_daemon_runbook_rows( Ok(rows) } -fn windows_service_install_script_path(args: &Args) -> PathBuf { - if let Some(path) = args.service_install_script_path.as_ref() { - return path.clone(); - } - PathBuf::from(".epiphany-run") - .join("daemon-services") - .join(format!("{}-install.ps1", sanitize_id(&args.service_id))) -} - -fn windows_service_name(args: &Args) -> String { - args.service_name - .clone() - .unwrap_or_else(|| sanitize_id(&args.service_id)) -} - -fn windows_service_query_status(exit_code: Option, stdout: &str, success: bool) -> String { - if success { - let lower = stdout.to_ascii_lowercase(); - if lower.contains("status=running") { - return "running".to_string(); - } - if lower.contains("status=stopped") { - return "stopped".to_string(); - } - return "present".to_string(); - } - if exit_code == Some(3) || stdout.trim().eq_ignore_ascii_case("missing") { - return "missing".to_string(); - } - "query-failed".to_string() -} - fn service_runbook_content( args: &Args, command_path: &std::path::Path, @@ -3397,152 +2304,6 @@ fn cluster_daemon_runbook_content( lines.join("\n") } -fn cluster_windows_service_install_script_content( - args: &Args, - command_path: &std::path::Path, - daemon_rows: &[ClusterDaemonRunbookRow], - service_specs: &[ClusterWindowsServiceSpec], -) -> String { - let start_type = windows_service_start_mode_to_arg(&args.service_start_type); - let mut lines = vec![ - "# Epiphany cluster daemon Windows service install plan".to_string(), - "# Generated by epiphany-daemon-supervisor cluster-service-install-plan.".to_string(), - "# Requires an elevated PowerShell session when executed.".to_string(), - "$ErrorActionPreference = 'Stop'".to_string(), - format!("$startType = {}", quote_powershell(&start_type)), - ]; - for (row, spec) in daemon_rows.iter().zip(service_specs.iter()) { - let binary = windows_service_binary_path(command_path, &row.args); - lines.push(String::new()); - lines.push(format!("# {} / {}", row.display_name, row.daemon_id)); - lines.push(format!( - "$serviceName = {}", - quote_powershell(&spec.service_name) - )); - lines.push(format!( - "$displayName = {}", - quote_powershell(&spec.display_name) - )); - lines.push(format!( - "$description = {}", - quote_powershell(&spec.description) - )); - lines.push(format!("$binaryPath = {}", quote_powershell(&binary))); - lines.push( - "if (Get-Service -Name $serviceName -ErrorAction SilentlyContinue) {".to_string(), - ); - lines.push(" throw \"Service '$serviceName' already exists. Remove it deliberately before reinstalling.\"".to_string()); - lines.push("}".to_string()); - lines.push("sc.exe create $serviceName binPath= $binaryPath start= $startType DisplayName= $displayName".to_string()); - lines.push("sc.exe description $serviceName $description".to_string()); - } - lines.push(String::new()); - lines.push(format!( - "\"planned cluster-daemon-service-count={} service={}\"", - service_specs.len(), - args.service_id - )); - lines.push(String::new()); - lines.join("\n") -} - -fn windows_service_install_script_content( - args: &Args, - service_name: &str, - command_path: &std::path::Path, - service_args: &[String], -) -> String { - let binary = windows_service_binary_path(command_path, service_args); - let display_name = args - .service_display_name - .clone() - .unwrap_or_else(|| service_name.to_string()); - let description = args.service_description.clone().unwrap_or_else(|| { - "Epiphany daemon supervisor serve loop. Scheduler decisions remain typed local Verse receipts." - .to_string() - }); - let start_type = args.service_start_type.clone(); - format!( - concat!( - "# Epiphany daemon supervisor Windows service install plan\n", - "# Generated by epiphany-daemon-supervisor windows-service-install.\n", - "# Requires an elevated PowerShell session when executed.\n", - "$ErrorActionPreference = 'Stop'\n", - "$serviceName = {service_name}\n", - "$displayName = {display_name}\n", - "$description = {description}\n", - "$binaryPath = {binary}\n", - "$startType = {start_type}\n", - "if (Get-Service -Name $serviceName -ErrorAction SilentlyContinue) {{\n", - " throw \"Service '$serviceName' already exists. Remove it deliberately before reinstalling.\"\n", - "}}\n", - "sc.exe create $serviceName binPath= $binaryPath start= $startType DisplayName= $displayName\n", - "sc.exe description $serviceName $description\n", - "\"install-command-succeeded service=$serviceName start=$startType\"\n" - ), - service_name = quote_powershell(service_name), - display_name = quote_powershell(&display_name), - description = quote_powershell(&description), - binary = quote_powershell(&binary), - start_type = quote_powershell(&start_type), - ) -} - -fn windows_service_binary_path(command_path: &std::path::Path, service_args: &[String]) -> String { - format!( - "{} {}", - quote_windows_bin_path_arg(&command_path.display().to_string()), - service_args - .iter() - .map(|arg| quote_windows_bin_path_arg(arg)) - .collect::>() - .join(" ") - ) -} - -fn windows_service_start_mode_to_arg(start_mode: &str) -> String { - match start_mode.to_ascii_lowercase().as_str() { - "auto" | "automatic" => "auto".to_string(), - "manual" | "demand" => "demand".to_string(), - "disabled" => "disabled".to_string(), - other => other.to_string(), - } -} - -fn windows_service_binary_matches(actual: &str, desired: &str) -> bool { - let normalize = |value: &str| { - value - .trim() - .trim_matches('"') - .replace("\\\\", "\\") - .to_ascii_lowercase() - }; - normalize(actual) == normalize(desired) -} - -fn windows_has_elevated_service_authority() -> Result { - let output = Command::new("powershell.exe") - .arg("-NoProfile") - .arg("-Command") - .arg("([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)") - .output() - .context("failed to check Windows service elevation")?; - if !output.status.success() { - return Ok(false); - } - Ok(String::from_utf8_lossy(&output.stdout) - .trim() - .eq_ignore_ascii_case("true")) -} - -fn quote_windows_bin_path_arg(value: &str) -> String { - if value.chars().any(|ch| ch.is_whitespace() || ch == '"') { - format!("\"{}\"", value.replace('"', "\\\"")) - } else { - value.to_string() - } -} - fn quote_powershell(value: &str) -> String { format!("'{}'", value.replace('\'', "''")) } @@ -3555,43 +2316,6 @@ struct ClusterDaemonRunbookRow { args: Vec, } -struct ClusterWindowsServiceSpec { - service_name: String, - display_name: String, - description: String, -} - -fn cluster_windows_service_spec( - args: &Args, - row: &ClusterDaemonRunbookRow, -) -> ClusterWindowsServiceSpec { - let service_prefix = args - .service_name - .clone() - .unwrap_or_else(|| args.service_id.clone()); - let service_name = format!( - "{}-{}", - sanitize_id(&service_prefix), - sanitize_id(&row.daemon_id) - ); - let display_prefix = args - .service_display_name - .clone() - .unwrap_or_else(|| "Epiphany Cluster Daemon".to_string()); - let display_name = format!("{display_prefix} - {}", row.display_name); - let description = args.service_description.clone().unwrap_or_else(|| { - format!( - "Epiphany cluster daemon body for {}; heartbeats remain typed local Verse status receipts.", - row.display_name - ) - }); - ClusterWindowsServiceSpec { - service_name, - display_name, - description, - } -} - fn service_lifecycle_receipt( args: &Args, action: &str, @@ -3610,11 +2334,11 @@ fn service_lifecycle_receipt( sanitize_id(&args.service_id), sanitize_id(action) ); - if action == "launch" { - format!("{base}-{}", started_at.timestamp_millis()) - } else { - base - } + format!( + "{base}-{}-{}", + started_at.timestamp_millis(), + Uuid::new_v4() + ) }); EpiphanyCultMeshDaemonServiceLifecycleReceiptEntry { schema_version: EPIPHANY_CULTMESH_DAEMON_SERVICE_LIFECYCLE_RECEIPT_SCHEMA_VERSION @@ -3946,6 +2670,97 @@ mod semantic_projector_authority_tests { assert!(!legacy_cli.contains(&index_arm)); assert!(!legacy_cli.contains(&claim_flag)); } + + #[test] + fn windows_scm_commands_are_hard_refused_at_dispatch() { + let source = include_str!("epiphany-daemon-supervisor.rs"); + for command in [ + "service-install-execute", + "service-reconcile", + "service-start", + "service-stop", + "service-execution-audit", + "cluster-service-install-execute", + "cluster-service-start", + "cluster-service-stop", + "cluster-service-execution-audit", + ] { + let dispatch = source + .find(&format!("\"{command}\"")) + .unwrap_or_else(|| panic!("missing legacy command {command}")); + let arm_tail = &source[dispatch + ..source[dispatch..] + .find('\n') + .map(|n| dispatch + n + 240) + .unwrap_or(source.len()) + .min(source.len())]; + assert!( + arm_tail.contains("refuse_false_windows_scm_authority"), + "{command} must terminate at the refusal gate" + ); + } + assert!(source.contains( + "this executable is a foreground console process, not a Windows service host" + )); + } + + #[test] + fn managed_service_task_definition_owns_foreground_idunn_lifecycle() { + let source = include_str!("epiphany-daemon-supervisor.rs"); + let start = source.find("fn managed_service_task_action").unwrap(); + let tail = &source[start..]; + let end = tail.find("\nfn managed_service_task_status").unwrap(); + let body = &tail[..end]; + for required in [ + "managed-service-serve", + "New-ScheduledTaskTrigger -AtLogOn", + "LogonType Interactive", + "RunLevel Limited", + "MultipleInstances IgnoreNew", + "RestartCount", + "RestartInterval", + "ExecutionTimeLimit ([TimeSpan]::Zero)", + "StartWhenAvailable", + "AllowStartIfOnBatteries", + "DontStopIfGoingOnBatteries", + "-WorkingDirectory", + ] { + assert!( + body.contains(required), + "task definition missing {required}" + ); + } + assert!(!body.contains("sc.exe")); + assert!(source.contains("let mut task_restart_count = 999_u32")); + assert!(source.contains("task_readback_has_drift")); + assert!(source.contains("\"install-drift\"")); + assert!(source.contains("\"in-sync\".to_string()")); + assert!(source.contains("managed-service-scheduler.fatal.log")); + assert!(source.contains("Resolve-TaskSid")); + assert!(source.contains("WindowsIdentity]::GetCurrent().User.Value")); + } + + #[test] + fn windows_argv_quoting_preserves_paths_and_escapes_parser_boundaries() { + assert_eq!( + windows_quote_argv(r"C:\Epiphany\bin.exe"), + r"C:\Epiphany\bin.exe" + ); + assert_eq!( + windows_quote_argv(r"C:\Program Files\Epiphany"), + r#""C:\Program Files\Epiphany""# + ); + assert_eq!(windows_quote_argv(""), "\"\""); + assert_eq!(windows_quote_argv(r#"a"b"#), r#""a\"b""#); + assert_eq!( + windows_quote_argv("ends with slash\\"), + r#""ends with slash\\""# + ); + assert_eq!( + windows_command_line(&[r"C:\Mind\state.cc".to_string(), "two words".to_string()]), + r#"C:\Mind\state.cc "two words""# + ); + } } #[derive(Clone)] @@ -3968,16 +2783,10 @@ struct Args { scheduler_id: String, service_id: String, service_name: Option, - service_display_name: Option, - service_description: Option, - service_start_type: String, restart_mode: String, service_command: Option, service_args: Vec, runbook_path: Option, - service_install_script_path: Option, - execute_install: bool, - execute_control: bool, loop_interval_seconds: i64, max_iterations: u64, wait_child: bool, @@ -3994,6 +2803,10 @@ struct Args { qdrant_url: Option, ollama_base_url: Option, ollama_model: String, + fatal_log: Option, + task_logon_delay_seconds: u64, + task_restart_interval_seconds: u64, + task_restart_count: u32, } impl Args { @@ -4018,16 +2831,10 @@ impl Args { let mut scheduler_id = "epiphany-daemon-supervisor".to_string(); let mut service_id = "epiphany-daemon-supervisor-service".to_string(); let mut service_name = None; - let mut service_display_name = None; - let mut service_description = None; - let mut service_start_type = "demand".to_string(); let mut restart_mode = "on-failure".to_string(); let mut service_command = None; let mut service_args = Vec::new(); let mut runbook_path = None; - let mut service_install_script_path = None; - let mut execute_install = false; - let mut execute_control = false; let mut loop_interval_seconds = 60_i64; let mut max_iterations = 0_u64; let mut wait_child = false; @@ -4044,6 +2851,10 @@ impl Args { let mut qdrant_url = None; let mut ollama_base_url = None; let mut ollama_model = "qwen3-embedding:0.6b".to_string(); + let mut task_logon_delay_seconds = 30_u64; + let mut task_restart_interval_seconds = 60_u64; + let mut task_restart_count = 999_u32; + let mut fatal_log = None; while let Some(arg) = values.next() { match arg.as_str() { @@ -4109,25 +2920,6 @@ impl Args { "--service-name" => { service_name = Some(values.next().context("missing --service-name value")?) } - "--service-display-name" => { - service_display_name = Some( - values - .next() - .context("missing --service-display-name value")?, - ) - } - "--service-description" => { - service_description = Some( - values - .next() - .context("missing --service-description value")?, - ) - } - "--service-start-type" => { - service_start_type = values - .next() - .context("missing --service-start-type value")?; - } "--restart-mode" => { restart_mode = values.next().context("missing --restart-mode value")?; } @@ -4144,15 +2936,6 @@ impl Args { values.next().context("missing --runbook-path value")?, )); } - "--service-install-script-path" => { - service_install_script_path = Some(PathBuf::from( - values - .next() - .context("missing --service-install-script-path value")?, - )); - } - "--execute-install" => execute_install = true, - "--execute-control" | "--execute-service-control" => execute_control = true, "--loop-interval-seconds" | "--serve-interval-seconds" => { loop_interval_seconds = values .next() @@ -4217,6 +3000,29 @@ impl Args { "--ollama-model" => { ollama_model = values.next().context("missing --ollama-model value")? } + "--task-logon-delay-seconds" => { + task_logon_delay_seconds = values + .next() + .context("missing --task-logon-delay-seconds value")? + .parse()?; + } + "--task-restart-interval-seconds" => { + task_restart_interval_seconds = values + .next() + .context("missing --task-restart-interval-seconds value")? + .parse()?; + } + "--task-restart-count" => { + task_restart_count = values + .next() + .context("missing --task-restart-count value")? + .parse()?; + } + "--fatal-log" => { + fatal_log = Some(PathBuf::from( + values.next().context("missing --fatal-log value")?, + )); + } other => anyhow::bail!("unknown argument {other:?}"), } } @@ -4240,6 +3046,12 @@ impl Args { | "daemon" | "managed-service-serve" | "service-desired-state-serve" + | "managed-service-task-plan" + | "managed-service-task-install" + | "managed-service-task-status" + | "managed-service-task-start" + | "managed-service-task-stop" + | "managed-service-task-uninstall" | "migrate-retired-operator-status" | "migrate-semantic-attempts-v0" | "semantic-projector-service-status" @@ -4312,9 +3124,6 @@ impl Args { if loop_interval_seconds < 0 { anyhow::bail!("--loop-interval-seconds must be non-negative"); } - if !matches!(service_start_type.as_str(), "auto" | "demand" | "disabled") { - anyhow::bail!("--service-start-type must be auto, demand, or disabled"); - } if !matches!(restart_mode.as_str(), "always" | "on-failure" | "never") { anyhow::bail!("--restart-mode must be always, on-failure, or never"); } @@ -4338,16 +3147,10 @@ impl Args { scheduler_id, service_id, service_name, - service_display_name, - service_description, - service_start_type, restart_mode, service_command, service_args, runbook_path, - service_install_script_path, - execute_install, - execute_control, loop_interval_seconds, max_iterations, wait_child, @@ -4364,6 +3167,10 @@ impl Args { qdrant_url, ollama_base_url, ollama_model, + fatal_log, + task_logon_delay_seconds, + task_restart_interval_seconds, + task_restart_count, }) } } diff --git a/epiphany-core/src/bin/epiphany-hands-action.rs b/epiphany-core/src/bin/epiphany-hands-action.rs index cef5d798..216674c6 100644 --- a/epiphany-core/src/bin/epiphany-hands-action.rs +++ b/epiphany-core/src/bin/epiphany-hands-action.rs @@ -586,6 +586,7 @@ mod tests { use epiphany_core::runtime_hands_commit_receipt; use epiphany_core::runtime_hands_patch_receipt; use epiphany_core::runtime_latest_hands_receipt_chain_after; + use sha2::Digest; #[test] fn records_patch_command_and_commit_receipts_against_approved_gate() -> Result<()> { @@ -789,6 +790,18 @@ mod tests { created_at: "2026-06-02T00:00:00Z".to_string(), }, )?; + let grant = epiphany_core::substrate_gate_coordinator_implementation_grant( + "substrate-grant-test".to_string(), + "hands-job-test".to_string(), + requested_paths.clone(), + "2026-06-02T00:00:00Z".to_string(), + ); + epiphany_core::put_substrate_gate_repo_access_grant_receipt(store, &grant)?; + let requested_action = if operations == ["patch"] { + "patch" + } else { + "continueImplementation" + }; let intent = HandsActionIntent { schema_version: HANDS_ACTION_INTENT_SCHEMA_VERSION.to_string(), intent_id: "hands-intent-test".to_string(), @@ -796,7 +809,7 @@ mod tests { binding_id: "implementation-worker".to_string(), role: "epiphany-hands".to_string(), authority_scope: "epiphany.role.implementation".to_string(), - requested_action: "continueImplementation".to_string(), + requested_action: requested_action.to_string(), requested_paths, substrate_gate_grant_receipt_id: "substrate-grant-test".to_string(), requested_at: "2026-06-02T00:00:01Z".to_string(), @@ -815,6 +828,100 @@ mod tests { "2026-06-02T00:00:02Z".to_string(), ); put_hands_action_review(store, &review)?; + seed_frontier_authority(store, &intent, &review)?; + Ok(()) + } + + fn seed_frontier_authority( + store: &PathBuf, + intent: &HandsActionIntent, + review: &HandsActionReview, + ) -> Result<()> { + let item = epiphany_core::RepoFrontierItem { + id: "hands-frontier-test".to_string(), + migration_body: "Exercise the bounded Hands receipt chain.".to_string(), + question: "Does the consequence preserve its authority chain?".to_string(), + gap: "An unrouted consequence is invalid.".to_string(), + target_claim_ids: vec!["hands-claim-test".to_string()], + source_scope: intent.requested_paths.clone(), + recommended_next_organ: "Hands".to_string(), + evidence_refs: vec!["hands-fixture".to_string()], + status: epiphany_core::RepoFrontierStatus::Active, + ..Default::default() + }; + let item_hash = format!( + "{:x}", + sha2::Sha256::digest(rmp_serde::to_vec_named(&item)?) + ); + let mut model = epiphany_core::EpiphanyMemoryGraphSnapshot { + schema_version: Some(epiphany_core::MEMORY_GRAPH_SCHEMA_VERSION.to_string()), + graph_id: "hands-action-test-model".to_string(), + model_revision: 1, + domains: vec![epiphany_core::EpiphanyMemoryDomain { + id: "repo".to_string(), + profile: epiphany_core::EpiphanyMemoryProfile::RepoArchitecture, + title: "Repository".to_string(), + lifecycle: epiphany_core::EpiphanyMemoryLifecycle::Accepted, + ..Default::default() + }], + nodes: vec![epiphany_core::EpiphanyMemoryNode { + id: "hands-claim-test".to_string(), + domain_id: "repo".to_string(), + profile: epiphany_core::EpiphanyMemoryProfile::RepoArchitecture, + kind: epiphany_core::EpiphanyMemoryNodeKind::RuntimeContract, + title: "Hands authority".to_string(), + claim: "Hands consequences require an exact frontier authority.".to_string(), + question: "Is the authority chain exact?".to_string(), + action_implication: "Bind every receipt to the routed scope.".to_string(), + source_hashes: vec!["anchor:missing".to_string()], + lifecycle: epiphany_core::EpiphanyMemoryLifecycle::Accepted, + ..Default::default() + }], + frontier: vec![item.clone()], + ..Default::default() + }; + model.model_hash = epiphany_core::memory_graph_model_hash(&model)?; + let entry = epiphany_core::EpiphanyMemoryGraphEntry::from_snapshot(&model)?; + let route = epiphany_core::RepoFrontierRoute { + schema_version: epiphany_core::REPO_FRONTIER_ROUTE_SCHEMA_VERSION.to_string(), + route_id: "hands-route-test".to_string(), + next_organ: epiphany_core::RepoFrontierNextOrgan::Hands, + model_revision: model.model_revision, + model_hash: model.model_hash.clone(), + admission_receipt_id: "hands-admission-test".to_string(), + frontier_item_id: item.id.clone(), + frontier_item_hash: item_hash.clone(), + migration_body: item.migration_body.clone(), + question: item.question.clone(), + gap: item.gap.clone(), + target_claim_ids: item.target_claim_ids.clone(), + source_scope: item.source_scope.clone(), + adopted_plan: None, + selected_at: "2026-06-02T00:00:02Z".to_string(), + contract: epiphany_core::REPO_FRONTIER_ROUTE_CONTRACT.to_string(), + }; + let mut cache = epiphany_core::runtime_spine_cache(store)?; + cache.put(epiphany_core::MEMORY_GRAPH_KEY, &entry)?; + cache.put(&route.route_id, &route)?; + epiphany_core::put_repo_frontier_hands_authority( + store, + &epiphany_core::RepoFrontierHandsAuthority { + schema_version: epiphany_core::REPO_FRONTIER_HANDS_AUTHORITY_SCHEMA_VERSION + .to_string(), + authority_id: "hands-authority-test".to_string(), + route_id: route.route_id, + model_revision: route.model_revision, + model_hash: route.model_hash, + frontier_item_id: route.frontier_item_id, + frontier_item_hash: item_hash, + hands_intent_id: intent.intent_id.clone(), + hands_review_id: review.review_id.clone(), + substrate_grant_receipt_id: intent.substrate_gate_grant_receipt_id.clone(), + requested_paths: intent.requested_paths.clone(), + granted_at: "2026-06-02T00:00:03Z".to_string(), + contract: epiphany_core::REPO_FRONTIER_HANDS_AUTHORITY_CONTRACT.to_string(), + }, + )?; Ok(()) } diff --git a/epiphany-core/src/bin/epiphany-mvp-coordinator.rs b/epiphany-core/src/bin/epiphany-mvp-coordinator.rs index b985217b..44bafd9f 100644 --- a/epiphany-core/src/bin/epiphany-mvp-coordinator.rs +++ b/epiphany-core/src/bin/epiphany-mvp-coordinator.rs @@ -1901,6 +1901,7 @@ mod tests { created_at: "2026-06-02T00:00:00Z".to_string(), }, )?; + seed_hands_frontier(&store)?; let status = json!({ "scene": { "scene": { @@ -1945,4 +1946,74 @@ mod tests { ); Ok(()) } + + fn seed_hands_frontier(store: &Path) -> Result<()> { + let item = epiphany_core::RepoFrontierItem { + id: "coordinator-hands-frontier-test".to_string(), + migration_body: "Exercise the coordinator-owned Hands gate.".to_string(), + question: "Does Self route the admitted implementation frontier?".to_string(), + gap: "Hands cannot act without a routed Modeling frontier.".to_string(), + target_claim_ids: vec!["coordinator-hands-claim-test".to_string()], + source_scope: vec!["epiphany-core/src".to_string()], + recommended_next_organ: "Hands".to_string(), + evidence_refs: vec!["fixture:mvp-coordinator".to_string()], + status: epiphany_core::RepoFrontierStatus::Active, + ..Default::default() + }; + let mut model = epiphany_core::EpiphanyMemoryGraphSnapshot { + schema_version: Some(epiphany_core::MEMORY_GRAPH_SCHEMA_VERSION.to_string()), + graph_id: "mvp-coordinator-test-model".to_string(), + model_revision: 1, + domains: vec![epiphany_core::EpiphanyMemoryDomain { + id: "repo".to_string(), + profile: epiphany_core::EpiphanyMemoryProfile::RepoArchitecture, + title: "Repository".to_string(), + lifecycle: epiphany_core::EpiphanyMemoryLifecycle::Accepted, + ..Default::default() + }], + nodes: vec![epiphany_core::EpiphanyMemoryNode { + id: "coordinator-hands-claim-test".to_string(), + domain_id: "repo".to_string(), + profile: epiphany_core::EpiphanyMemoryProfile::RepoArchitecture, + kind: epiphany_core::EpiphanyMemoryNodeKind::RuntimeContract, + title: "Coordinator Hands gate".to_string(), + claim: "Implementation begins from an admitted routed frontier.".to_string(), + question: "Is the route current?".to_string(), + action_implication: "Route the exact admitted source scope.".to_string(), + source_hashes: vec!["anchor:missing".to_string()], + lifecycle: epiphany_core::EpiphanyMemoryLifecycle::Accepted, + ..Default::default() + }], + frontier: vec![item], + ..Default::default() + }; + model.model_hash = epiphany_core::memory_graph_model_hash(&model)?; + let entry = epiphany_core::EpiphanyMemoryGraphEntry::from_snapshot(&model)?; + let admission = epiphany_core::RepoModelAdmissionReceipt { + schema_version: epiphany_core::REPO_MODEL_ADMISSION_RECEIPT_SCHEMA_VERSION.to_string(), + receipt_id: "coordinator-hands-admission-test".to_string(), + review_id: "coordinator-hands-model-review-test".to_string(), + result_id: "coordinator-hands-model-result-test".to_string(), + patch_id: "coordinator-hands-model-patch-test".to_string(), + patch_sha256: "fixture".to_string(), + previous_revision: 0, + previous_hash: String::new(), + admitted_revision: model.model_revision, + admitted_hash: model.model_hash.clone(), + admitted_at: "2026-06-02T00:00:00Z".to_string(), + contract: epiphany_core::REPO_MODEL_ADMISSION_CONTRACT.to_string(), + purpose: epiphany_core::RepoModelPatchPurpose::Evolution, + frontier_route_id: String::new(), + verification_request_id: String::new(), + soul_verdict_receipt_id: String::new(), + frontier_modeling_request_id: String::new(), + proposal_modeling_request_id: String::new(), + claim_repair_request_id: String::new(), + frontier_plan_decision_id: String::new(), + }; + let mut cache = epiphany_core::runtime_spine_cache(store)?; + cache.put(epiphany_core::MEMORY_GRAPH_KEY, &entry)?; + cache.put(&admission.receipt_id, &admission)?; + Ok(()) + } } diff --git a/epiphany-core/src/bin/epiphany-verse-query.rs b/epiphany-core/src/bin/epiphany-verse-query.rs index ce52319e..bc945f25 100644 --- a/epiphany-core/src/bin/epiphany-verse-query.rs +++ b/epiphany-core/src/bin/epiphany-verse-query.rs @@ -131,33 +131,29 @@ const DIRECT_IDUNN_DEPLOYMENT_AFTERCARE_AUDIT_COMMAND: &str = "epiphany-work dep const WRAPPER_SERVICE_TICK_COMMAND: &str = "tools/epiphany_local_run.ps1 -Mode service-tick"; const WRAPPER_SERVICE_POLICY_DIRECTORY_COMMAND: &str = "tools/epiphany_local_run.ps1 -Mode service-policy-directory"; -const WRAPPER_SERVICE_EXECUTION_AUDIT_COMMAND: &str = - "tools/epiphany_local_run.ps1 -Mode service-execution-audit"; -const WRAPPER_SERVICE_EXECUTION_RUNBOOK_COMMAND: &str = - "tools/epiphany_local_run.ps1 -Mode service-execution-runbook"; -const WRAPPER_SERVICE_EXECUTION_READINESS_COMMAND: &str = - "tools/epiphany_local_run.ps1 -Mode service-execution-readiness"; +const WRAPPER_MANAGED_SERVICE_TASK_PLAN_COMMAND: &str = + "tools/epiphany_local_run.ps1 -Mode managed-service-task-plan"; const WRAPPER_SERVICE_INSTALL_EXECUTE_COMMAND: &str = - "tools/epiphany_local_run.ps1 -Mode service-install-execute"; -const WRAPPER_SERVICE_STATUS_COMMAND: &str = "tools/epiphany_local_run.ps1 -Mode service-status"; -const WRAPPER_SERVICE_RECONCILE_COMMAND: &str = - "tools/epiphany_local_run.ps1 -Mode service-reconcile"; + "tools/epiphany_local_run.ps1 -Mode managed-service-task-install"; +const WRAPPER_SERVICE_STATUS_COMMAND: &str = + "tools/epiphany_local_run.ps1 -Mode managed-service-task-status"; const WRAPPER_SERVICE_START_EXECUTE_COMMAND: &str = - "tools/epiphany_local_run.ps1 -Mode service-start-execute"; + "tools/epiphany_local_run.ps1 -Mode managed-service-task-start"; const WRAPPER_SERVICE_STOP_EXECUTE_COMMAND: &str = - "tools/epiphany_local_run.ps1 -Mode service-stop-execute"; -const WRAPPER_CLUSTER_SERVICE_EXECUTION_AUDIT_COMMAND: &str = - "tools/epiphany_local_run.ps1 -Mode cluster-service-execution-audit"; -const WRAPPER_CLUSTER_SERVICE_EXECUTION_RUNBOOK_COMMAND: &str = - "tools/epiphany_local_run.ps1 -Mode cluster-service-execution-runbook"; -const WRAPPER_CLUSTER_SERVICE_EXECUTION_READINESS_COMMAND: &str = - "tools/epiphany_local_run.ps1 -Mode cluster-service-execution-readiness"; -const WRAPPER_CLUSTER_SERVICE_INSTALL_EXECUTE_COMMAND: &str = - "tools/epiphany_local_run.ps1 -Mode cluster-service-install-execute"; -const WRAPPER_CLUSTER_SERVICE_START_EXECUTE_COMMAND: &str = - "tools/epiphany_local_run.ps1 -Mode cluster-service-start-execute"; -const WRAPPER_CLUSTER_SERVICE_STOP_EXECUTE_COMMAND: &str = - "tools/epiphany_local_run.ps1 -Mode cluster-service-stop-execute"; + "tools/epiphany_local_run.ps1 -Mode managed-service-task-stop"; +// Historical lifecycle receipts still use these typed families. Their follow-up +// routes are deliberately demoted: cluster SCM has no replacement owner, while +// single-service SCM receipts lead to the Idunn scheduled-task plan. +const WRAPPER_SERVICE_EXECUTION_AUDIT_COMMAND: &str = WRAPPER_MANAGED_SERVICE_TASK_PLAN_COMMAND; +const WRAPPER_SERVICE_EXECUTION_RUNBOOK_COMMAND: &str = WRAPPER_MANAGED_SERVICE_TASK_PLAN_COMMAND; +const WRAPPER_SERVICE_EXECUTION_READINESS_COMMAND: &str = WRAPPER_MANAGED_SERVICE_TASK_PLAN_COMMAND; +const WRAPPER_SERVICE_RECONCILE_COMMAND: &str = WRAPPER_SERVICE_STATUS_COMMAND; +const WRAPPER_CLUSTER_SERVICE_EXECUTION_AUDIT_COMMAND: &str = WRAPPER_OVERVIEW_COMMAND; +const WRAPPER_CLUSTER_SERVICE_EXECUTION_RUNBOOK_COMMAND: &str = WRAPPER_OVERVIEW_COMMAND; +const WRAPPER_CLUSTER_SERVICE_EXECUTION_READINESS_COMMAND: &str = WRAPPER_OVERVIEW_COMMAND; +const WRAPPER_CLUSTER_SERVICE_INSTALL_EXECUTE_COMMAND: &str = WRAPPER_OVERVIEW_COMMAND; +const WRAPPER_CLUSTER_SERVICE_START_EXECUTE_COMMAND: &str = WRAPPER_OVERVIEW_COMMAND; +const WRAPPER_CLUSTER_SERVICE_STOP_EXECUTE_COMMAND: &str = WRAPPER_OVERVIEW_COMMAND; const DIRECT_BIFROST_PUBLICATION_COMMAND: &str = "epiphany-verse-query bifrost-publication --target-repository --changed-path "; const SERVICE_LIFECYCLE_OWNER: &str = "Idunn"; @@ -7494,24 +7490,19 @@ fn service_execution_audit_check_tui_row(check: &EpiphanyServiceExecutionAuditCh fn service_execution_check_follow_up_command(action: &str) -> &'static str { match action { - "cluster-windows-service-execution-runbook" => { - WRAPPER_CLUSTER_SERVICE_EXECUTION_RUNBOOK_COMMAND + "cluster-windows-service-execution-runbook" + | "cluster-windows-service-execution-readiness" + | "cluster-windows-service-install" + | "cluster-windows-service-start" + | "cluster-windows-service-execution-audit" + | "cluster-windows-service-stop" => "none", + "windows-service-execution-runbook" | "windows-service-execution-readiness" => { + WRAPPER_MANAGED_SERVICE_TASK_PLAN_COMMAND } - "cluster-windows-service-execution-readiness" => { - WRAPPER_CLUSTER_SERVICE_EXECUTION_READINESS_COMMAND - } - "cluster-windows-service-install" => WRAPPER_CLUSTER_SERVICE_INSTALL_EXECUTE_COMMAND, - "cluster-windows-service-start" => WRAPPER_CLUSTER_SERVICE_START_EXECUTE_COMMAND, - "cluster-windows-service-execution-audit" => { - WRAPPER_CLUSTER_SERVICE_EXECUTION_AUDIT_COMMAND - } - "cluster-windows-service-stop" => WRAPPER_CLUSTER_SERVICE_STOP_EXECUTE_COMMAND, - "windows-service-execution-runbook" => WRAPPER_SERVICE_EXECUTION_RUNBOOK_COMMAND, - "windows-service-execution-readiness" => WRAPPER_SERVICE_EXECUTION_READINESS_COMMAND, "windows-service-install" => WRAPPER_SERVICE_INSTALL_EXECUTE_COMMAND, "windows-service-start" => WRAPPER_SERVICE_START_EXECUTE_COMMAND, "windows-service-status" => WRAPPER_SERVICE_STATUS_COMMAND, - "windows-service-reconcile" => WRAPPER_SERVICE_RECONCILE_COMMAND, + "windows-service-reconcile" => WRAPPER_SERVICE_STATUS_COMMAND, "windows-service-stop" => WRAPPER_SERVICE_STOP_EXECUTE_COMMAND, _ => "none", } diff --git a/notes/fresh-workspace-handoff.md b/notes/fresh-workspace-handoff.md index 95a2e7ef..9e815a69 100644 --- a/notes/fresh-workspace-handoff.md +++ b/notes/fresh-workspace-handoff.md @@ -1376,3 +1376,42 @@ not implement the Windows service-control handshake. Build a real service host or explicit Task Scheduler owner with restart policy, give the Qdrant tunnel the same durable treatment, and prove startup/reboot recovery. Live abandoned claim recovery remains unproven; use a safe fixture rather than forging work. + +## Windows survival authority rebuild (2026-07-15) + +The workstation survival cut rejected the generic Windows SCM path. The +Epiphany binaries are ordinary console programs and never implemented the +Windows service-control dispatcher, so `sc.exe create`, elevated service +runbooks/audits, cluster-service install/control, and `swarm-online-runbook` +were presentation of authority the body did not possess. Those routes and +their old receipts are historical evidence only; they must not steer current +deployment, readiness, or follow-up work. + +Host survival is now deliberately narrower and honest. Current-user tasks +`Epiphany-Idunn-Managed-Service-Reconciler` and +`GameCult-Yggdrasil-Tunnel` run as Meta with `InteractiveToken`/Limited +privilege, `AtLogOn` plus a one-minute recurring recovery trigger, +restart-on-failure at one-minute intervals for 999 attempts, `IgnoreNew`, zero +execution limit, and exact direct foreground actions. Task Scheduler owns +process presence after login. It does not own projector children or semantic +state. Idunn still owns the managed-service reconciliation loop and projector +launch/restart correlation. `gamecult-ops` owns the Yggdrasil tunnel profile +and foreground SSH process; Epiphany merely consumes the pinned local Qdrant +ports. No wrapper, detached grandchild, or service-manager receipt is a second +survival owner. + +Live recurrence proof killed reconciler PID 24152 and observed exactly one +replacement PID 25428. Killing projector PID 29868 then caused Idunn to launch +PID 19888 with launch receipt `4d406060-6582-473a-9322-9f4ccc8b322f`; +ready heartbeat `79fe3143-28a8-40fe-9547-0287159082e6` names provider +incarnation `projector-59217a66-e3e2-44c3-b000-36af3d43b043` and the exact +startup correlation. Tunnel recurrence killed SSH PID 22652 and observed PID +30444 start at 11:37:44 with Qdrant REST healthy. + +The limitation is part of the contract: this is after-login recovery, not +pre-login, boot-time, machine-wide, or Windows-service operation. The next +Soul proof is a real reboot/logon cycle showing both scheduled tasks running, +the tunnel ports restored, one Idunn-managed projector child, and a fresh +launch-correlated ready heartbeat. After that, use an isolated exact +abandoned-claim fixture to prove recovery/resumption without forging canonical +Mind or Modeling work. Do not spend another pass polishing the dead SCM rite. diff --git a/state/ledgers.msgpack b/state/ledgers.msgpack index bb9a5bd2c8aa1aa4f9b6593002ee4fab30812328..bc19350e6449457ee149cbd1977d0c614acab600 100644 GIT binary patch delta 587 zcmXYuKZ_JG7>C{U^g1goY>pRfFUw3OGn-lAKm*&*s(X~9AF*alMvmciZKR)<; zYjiOheR;DPzuk;SDckvS%0bGNl&fDi|UUSa(G%cc${niQ95P&r-tI{A<$p=cKxecO>db%7 diff --git a/state/map.yaml b/state/map.yaml index b755974e..79d4acff 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -72,7 +72,7 @@ data_flow: active_subgoals: - Use `notes/conceptual-substitution-audit-2026-07-12.md` plus `notes/receipt-writer-provenance-audit-2026-07-12.md` as the current authority-corrosion maps. Snapshot promotion, synthetic smoke escape, unbound operator completion, hidden service execution flags, provider/bootstrap collapse, central/all-provider publication, production-side implicit bootstrap, orphaned external response authority, bulk readiness authority, public topology-derived provider builders, bootstrap/template-authored operator observation, writerless operator-status schema authority, SoA projection body creation, path-selection-as-body-creation, executable-name-as-provider-participation, and orphaned daemon-tool response authority are cut. Provider responses and readiness without observers are test-only; the ownerless operator-status family and counterfeit Eve provider CLI are deleted; templates are private; query mutations require the body. CultCache's backing store owns missing-read purity by returning before lock acquisition. Next audit remaining projections, caches, schedulers, adapters, and non-receipt smokes for borrowed authority or path escape. - `swarm-overview` is an Epiphany-local, renderer-neutral diagnostic projection over provider-owned typed state. It owns no lifecycle, discovery, composition, or mutation authority. Epiphany publishes clean provider-owned Eve surfaces; Odin may discover them; downstream lowerers and compositors remain outside Epiphany's concern. - - Idunn owns daemon survival and service lifecycle physiology. Codex/operator sessions may inspect, generate runbooks, and route explicit operator authority, but keeping daemons alive belongs to Idunn-owned service lifecycle receipts, scheduler policy, audits, and aftercare. Do not let Self, diagnostic projections, or ad hoc wrapper summaries become a second daemon-keeper. Global tool readbacks for Self `service-health`, `service-policy-directory`, and `swarm-online-runbook` state `lifecycleOwner=Idunn` and `hostedBody=Epiphany` without service mutation or private-state exposure. + - Idunn owns managed-service reconciliation and projector child lifecycle. On this workstation, Task Scheduler owns only current-user after-login presence of the Idunn reconciler; `gamecult-ops` owns only current-user after-login presence and configuration of the foreground Yggdrasil tunnel. Self, wrappers, diagnostic projections, lifecycle receipts, Task Scheduler, and the service manager are forbidden from becoming alternate projector, semantic-readiness, or recovery owners. Historical `swarm-online-runbook` and SCM service-control routes are rejected-path evidence, not current operator actions. - Use `notes/epiphany-cultmesh-dreaming-roadmap.md` as the design authority for distributed Epiphany dreaming. Public dreams are separately authored typed documents distributed through CultMesh/CultNet; private state remains local and cannot be shared by flipping a visibility flag. Foreign dreams are external thought weather until a reviewed local adoption receipt digests them into memory, planning, doctrine, or governance. - CultMesh is now the preferred ergonomic Rust abstraction over CultCache and CultNet for Epiphany-owned local database surfaces. The Rust substrate is repo-contained under `vendor/cultcache-rs`, `vendor/cultnet-rs`, and `vendor/cultmesh-rs`; the old `E:\Projects\CultLib\crates\*` path is dead in this checkout and must not be reintroduced as a compile-time dependency. Future local Verse/dream/status work should start at CultMesh unless it is explicitly polishing CultMesh internals. `CultMeshNode::get_all_with_keys()` now exposes typed document enumeration through the mesh organ, and `CultMeshNode::soa()` exposes CultCache's typed SoA table projection without bypassing the mesh organ. Compact audits can scan local Verse receipt families, and agent-state summary documents now prove SoA readback through CultMesh columns. `epiphany-verse-query agent-state` explicitly mirrors the persisted seven-row agent-state SoA into an existing bootstrapped local Verse; `agent-state-report` reads it without mutation; wrapper `-Mode agent-state-soa` visibly composes sync then report. Compact SoA rows now include each standing faculty's role id, agent id, profile kind, portable contract, memory/goal/value counts, and `private=false`; wrapper proof `local-20260618-175913-074-b22ff01c` printed all seven faculties in one sealed line. Preserve the three-Verse boundary: `epiphany-internal` for private sub-agent typed state, `gamecult-local` for trusted LAN/Yggdrasil-tunnel GameCult project sharing, and `epiphany-global` for untrusted public dreams plus topic-specific threaded public rooms for Persona posts. - Use `notes/epiphany-verse-architecture.md` as the source note for local Verse/Odin/Yggdrasil prompt-context architecture. Odin is the all-seer discovery/coordinator surface for advertised Verse metadata, not a trust-boundary bypass. Yggdrasil is the hosting spine for important trusted GameCult Verses such as Bifrost. The first native local Verse query surface is `epiphany-verse-query`, which seeds/queries compact CultMesh policy/status/contract context for dynamic prompt assembly and operator inspection; `epiphany-prompt-context-smoke` proves that compact local Verse context plus a semantic memory graph context cut can render a bounded dynamic prompt packet, omit unrelated private-looking text, and preserve the packet inside a role worker launch document. Live bridge role/reorient launch paths now feed `dynamicPromptContext` from sibling `local-verse.ccmp` and `memory-graph.msgpack` stores beside runtime-spine, refreshing the memory graph from current typed thread-state repo graphs when absent; the bridge launch-context test proves that packet survives into a persisted runtime-spine worker launch request. `tools/epiphany_local_run.ps1 -Mode status` now uses `.epiphany-run/cultmesh/local-verse.ccmp` as the shared local operator Verse store for run intent, operator snapshot, run receipt, and final `local-verse-context.json`, so the read path sees the latest operator witnesses instead of a policy-only seed packet. Local runs also refresh persisted `epiphany.agent_state_soa.v0` from `state/agents.msgpack` and mirror it into local Verse as sealed `epiphany.cultmesh.agent_state_soa_summary.v0`, preserving seven-faculty row/column shape and contract/count metadata without exposing memory text. `epiphany-verse-query poke-daemon --daemon-id ` now records an operator-safe `epiphany.cultmesh.daemon_poke_intent.v0` plus `epiphany.cultmesh.daemon_poke_receipt.v0` in local Verse, so a down daemon has a typed lifecycle intervention path without private Verse rummaging; follow-up daemon status remains the liveness authority. `epiphany-verse-query connect-eve` now selects an Odin advertisement by id or target cluster through the narrow Eve surface directory, records Eve connection intent/receipt documents, and verifies latest receipts through narrow CultMesh loaders; `tools/epiphany_local_run.ps1 -Mode eve-connect` wraps the same receipt path, defaulting to the Persona public discussion surface without full local Verse serialization. `epiphany-verse-query bifrost-publication` now records the Bifrost body-change publication intent, Bifrost publication receipt, and GitHub publication receipt from explicit changed path, verification, review, authorship, credit, ledger, Hands PR, and publication URL inputs, verifies the latest chain through narrow CultMesh loaders, and is wrapped by `tools/epiphany_local_run.ps1 -Mode bifrost-publication` for non-mutating operator proof defaults. `epiphany-verse-query collaboration-feedback` now records public Persona/peer collaboration feedback plus an Imagination consensus-discovery receipt from explicit public refs, Eve receipt, topic, summary, and candidate action refs, verifies latest feedback/consensus through narrow CultMesh loaders, and is wrapped by `tools/epiphany_local_run.ps1 -Mode collaboration-feedback`, which first records the prerequisite Eve receipt; adoption still requires Mind/Bifrost gates. `-Mode mvp` now seeds the same local Verse, routes Persona front-door `bubble` speech audits into that store, and writes final `local-verse-context.json` so the Persona mouth-edge witness is visible beside operator run witnesses. `epiphany-operator-snapshot from-status` now also distills the latest native status `tools.invocations[]` row into local Verse daemon-tool invocation intent/receipt mirrors, so operator-safe runtime tool calls become discoverable as `epiphany.cultmesh.daemon_tool_invocation_*` while runtime-spine remains lifecycle owner and raw MCP JSON stays quarantined behind typed tool documents. Plan/run/MVP coordinator summaries can now mirror their runtime-spine `epiphany.coordinator_run_receipt.v0` into local Verse as `epiphany.cultmesh.coordinator_run_receipt.v0`, referencing sealed artifacts by path only while keeping runtime-spine as lifecycle owner. When that summary contains a `finalAction.handsActionGate`, the same distillation pass writes `epiphany.cultmesh.hands_action_gate.v0` into local Verse with the Hands intent/review ids, Substrate Gate grant id, requested paths, required receipt types, and record-pass command template; runtime-spine Hands/Substrate Gate receipts remain the only action owners. When coordinator steps contain `roleAccept` or `roleFailureReview`, the same pass writes the latest operator-safe `epiphany.cultmesh.role_review_event.v0` mirror with surface, role id, status, source coordinator receipt id, and any exposed acceptance receipt/result/job ids; thread-state acceptance receipts remain the review owner. @@ -714,4 +714,7 @@ current_status: - Service lifecycle receipt history is now event-only: `load_epiphany_cultmesh_daemon_service_lifecycle_receipts` filters out the `epiphany-local/daemon-service-lifecycle-receipt/latest` mirror, while `load_latest_epiphany_cultmesh_daemon_service_lifecycle_receipt` remains the latest-state reader. Focused unit test `service_lifecycle_receipt_history_excludes_latest_mirror` proves two per-receipt writes return exactly two history records and latest points at the second. Fresh smoke `verse-query-lifecycle-history-filter-smoke` and live overview/triage `local-20260618-145632-497-efac6f0d` / `local-20260618-145632-321-867947af` prove compact swarm sights still expose service lifecycle attention plus the present elevated runbook artifact without daemon pokes or service mutation. Do not use a broad unscoped cargo test filter for this check on Windows; it fans out across bin harnesses and hit a broken pipe during this pass. - `epiphany-verse-query seed-compact` is the compact seed receipt for wrappers and read-only rites. The older `seed` command still prints the full local Verse context for explicit debugging, but compact wrapper modes now avoid full `seed`/final `query` readback after the combined overview proof exposed Windows debug stack overflow from full-context serialization pressure. last_completed_repo_swarm_cut: Huginn live-fire has crossed the first real repo Body from Persona pressure to branch-local Hands commit and Soul/Mind closure, and now has repeatable closure and redacted-public-proof mouths. On `E:\Projects\Huginn` branch `epiphany/huginn/first-awakening`, `tools/epiphany_local_run.ps1 -Mode repo-persona-intake` accepted item `huginn-inspection-readback-livefire`, Weksa/Persona intake remained sealed, and `epiphany-work derive-plan` lowered it to model-authored safe family `repo.status_section` for `README.md` only. `tools/epiphany_local_run.ps1 -Mode repo-swarm-run` advanced four bounded pulses: run-from-plan, adopt-from-plan, execute-from-plan, and close-from-execute. Huginn commit `e9b07c38fceabbdc89ea35c2156326c80c0393a2` updated only `README.md`; closure receipt `.epiphany\work\work-close-huginn-inspection-readback-livefire.json` records status `closed`, Soul verdict `passed`, Mind state commit `repo-work-close-huginn-inspection-readback-livefire-mind-commit`, safe family `repo.status_section`, path scope `README.md`, Bifrost publication gate, and `privateStateExposed=false`. Verification also ran Huginn's TypeScript compiler through concrete Node path: `C:\Program Files\nodejs\node.exe E:\Projects\Huginn\node_modules\typescript\bin\tsc -p tsconfig.json` passed. Native verifier `epiphany-repo-livefire-closure-smoke --workspace E:\Projects\Huginn --item huginn-inspection-readback-livefire --expected-path README.md --expected-family repo.status_section --expected-gate awaiting-publication --expected-blocker bifrost-publication-missing` checks closure receipt, Mind map entry, overview gate/blocker, authority seals, and `git show` changed paths, returning `status=ok`, `soulVerdict=passed`, `publicationGate=Bifrost`, and `privateStateExposed=false`. Wrapper `tools/epiphany_local_run.ps1 -Mode repo-livefire-closure -Workspace E:\Projects\Huginn -RepoWorkItem huginn-inspection-readback-livefire -SkipBuild` exposes the same read-only proof through the local operator mouth; artifact `local-20260621-190156-803-2f85ba15` printed commit, path, gate, blocker, Soul verdict, Mind commit receipt, and `privateStateExposed=False` without push, merge, publication, or target mutation. New wrapper `tools/epiphany_local_run.ps1 -Mode repo-work-public-proof -Workspace E:\Projects\Huginn -RepoWorkItem huginn-inspection-readback-livefire -SkipBuild` exports the redacted proof bundle and repo-local Verse index without publishing; artifact `local-20260621-190645-345-4555b1dc` wrote `.epiphany\public\proof-bundles\repo-work-public-proof-huginn-inspection-readback-livefire.json`, proof id `repo-work-public-proof-huginn-inspection-readback-livefire`, SHA-256 `857545b199f3e37eb06c2e757e5cfc201958e46667bb1f8403fd333a22620f63`, nine artifact rows, zero publication rows, and `privateStateExposed=False`. swarm overview readback `local-20260621-190701-140-b9519ec6` sees the `REPO-WORK-PUBLIC-PROOF` row and action row from Huginn's repo-local Verse with the same SHA-256 while Bifrost publication remains missing. Launcher compact modes are now named compact operator-context modes instead of read-only modes, and `bifrost-public-proof` summaries print item, target public Verse, room, proof SHA-256, review receipts, and credit receipts so publication receipts are auditable without claiming upstream sync. - next_action: Finish durable survival ownership for the now-live workstation semantic projector. The reserved policy is published against Yggdrasil Qdrant at `127.0.0.1:16333` and Ollama at `10.77.0.1:11435`. Explicit migrations removed the extinct ownerless operator-status projection and retired pre-Idunn claims, attempts, and index receipts from active semantic authority without fabricating grants; current Mind and Modeling projections were rebuilt through fresh Idunn grants and are query-eligible at 1024 dimensions (43 and 3 documents). Launch receipts now bind exact policy digest, child PID, executable SHA-256, fixed provider, and startup correlation; the child checks its own PID and bytes before constructing the body. Live proof shows a clean two-source idle pulse, ready correlated heartbeat, semantic ranking for both partitions, host singleton refusal, and Idunn restart after a forced child kill from PID 22992 to 29868 with launch receipt `daa89e4f-4301-40eb-aa87-5b52c5017221` and matching ready heartbeat. Library proof is 375 passed/1 ignored, supervisor proof 7/7, all binaries compile. Remaining deployment gap: the currently detached reconciler PID 22072 and ops SSH tunnel PID 22728 are not durable OS-owned physiology. Replace the false ordinary-console `sc.exe` service path with a real service host or explicit Task Scheduler ownership and restart policy, make the Qdrant tunnel equally durable, then prove reboot/startup recovery. Exact abandoned-claim recovery is unit/hostile proved but still needs a safe live recovery fixture; do not forge canonical work merely to make the dashboard green. + - Windows host survival authority is Task Scheduler, not SCM. Tasks `Epiphany-Idunn-Managed-Service-Reconciler` and `GameCult-Yggdrasil-Tunnel` run as current user Meta with `InteractiveToken`/Limited privilege, `AtLogOn` plus a one-minute recurring recovery trigger, `IgnoreNew`, zero execution limit, and restart-on-failure at one-minute intervals (999 attempts). Each task owns only after-login process presence through one exact direct foreground action. Idunn remains sole owner of managed-service reconciliation and projector child lifecycle; Task Scheduler cannot mint projector readiness, claims, grants, recovery, or terminal evidence. `gamecult-ops` owns the Yggdrasil tunnel profile and foreground SSH process; Epiphany only consumes pinned local Qdrant ports. This is explicitly after-login physiology, not pre-login, boot-time, machine-wide, or Windows-service survival. + - Live recurrence proof killed reconciler PID 24152 and observed Task Scheduler start exactly one replacement PID 25428. Killing projector PID 29868 then caused Idunn to launch PID 19888 with launch receipt `4d406060-6582-473a-9322-9f4ccc8b322f`; ready heartbeat `79fe3143-28a8-40fe-9547-0287159082e6` names provider incarnation `projector-59217a66-e3e2-44c3-b000-36af3d43b043` and the exact startup correlation. Tunnel recurrence killed SSH PID 22652 and observed PID 30444 start at 11:37:44 with Qdrant REST healthy. + - Historical Windows SCM material in this map is rejected-path evidence, not live authority. Arbitrary console `sc.exe` installation, elevated service execution runbooks/audits, cluster-service install/start/stop/control, `swarm-online-runbook`, service-manager action queues, and their wrapper routes no longer describe deployable physiology or actionable next work. Their receipts may remain sealed historical evidence, but they cannot decide liveness, readiness, recovery, or operator action. The console binaries never implemented the Windows service-control dispatcher and therefore could not honestly own SCM service state. + next_action: Prove this Task Scheduler boundary across a real reboot and user logon: both tasks must return to Running, the gamecult-ops foreground tunnel must restore `127.0.0.1:16333`/`:16334`, Idunn must reconcile exactly one projector child, and that child must publish a fresh launch-correlated ready heartbeat without operator-shell custody. Record that this proves after-login recovery only. Separately, construct an isolated exact abandoned-claim recovery fixture and prove fencing/recovery/resumption without fabricating canonical Mind or Modeling work. Do not resume SCM hardening or resurrect cluster-service/elevated-runbook wrappers. diff --git a/tools/epiphany_local_run.ps1 b/tools/epiphany_local_run.ps1 index aa15a0d4..fa1fd67e 100644 --- a/tools/epiphany_local_run.ps1 +++ b/tools/epiphany_local_run.ps1 @@ -1,5 +1,5 @@ param( - [ValidateSet("status", "plan", "smoke", "run", "mvp", "agent-state-soa", "swarm-status", "swarm-poke-down", "swarm-triage", "cluster-topology", "eve-surfaces", "eve-connect", "collaboration-feedback", "persona-discord", "persona-reddit", "persona-other", "persona-bridge", "daemon-survival-rehearsal", "repo-livefire-closure", "bifrost-publication", "bifrost-public-proof", "bifrost-artifact-acceptance", "bifrost-metrics", "bifrost-ledger", "receipt-directory", "tool-directory", "tool-invoke", "swarm-overview", "repo-persona-intake", "repo-swarm-run", "repo-work-queue-run", "repo-work-public-proof", "repo-work-readiness", "repo-deployment-config-audit", "repo-deployment-runbook", "repo-deployment-aftercare-audit", "repo-work-service-plan", "repo-work-service-runbook", "repo-work-service-launch", "repo-work-service-audit", "swarm-online-runbook", "service-policy-directory", "service-plan", "service-launch", "service-runbook", "cluster-service-runbook", "cluster-service-install-plan", "cluster-service-install-execute", "cluster-service-audit", "cluster-service-start-plan", "cluster-service-stop-plan", "cluster-service-start-execute", "cluster-service-stop-execute", "cluster-service-execution-readiness", "cluster-service-execution-runbook", "cluster-service-execution-audit", "service-execution-runbook", "service-install-plan", "service-install-execute", "service-tick", "service-status", "service-reconcile", "service-execution-readiness", "service-execution-audit", "service-start-plan", "service-stop-plan", "service-start-execute", "service-stop-execute")] + [ValidateSet("status", "plan", "smoke", "run", "mvp", "agent-state-soa", "swarm-status", "swarm-poke-down", "swarm-triage", "cluster-topology", "eve-surfaces", "eve-connect", "collaboration-feedback", "persona-discord", "persona-reddit", "persona-other", "persona-bridge", "daemon-survival-rehearsal", "repo-livefire-closure", "bifrost-publication", "bifrost-public-proof", "bifrost-artifact-acceptance", "bifrost-metrics", "bifrost-ledger", "receipt-directory", "tool-directory", "tool-invoke", "swarm-overview", "repo-persona-intake", "repo-swarm-run", "repo-work-queue-run", "repo-work-public-proof", "repo-work-readiness", "repo-deployment-config-audit", "repo-deployment-runbook", "repo-deployment-aftercare-audit", "repo-work-service-plan", "repo-work-service-runbook", "repo-work-service-launch", "repo-work-service-audit", "service-policy-directory", "service-plan", "service-launch", "service-runbook", "service-tick", "managed-service-task-plan", "managed-service-task-install", "managed-service-task-status", "managed-service-task-start", "managed-service-task-stop", "managed-service-task-uninstall")] [string]$Mode = "smoke", [string]$Root = (Resolve-Path ".").Path, [string]$Workspace = "", @@ -13,11 +13,8 @@ param( [int]$TimeoutSeconds = 600, [int]$MaxRuntimeSeconds = 180, [string]$DaemonId = "*", - [string]$ServiceId = "epiphany-daemon-supervisor-service", - [string]$ServiceName = "", - [string]$ServiceDisplayName = "", - [ValidateSet("auto", "demand", "disabled")] - [string]$ServiceStartType = "demand", + [string]$ServiceId = "epiphany-memory-semantic-projector-service", + [string]$ServiceName = "Epiphany-Idunn-Managed-Service-Reconciler", [string]$SchedulerId = "epiphany-daemon-supervisor", [int]$LoopIntervalSeconds = 60, [int]$ServiceMaxIterations = 0, @@ -178,57 +175,6 @@ function Invoke-Checked { } } -function Format-ServiceExecutionFailedChecks { - param([object]$Rows) - - $rowList = @($Rows) - if ($null -eq $Rows -or $rowList.Count -eq 0) { - return "none" - } - - return (($rowList | ForEach-Object { - $observed = $_.observedStatus - if ($null -eq $observed -or $observed -eq "") { - $observed = "missing" - } - $artifact = $_.operatorArtifactRef - if ($null -eq $artifact -or $artifact -eq "") { - $artifact = "none" - } - $artifactSha256 = "none" - if ($artifact -ne "none" -and (Test-Path -LiteralPath $artifact -PathType Leaf)) { - $artifactSha256 = (Get-FileHash -LiteralPath $artifact -Algorithm SHA256).Hash.ToLowerInvariant() - } - $serviceId = $_.serviceId - if ($null -eq $serviceId -or $serviceId -eq "") { - $serviceId = "unknown-service" - } - $followUp = Get-ServiceExecutionCheckFollowUpCommand $_.action - "${serviceId}::$($_.action)=${observed}:artifact=${artifact}:sha256=${artifactSha256}:followUp=${followUp}" - }) -join "; ") -} - -function Get-ServiceExecutionCheckFollowUpCommand { - param([string]$Action) - - switch ($Action) { - "cluster-windows-service-execution-runbook" { return "tools/epiphany_local_run.ps1 -Mode cluster-service-execution-runbook" } - "cluster-windows-service-execution-readiness" { return "tools/epiphany_local_run.ps1 -Mode cluster-service-execution-readiness" } - "cluster-windows-service-install" { return "tools/epiphany_local_run.ps1 -Mode cluster-service-install-execute" } - "cluster-windows-service-start" { return "tools/epiphany_local_run.ps1 -Mode cluster-service-start-execute" } - "cluster-windows-service-execution-audit" { return "tools/epiphany_local_run.ps1 -Mode cluster-service-execution-audit" } - "cluster-windows-service-stop" { return "tools/epiphany_local_run.ps1 -Mode cluster-service-stop-execute" } - "windows-service-execution-runbook" { return "tools/epiphany_local_run.ps1 -Mode service-execution-runbook" } - "windows-service-execution-readiness" { return "tools/epiphany_local_run.ps1 -Mode service-execution-readiness" } - "windows-service-install" { return "tools/epiphany_local_run.ps1 -Mode service-install-execute" } - "windows-service-start" { return "tools/epiphany_local_run.ps1 -Mode service-start-execute" } - "windows-service-status" { return "tools/epiphany_local_run.ps1 -Mode service-status" } - "windows-service-reconcile" { return "tools/epiphany_local_run.ps1 -Mode service-reconcile" } - "windows-service-stop" { return "tools/epiphany_local_run.ps1 -Mode service-stop-execute" } - default { return "tools/epiphany_local_run.ps1 -Mode swarm-overview" } - } -} - function ConvertTo-PowerShellLiteral { param([string]$Value) @@ -384,7 +330,7 @@ $runtimeStore = Join-Path $Workspace "state\runtime-spine.msgpack" $liveRuntimeMode = @("run", "mvp") -contains $Mode if (-not $SkipBuild) { - if ($Mode -ne "status" -and $Mode -ne "agent-state-soa" -and $Mode -ne "swarm-status" -and $Mode -ne "swarm-poke-down" -and $Mode -ne "swarm-triage" -and $Mode -ne "cluster-topology" -and $Mode -ne "eve-surfaces" -and $Mode -ne "eve-connect" -and $Mode -ne "collaboration-feedback" -and $Mode -ne "persona-discord" -and $Mode -ne "persona-reddit" -and $Mode -ne "persona-other" -and $Mode -ne "persona-bridge" -and $Mode -ne "daemon-survival-rehearsal" -and $Mode -ne "repo-livefire-closure" -and $Mode -ne "repo-work-public-proof" -and $Mode -ne "bifrost-publication" -and $Mode -ne "bifrost-public-proof" -and $Mode -ne "bifrost-artifact-acceptance" -and $Mode -ne "bifrost-metrics" -and $Mode -ne "bifrost-ledger" -and $Mode -ne "receipt-directory" -and $Mode -ne "tool-directory" -and $Mode -ne "tool-invoke" -and $Mode -ne "swarm-overview" -and $Mode -ne "repo-persona-intake" -and $Mode -ne "repo-swarm-run" -and $Mode -ne "repo-work-queue-run" -and $Mode -ne "repo-work-readiness" -and $Mode -ne "repo-deployment-config-audit" -and $Mode -ne "repo-deployment-runbook" -and $Mode -ne "repo-deployment-aftercare-audit" -and $Mode -ne "repo-work-service-plan" -and $Mode -ne "repo-work-service-runbook" -and $Mode -ne "repo-work-service-launch" -and $Mode -ne "repo-work-service-audit" -and $Mode -ne "swarm-online-runbook" -and $Mode -ne "service-policy-directory" -and $Mode -ne "service-plan" -and $Mode -ne "service-launch" -and $Mode -ne "service-runbook" -and $Mode -ne "cluster-service-runbook" -and $Mode -ne "cluster-service-install-plan" -and $Mode -ne "cluster-service-install-execute" -and $Mode -ne "cluster-service-audit" -and $Mode -ne "cluster-service-start-plan" -and $Mode -ne "cluster-service-stop-plan" -and $Mode -ne "cluster-service-start-execute" -and $Mode -ne "cluster-service-stop-execute" -and $Mode -ne "cluster-service-execution-readiness" -and $Mode -ne "cluster-service-execution-runbook" -and $Mode -ne "cluster-service-execution-audit" -and $Mode -ne "service-execution-runbook" -and $Mode -ne "service-install-plan" -and $Mode -ne "service-install-execute" -and $Mode -ne "service-tick" -and $Mode -ne "service-status" -and $Mode -ne "service-reconcile" -and $Mode -ne "service-execution-readiness" -and $Mode -ne "service-execution-audit" -and $Mode -ne "service-start-plan" -and $Mode -ne "service-stop-plan" -and $Mode -ne "service-start-execute" -and $Mode -ne "service-stop-execute") { + if (@("plan", "smoke", "run", "mvp") -contains $Mode) { Invoke-Checked ` -Label "build Codex app-server compatibility organ" ` -FilePath $cargoExe ` @@ -453,10 +399,24 @@ if ($Mode -eq "daemon-survival-rehearsal") { if ($Mode -eq "repo-livefire-closure") { $requiredBinaries += @($repoLivefireClosureSmokeExe) } + +function Format-ServiceExecutionFailedChecks { + param([object]$Rows) + + $rowList = @($Rows) + if ($null -eq $Rows -or $rowList.Count -eq 0) { + return "none" + } + return (($rowList | ForEach-Object { + $serviceId = if ($null -eq $_.serviceId -or $_.serviceId -eq "") { "unknown-service" } else { $_.serviceId } + $observed = if ($null -eq $_.observedStatus -or $_.observedStatus -eq "") { "missing" } else { $_.observedStatus } + "${serviceId}::$($_.action)=${observed}:followUp=tools/epiphany_local_run.ps1 -Mode swarm-overview" + }) -join "; ") +} if ($Mode -eq "persona-bridge") { $requiredBinaries += @($PersonaBridgeSmokeExe) } -if ($Mode -ne "status" -and $Mode -ne "agent-state-soa" -and $Mode -ne "swarm-status" -and $Mode -ne "swarm-poke-down" -and $Mode -ne "swarm-triage" -and $Mode -ne "cluster-topology" -and $Mode -ne "eve-surfaces" -and $Mode -ne "eve-connect" -and $Mode -ne "collaboration-feedback" -and $Mode -ne "persona-discord" -and $Mode -ne "persona-reddit" -and $Mode -ne "persona-other" -and $Mode -ne "persona-bridge" -and $Mode -ne "daemon-survival-rehearsal" -and $Mode -ne "repo-livefire-closure" -and $Mode -ne "repo-work-public-proof" -and $Mode -ne "bifrost-publication" -and $Mode -ne "bifrost-public-proof" -and $Mode -ne "bifrost-artifact-acceptance" -and $Mode -ne "bifrost-metrics" -and $Mode -ne "bifrost-ledger" -and $Mode -ne "receipt-directory" -and $Mode -ne "tool-directory" -and $Mode -ne "tool-invoke" -and $Mode -ne "swarm-overview" -and $Mode -ne "repo-persona-intake" -and $Mode -ne "repo-swarm-run" -and $Mode -ne "repo-work-queue-run" -and $Mode -ne "repo-work-readiness" -and $Mode -ne "repo-deployment-config-audit" -and $Mode -ne "repo-deployment-runbook" -and $Mode -ne "repo-deployment-aftercare-audit" -and $Mode -ne "repo-work-service-plan" -and $Mode -ne "repo-work-service-runbook" -and $Mode -ne "repo-work-service-launch" -and $Mode -ne "repo-work-service-audit" -and $Mode -ne "swarm-online-runbook" -and $Mode -ne "service-policy-directory" -and $Mode -ne "service-plan" -and $Mode -ne "service-launch" -and $Mode -ne "service-runbook" -and $Mode -ne "cluster-service-runbook" -and $Mode -ne "cluster-service-install-plan" -and $Mode -ne "cluster-service-install-execute" -and $Mode -ne "cluster-service-audit" -and $Mode -ne "cluster-service-start-plan" -and $Mode -ne "cluster-service-stop-plan" -and $Mode -ne "cluster-service-start-execute" -and $Mode -ne "cluster-service-stop-execute" -and $Mode -ne "cluster-service-execution-readiness" -and $Mode -ne "cluster-service-execution-runbook" -and $Mode -ne "cluster-service-execution-audit" -and $Mode -ne "service-execution-runbook" -and $Mode -ne "service-install-plan" -and $Mode -ne "service-install-execute" -and $Mode -ne "service-tick" -and $Mode -ne "service-status" -and $Mode -ne "service-reconcile" -and $Mode -ne "service-execution-readiness" -and $Mode -ne "service-execution-audit" -and $Mode -ne "service-start-plan" -and $Mode -ne "service-stop-plan" -and $Mode -ne "service-start-execute" -and $Mode -ne "service-stop-execute") { +if (@("plan", "smoke", "run", "mvp") -contains $Mode) { $requiredBinaries += @($codexAppServer, $coordinatorExe) } function Assert-SwarmBrakeAllowsLiveRun { @@ -529,7 +489,7 @@ function Assert-SwarmBrakeAllowsLiveRun { } } -if (@("plan", "run", "mvp", "repo-persona-intake", "repo-swarm-run", "repo-work-queue-run", "repo-livefire-closure", "repo-work-public-proof", "repo-work-readiness", "repo-deployment-config-audit", "repo-deployment-runbook", "repo-deployment-aftercare-audit", "repo-work-service-plan", "repo-work-service-runbook", "repo-work-service-launch", "repo-work-service-audit", "service-plan", "service-launch", "service-runbook", "cluster-service-runbook", "cluster-service-install-plan", "cluster-service-install-execute", "cluster-service-audit", "cluster-service-start-plan", "cluster-service-stop-plan", "cluster-service-start-execute", "cluster-service-stop-execute", "cluster-service-execution-readiness", "cluster-service-execution-runbook", "cluster-service-execution-audit", "service-execution-runbook", "service-install-plan", "service-install-execute", "service-tick", "service-status", "service-reconcile", "service-execution-readiness", "service-execution-audit", "service-start-plan", "service-stop-plan", "service-start-execute", "service-stop-execute") -contains $Mode) { +if (@("plan", "run", "mvp", "repo-persona-intake", "repo-swarm-run", "repo-work-queue-run", "repo-livefire-closure", "repo-work-public-proof", "repo-work-readiness", "repo-deployment-config-audit", "repo-deployment-runbook", "repo-deployment-aftercare-audit", "repo-work-service-plan", "repo-work-service-runbook", "repo-work-service-launch", "repo-work-service-audit", "service-plan", "service-launch", "service-runbook", "service-tick", "managed-service-task-plan", "managed-service-task-install", "managed-service-task-status", "managed-service-task-start", "managed-service-task-stop", "managed-service-task-uninstall") -contains $Mode) { if (-not (Test-Path -LiteralPath $verseQueryExe)) { throw "required binary not found for swarm brake preflight: $verseQueryExe" } @@ -539,7 +499,7 @@ if (@("plan", "run", "mvp", "repo-persona-intake", "repo-swarm-run", "repo-work- if ($liveRuntimeMode) { $requiredBinaries += @($modelRuntimeExe, $toolAdapterExe) } -if (@("service-policy-directory", "service-plan", "service-launch", "service-tick", "service-runbook", "cluster-service-runbook", "cluster-service-install-plan", "cluster-service-install-execute", "cluster-service-audit", "cluster-service-start-plan", "cluster-service-stop-plan", "cluster-service-start-execute", "cluster-service-stop-execute", "cluster-service-execution-readiness", "cluster-service-execution-runbook", "cluster-service-execution-audit", "service-execution-runbook", "service-install-plan", "service-install-execute", "service-status", "service-reconcile", "service-execution-readiness", "service-execution-audit", "service-start-plan", "service-stop-plan", "service-start-execute", "service-stop-execute") -contains $Mode) { +if ($false) { $requiredBinaries += @($clusterDaemonExe) } if ($Mode -eq "mvp") { @@ -628,7 +588,7 @@ Invoke-Checked ` -StdoutPath (Join-Path $artifactRoot "operator-run-intent.stdout.json") ` -StderrPath (Join-Path $artifactRoot "operator-run-intent.stderr.log") -$compactOperatorContextModes = @("agent-state-soa", "swarm-status", "swarm-poke-down", "swarm-triage", "cluster-topology", "eve-surfaces", "eve-connect", "collaboration-feedback", "daemon-survival-rehearsal", "repo-livefire-closure", "bifrost-publication", "bifrost-public-proof", "bifrost-artifact-acceptance", "bifrost-metrics", "bifrost-ledger", "receipt-directory", "tool-directory", "tool-invoke", "swarm-overview", "swarm-online-runbook", "service-policy-directory") +$compactOperatorContextModes = @("agent-state-soa", "swarm-status", "swarm-poke-down", "swarm-triage", "cluster-topology", "eve-surfaces", "eve-connect", "collaboration-feedback", "daemon-survival-rehearsal", "repo-livefire-closure", "bifrost-publication", "bifrost-public-proof", "bifrost-artifact-acceptance", "bifrost-metrics", "bifrost-ledger", "receipt-directory", "tool-directory", "tool-invoke", "swarm-overview", "service-policy-directory") $usesCompactOperatorContext = $compactOperatorContextModes -contains $Mode $shouldReadLocalVerse = $Mode -ne "smoke" -and -not $usesCompactOperatorContext if ($shouldReadLocalVerse -and $Mode -ne "status" -and $Mode -ne "mvp") { @@ -1158,242 +1118,6 @@ if ($Mode -eq "swarm-overview") { -StderrPath (Join-Path $artifactRoot "swarm-overview.stderr.log") } -if ($Mode -eq "swarm-online-runbook") { - $overviewPath = Join-Path $artifactRoot "swarm-online-overview.stdout.json" - $runbookPath = Join-Path $artifactRoot "epiphany-swarm-online-runbook.ps1" - $resultPath = Join-Path $artifactRoot "swarm-online-runbook.stdout.json" - $wrapperScriptPath = $PSCommandPath - if ($wrapperScriptPath -eq "") { - $wrapperScriptPath = Join-Path $Root "tools\epiphany_local_run.ps1" - } - $wrapperScriptPath = (Resolve-Path $wrapperScriptPath).Path - Invoke-Checked ` - -Label "read compact swarm overview for online runbook" ` - -FilePath $verseQueryExe ` - -Arguments @( - "swarm-overview", - "--store", $localVerseStore, - "--runtime-id", $LocalVerseRuntimeId - ) ` - -WorkingDirectory $Root ` - -StdoutPath $overviewPath ` - -StderrPath (Join-Path $artifactRoot "swarm-online-overview.stderr.log") - - $overview = Get-Content -LiteralPath $overviewPath -Raw | ConvertFrom-Json - $authorityRows = @($overview.swarmActionRows | Where-Object { - $_.family -eq "service-execution-authority" -and - $_.operatorArtifactStatus -eq "present" -and - $null -ne $_.operatorArtifactExecutionCommand -and - $_.operatorArtifactExecutionCommand -ne "" -and - $_.operatorArtifactExecutionCommand -ne "none" - } | Sort-Object priority) - if ($authorityRows.Count -eq 0) { - throw "swarm-online-runbook found no present elevated service execution artifacts in swarm overview" - } - - $childArtifactPreflightRows = @($authorityRows | ForEach-Object { - $artifactPresent = $false - $artifactActualSha256 = "none" - $artifactSha256Matches = $false - if ($null -ne $_.operatorArtifactRef -and $_.operatorArtifactRef -ne "" -and $_.operatorArtifactRef -ne "none" -and (Test-Path -LiteralPath $_.operatorArtifactRef -PathType Leaf)) { - $artifactPresent = $true - $artifactActualSha256 = (Get-FileHash -LiteralPath $_.operatorArtifactRef -Algorithm SHA256).Hash.ToLowerInvariant() - $artifactSha256Matches = ($_.operatorArtifactSha256 -ne "none" -and $artifactActualSha256 -eq $_.operatorArtifactSha256) - } - [pscustomobject]@{ - serviceId = $_.serviceId - serviceRoute = $_.serviceRoute - artifactRef = $_.operatorArtifactRef - expectedSha256 = $_.operatorArtifactSha256 - actualSha256 = $artifactActualSha256 - artifactPresent = $artifactPresent - artifactSha256Matches = $artifactSha256Matches - privateStateExposed = $_.privateStateExposed - } - }) - $childArtifactPresentCount = @($childArtifactPreflightRows | Where-Object { $_.artifactPresent }).Count - $childArtifactHashVerifiedCount = @($childArtifactPreflightRows | Where-Object { $_.artifactSha256Matches }).Count - $childArtifactMismatchCount = @($childArtifactPreflightRows | Where-Object { $_.artifactPresent -and -not $_.artifactSha256Matches }).Count - $childArtifactMissingCount = @($childArtifactPreflightRows | Where-Object { -not $_.artifactPresent }).Count - $serviceExecutionFailedCheckCount = 0 - $serviceExecutionMissingCheckCount = 0 - $privateStateExposed = $false - foreach ($row in $authorityRows) { - $serviceExecutionFailedCheckCount += [int]$row.serviceExecutionFailedCheckCount - $serviceExecutionMissingCheckCount += [int]$row.serviceExecutionMissingCheckCount - if ($row.privateStateExposed) { - $privateStateExposed = $true - } - } - $preflightStatus = "ready-for-elevated-operator" - if ($childArtifactMissingCount -gt 0 -or $childArtifactMismatchCount -gt 0 -or $privateStateExposed) { - $preflightStatus = "blocked-regenerate-runbooks" - } - - $aftercareCommands = @($authorityRows | ForEach-Object { $_.operatorAftercareCommand } | Where-Object { $null -ne $_ -and $_ -ne "" -and $_ -ne "none" } | Select-Object -Unique) - $aftercareRows = @($authorityRows | Where-Object { - $null -ne $_.operatorAftercareCommand -and - $_.operatorAftercareCommand -ne "" -and - $_.operatorAftercareCommand -ne "none" -and - $null -ne $_.completionAuditWrapperMode -and - $_.completionAuditWrapperMode -ne "" -and - $_.completionAuditWrapperMode -ne "none" - } | Sort-Object operatorAftercareCommand, serviceId -Unique) - $quotedRoot = ConvertTo-PowerShellLiteral $Root - $runbookLines = @( - "# Epiphany swarm online runbook.", - "# Lifecycle owner: Idunn. This artifact routes operator authority to Idunn-owned service lifecycle receipts.", - "# Generated from compact CultMesh swarm action rows.", - "# Run only with explicit operator authority. The individual sealed runbooks write lifecycle receipts.", - "# After execution, this runbook executes the advertised aftercare audit command(s).", - "Set-StrictMode -Version Latest", - '$ErrorActionPreference = "Stop"', - "Set-Location -LiteralPath $quotedRoot", - '$failureCount = 0', - "" - ) - foreach ($row in $authorityRows) { - $serviceLiteral = ConvertTo-PowerShellLiteral $row.serviceId - $routeLiteral = ConvertTo-PowerShellLiteral $row.serviceRoute - $artifactLiteral = ConvertTo-PowerShellLiteral $row.operatorArtifactRef - $shaLiteral = ConvertTo-PowerShellLiteral $row.operatorArtifactSha256 - $aftercareLiteral = ConvertTo-PowerShellLiteral $row.operatorAftercareCommand - $runbookLines += "# Service: $($row.serviceId)" - $runbookLines += "# Route: $($row.serviceRoute)" - $runbookLines += "# Artifact: $($row.operatorArtifactRef)" - $runbookLines += "# SHA-256: $($row.operatorArtifactSha256)" - $runbookLines += "# Aftercare: $($row.operatorAftercareCommand)" - $runbookLines += "Write-Host `"Running Idunn service lifecycle runbook for Epiphany: service=$serviceLiteral route=$routeLiteral artifact=$artifactLiteral sha256=$shaLiteral aftercare=$aftercareLiteral`"" - $runbookLines += "try {" - $runbookLines += " if (-not (Test-Path -LiteralPath $artifactLiteral -PathType Leaf)) { throw `"missing child runbook artifact: $artifactLiteral`" }" - $runbookLines += " if ($shaLiteral -ne 'none') {" - $runbookLines += " `$actualSha256 = (Get-FileHash -LiteralPath $artifactLiteral -Algorithm SHA256).Hash.ToLowerInvariant()" - $runbookLines += " if (`$actualSha256 -ne $shaLiteral) { throw `"child runbook SHA-256 mismatch: expected=$shaLiteral actual=`$actualSha256 artifact=$artifactLiteral`" }" - $runbookLines += " }" - $runbookLines += " `$process = Start-Process PowerShell -Verb RunAs -Wait -PassThru -ArgumentList @('-NoProfile','-ExecutionPolicy','Bypass','-File',$artifactLiteral)" - $runbookLines += " `$exitCode = `$process.ExitCode" - $runbookLines += " if (`$null -eq `$exitCode) { `$exitCode = -1 }" - $runbookLines += " Write-Host `"Idunn service lifecycle runbook exit: service=$serviceLiteral route=$routeLiteral exitCode=`$exitCode`"" - $runbookLines += " if (`$exitCode -ne 0) { `$failureCount += 1 }" - $runbookLines += "} catch {" - $runbookLines += " `$failureCount += 1" - $runbookLines += " Write-Host `"Idunn service lifecycle runbook launch failed: service=$serviceLiteral route=$routeLiteral error=`$(`$_.Exception.Message)`"" - $runbookLines += "}" - $runbookLines += "" - } - $runbookLines += "# Aftercare audit command(s):" - foreach ($aftercareRow in $aftercareRows) { - $aftercareCommandLiteral = ConvertTo-PowerShellLiteral $aftercareRow.operatorAftercareCommand - $aftercareModeLiteral = ConvertTo-PowerShellLiteral $aftercareRow.completionAuditWrapperMode - $aftercareServiceLiteral = ConvertTo-PowerShellLiteral $aftercareRow.serviceId - $aftercareScriptLiteral = ConvertTo-PowerShellLiteral $wrapperScriptPath - $aftercareArgs = @( - "-Mode", $aftercareRow.completionAuditWrapperMode, - "-SkipBuild", - "-Root", $Root, - "-Workspace", $Workspace, - "-CodexHome", $CodexHome, - "-TargetDir", $TargetDir, - "-DaemonId", $DaemonId, - "-ServiceId", $aftercareRow.serviceId, - "-ServiceStartType", $ServiceStartType, - "-SchedulerId", $SchedulerId, - "-LoopIntervalSeconds", "$LoopIntervalSeconds", - "-ServiceMaxIterations", "$ServiceMaxIterations" - ) - $aftercareArgsLiteral = ConvertTo-PowerShellArrayLiteral $aftercareArgs - $runbookLines += "Write-Host `"Running Idunn aftercare audit for Epiphany service lifecycle: command=$aftercareCommandLiteral mode=$aftercareModeLiteral service=$aftercareServiceLiteral`"" - $runbookLines += "try {" - $runbookLines += " `$global:LASTEXITCODE = 0" - $runbookLines += " & $aftercareScriptLiteral $aftercareArgsLiteral" - $runbookLines += " if (`$null -ne `$global:LASTEXITCODE -and `$global:LASTEXITCODE -ne 0) { throw `"Aftercare exited with code `$global:LASTEXITCODE`" }" - $runbookLines += "} catch {" - $runbookLines += " `$failureCount += 1" - $runbookLines += " Write-Host `"Idunn aftercare audit failed: command=$aftercareCommandLiteral mode=$aftercareModeLiteral service=$aftercareServiceLiteral error=`$(`$_.Exception.Message)`"" - $runbookLines += "}" - } - $runbookLines += 'if ($failureCount -gt 0) {' - $runbookLines += ' Write-Host "Idunn service lifecycle runbook for Epiphany failed: failures=$failureCount"' - $runbookLines += ' exit 1' - $runbookLines += '}' - $runbookLines += 'Write-Host "Idunn service lifecycle runbook for Epiphany completed: failures=0"' - Set-Content -LiteralPath $runbookPath -Value ($runbookLines -join [Environment]::NewLine) -Encoding UTF8 - $runbookSha256 = Get-LocalArtifactSha256 $runbookPath - $elevatedCommand = Get-ElevatedRunbookCommand $runbookPath - $childRunbookRows = @($authorityRows | ForEach-Object { - $authorityRow = $_ - $preflightRow = @($childArtifactPreflightRows | Where-Object { $_.artifactRef -eq $authorityRow.operatorArtifactRef } | Select-Object -First 1) - [pscustomobject]@{ - priority = $authorityRow.priority - family = $authorityRow.family - status = $authorityRow.status - lifecycleOwner = $authorityRow.lifecycleOwner - hostedBody = $authorityRow.hostedBody - serviceId = $authorityRow.serviceId - serviceRoute = $authorityRow.serviceRoute - wrapperMode = $authorityRow.wrapperMode - wrapperCommand = $authorityRow.wrapperCommand - authorityGate = $authorityRow.authorityGate - effectClass = $authorityRow.effectClass - mutatesState = $authorityRow.mutatesState - requiresElevatedAuthority = $authorityRow.requiresElevatedAuthority - artifactRef = $authorityRow.operatorArtifactRef - artifactStatus = $authorityRow.operatorArtifactStatus - artifactSha256 = $authorityRow.operatorArtifactSha256 - artifactActualSha256 = $preflightRow.actualSha256 - artifactSha256Verified = $preflightRow.artifactSha256Matches - artifactExecutionCommand = $authorityRow.operatorArtifactExecutionCommand - aftercareCommand = $authorityRow.operatorAftercareCommand - completionAuditWrapperMode = $authorityRow.completionAuditWrapperMode - completionAuditWrapperCommand = $authorityRow.completionAuditWrapperCommand - serviceExecutionFailedCheckCount = $authorityRow.serviceExecutionFailedCheckCount - serviceExecutionMissingCheckCount = $authorityRow.serviceExecutionMissingCheckCount - privateStateExposed = $authorityRow.privateStateExposed - } - }) - $aftercareRunbookRows = @($aftercareRows | ForEach-Object { - [pscustomobject]@{ - serviceId = $_.serviceId - serviceRoute = $_.serviceRoute - command = $_.operatorAftercareCommand - wrapperMode = $_.completionAuditWrapperMode - wrapperCommand = $_.completionAuditWrapperCommand - scriptPath = $wrapperScriptPath - privateStateExposed = $_.privateStateExposed - } - }) - [pscustomobject]@{ - status = "ok" - preflightStatus = $preflightStatus - store = $localVerseStore - runtimeId = $LocalVerseRuntimeId - runbookPath = $runbookPath - artifactSha256 = $runbookSha256 - elevatedCommand = $elevatedCommand - commandCount = $authorityRows.Count - childArtifactPresentCount = $childArtifactPresentCount - childArtifactHashVerifiedCount = $childArtifactHashVerifiedCount - childArtifactMismatchCount = $childArtifactMismatchCount - childArtifactMissingCount = $childArtifactMissingCount - serviceExecutionFailedCheckCount = $serviceExecutionFailedCheckCount - serviceExecutionMissingCheckCount = $serviceExecutionMissingCheckCount - lifecycleOwner = "Idunn" - hostedBody = "Epiphany" - commands = @($authorityRows | ForEach-Object { $_.operatorArtifactExecutionCommand }) - aftercareCommands = $aftercareCommands - childRunbookRows = $childRunbookRows - childArtifactPreflightRows = $childArtifactPreflightRows - aftercareRows = $aftercareRunbookRows - detectsChildExitCodes = $true - continuesAfterChildFailure = $true - verifiesChildArtifactSha256 = $true - usesExplicitAftercareArguments = $true - exitsNonzeroAfterChildOrAftercareFailure = $true - sourceOverviewPath = $overviewPath - privateStateExposed = $privateStateExposed - } | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath $resultPath -Encoding UTF8 -} - if ($Mode -eq "service-policy-directory") { $resultPath = Join-Path $artifactRoot "daemon-service-policy-directory.stdout.json" Invoke-Checked ` @@ -1480,482 +1204,6 @@ if ($Mode -eq "service-runbook") { -StderrPath (Join-Path $artifactRoot "daemon-service-runbook.stderr.log") } -if ($Mode -eq "cluster-service-runbook") { - $resultPath = Join-Path $artifactRoot "cluster-daemon-service-runbook.stdout.json" - $runbookPath = Join-Path $artifactRoot "epiphany-cluster-daemon-services.ps1" - $clusterServiceId = $ServiceId - if ($clusterServiceId -eq "epiphany-daemon-supervisor-service") { - $clusterServiceId = "epiphany-cluster-daemon-services" - } - $runbookArgs = @( - "cluster-service-runbook", - "--store", $localVerseStore, - "--runtime-id", $LocalVerseRuntimeId, - "--daemon-id", $DaemonId, - "--scheduler-id", $SchedulerId, - "--service-id", $clusterServiceId, - "--service-command", $clusterDaemonExe, - "--loop-interval-seconds", "$LoopIntervalSeconds", - "--runbook-path", $runbookPath - ) - if ($ServiceMaxIterations -gt 0) { - $runbookArgs += @("--max-iterations", "$ServiceMaxIterations") - } - Invoke-Checked ` - -Label "write cluster daemon service runbook" ` - -FilePath $daemonSupervisorExe ` - -Arguments $runbookArgs ` - -WorkingDirectory $Root ` - -StdoutPath $resultPath ` - -StderrPath (Join-Path $artifactRoot "cluster-daemon-service-runbook.stderr.log") -} - -if ($Mode -eq "cluster-service-install-plan" -or $Mode -eq "cluster-service-install-execute") { - $clusterInstallModeName = if ($Mode -eq "cluster-service-install-execute") { "execute" } else { "plan" } - $resultPath = Join-Path $artifactRoot "cluster-daemon-service-install-$clusterInstallModeName.stdout.json" - $installScriptPath = Join-Path $artifactRoot "epiphany-cluster-daemon-services-install.ps1" - $clusterServiceId = $ServiceId - if ($clusterServiceId -eq "epiphany-daemon-supervisor-service") { - $clusterServiceId = "epiphany-cluster-daemon-services" - } - $installArgs = @( - "cluster-service-install-$clusterInstallModeName", - "--store", $localVerseStore, - "--runtime-id", $LocalVerseRuntimeId, - "--daemon-id", $DaemonId, - "--scheduler-id", $SchedulerId, - "--service-id", $clusterServiceId, - "--service-command", $clusterDaemonExe, - "--service-start-type", $ServiceStartType, - "--loop-interval-seconds", "$LoopIntervalSeconds", - "--service-install-script-path", $installScriptPath - ) - if ($ServiceName -ne "") { - $installArgs += @("--service-name", $ServiceName) - } - if ($ServiceDisplayName -ne "") { - $installArgs += @("--service-display-name", $ServiceDisplayName) - } - if ($ServiceMaxIterations -gt 0) { - $installArgs += @("--max-iterations", "$ServiceMaxIterations") - } - Invoke-Checked ` - -Label "write cluster daemon Windows service install $clusterInstallModeName" ` - -FilePath $daemonSupervisorExe ` - -Arguments $installArgs ` - -WorkingDirectory $Root ` - -StdoutPath $resultPath ` - -StderrPath (Join-Path $artifactRoot "cluster-daemon-service-install-$clusterInstallModeName.stderr.log") -} - -if ($Mode -eq "cluster-service-audit") { - $resultPath = Join-Path $artifactRoot "cluster-daemon-service-audit.stdout.json" - $clusterServiceId = $ServiceId - if ($clusterServiceId -eq "epiphany-daemon-supervisor-service") { - $clusterServiceId = "epiphany-cluster-daemon-services" - } - $auditArgs = @( - "cluster-service-audit", - "--store", $localVerseStore, - "--runtime-id", $LocalVerseRuntimeId, - "--daemon-id", $DaemonId, - "--scheduler-id", $SchedulerId, - "--service-id", $clusterServiceId - ) - if ($ServiceName -ne "") { - $auditArgs += @("--service-name", $ServiceName) - } - if ($ServiceDisplayName -ne "") { - $auditArgs += @("--service-display-name", $ServiceDisplayName) - } - Invoke-Checked ` - -Label "audit cluster daemon Windows services" ` - -FilePath $daemonSupervisorExe ` - -Arguments $auditArgs ` - -WorkingDirectory $Root ` - -StdoutPath $resultPath ` - -StderrPath (Join-Path $artifactRoot "cluster-daemon-service-audit.stderr.log") -} - -if ($Mode -eq "cluster-service-start-plan" -or $Mode -eq "cluster-service-stop-plan" -or $Mode -eq "cluster-service-start-execute" -or $Mode -eq "cluster-service-stop-execute") { - $control = if ($Mode -eq "cluster-service-start-plan" -or $Mode -eq "cluster-service-start-execute") { "start" } else { "stop" } - $controlModeName = if ($Mode -eq "cluster-service-start-execute" -or $Mode -eq "cluster-service-stop-execute") { "execute" } else { "plan" } - $resultPath = Join-Path $artifactRoot "cluster-daemon-service-$control-$controlModeName.stdout.json" - $clusterServiceId = $ServiceId - if ($clusterServiceId -eq "epiphany-daemon-supervisor-service") { - $clusterServiceId = "epiphany-cluster-daemon-services" - } - $controlArgs = @( - "cluster-service-$control", - "--store", $localVerseStore, - "--runtime-id", $LocalVerseRuntimeId, - "--daemon-id", $DaemonId, - "--scheduler-id", $SchedulerId, - "--service-id", $clusterServiceId - ) - if ($ServiceName -ne "") { - $controlArgs += @("--service-name", $ServiceName) - } - if ($ServiceDisplayName -ne "") { - $controlArgs += @("--service-display-name", $ServiceDisplayName) - } - if ($controlModeName -eq "execute") { - $controlArgs += @("--execute-control") - } - Invoke-Checked ` - -Label "$controlModeName cluster daemon Windows service $control" ` - -FilePath $daemonSupervisorExe ` - -Arguments $controlArgs ` - -WorkingDirectory $Root ` - -StdoutPath $resultPath ` - -StderrPath (Join-Path $artifactRoot "cluster-daemon-service-$control-$controlModeName.stderr.log") -} - -if ($Mode -eq "cluster-service-execution-runbook") { - $resultPath = Join-Path $artifactRoot "cluster-daemon-service-execution-runbook.stdout.json" - $executionRunbookPath = Join-Path $artifactRoot "epiphany-cluster-daemon-services-execution-runbook.ps1" - $clusterServiceId = $ServiceId - if ($clusterServiceId -eq "epiphany-daemon-supervisor-service") { - $clusterServiceId = "epiphany-cluster-daemon-services" - } - $wrapperScriptPath = $PSCommandPath - if ($wrapperScriptPath -eq "") { - $wrapperScriptPath = Join-Path $Root "tools\epiphany_local_run.ps1" - } - $wrapperScriptPath = (Resolve-Path $wrapperScriptPath).Path - - $baseWrapperArgs = @( - "-SkipBuild", - "-Root", $Root, - "-Workspace", $Workspace, - "-CodexHome", $CodexHome, - "-TargetDir", $TargetDir, - "-DaemonId", $DaemonId, - "-ServiceId", $clusterServiceId, - "-ServiceStartType", $ServiceStartType, - "-SchedulerId", $SchedulerId, - "-LoopIntervalSeconds", "$LoopIntervalSeconds", - "-ServiceMaxIterations", "$ServiceMaxIterations" - ) - if ($ServiceName -ne "") { - $baseWrapperArgs += @("-ServiceName", $ServiceName) - } - if ($ServiceDisplayName -ne "") { - $baseWrapperArgs += @("-ServiceDisplayName", $ServiceDisplayName) - } - - $clusterExecutionRunbookTryModes = @( - "cluster-service-execution-readiness", - "cluster-service-install-execute", - "cluster-service-start-execute", - "cluster-service-audit", - "cluster-service-stop-execute", - "cluster-service-audit" - ) - $clusterExecutionRunbookFinalAuditMode = "cluster-service-execution-audit" - - function New-ClusterRunbookArgs { - param([string]$RunbookMode) - - return @("-Mode", $RunbookMode) + $baseWrapperArgs - } - - function New-RunbookStepLine { - param([string]$RunbookMode) - - return "Invoke-EpiphanyRunbookStep -Name " + (ConvertTo-PowerShellLiteral $RunbookMode) + " -ScriptPath " + (ConvertTo-PowerShellLiteral $wrapperScriptPath) + " -Arguments " + (ConvertTo-PowerShellArrayLiteral (New-ClusterRunbookArgs $RunbookMode)) - } - - $runbookLines = @( - "#Requires -RunAsAdministrator", - "Set-StrictMode -Version Latest", - '$ErrorActionPreference = "Stop"', - '$epiphanyRunbookFailures = New-Object System.Collections.Generic.List[string]', - "", - "# Elevated Epiphany cluster daemon service execution rite.", - "# Run only under explicit operator authority. Each command writes typed operator and lifecycle receipts into the local Verse store.", - "# Individual steps continue after failure so the local Verse gets the fullest possible receipt trail.", - "# ServiceId: $clusterServiceId", - "# ServiceName prefix: $ServiceName", - "", - "function Invoke-EpiphanyRunbookStep {", - " param(", - " [Parameter(Mandatory = `$true)][string]`$Name,", - " [Parameter(Mandatory = `$true)][string]`$ScriptPath,", - " [Parameter(Mandatory = `$true)][string[]]`$Arguments", - " )", - " Write-Host `"==> `$Name`"", - " try {", - " `$global:LASTEXITCODE = 0", - " & `$ScriptPath @Arguments", - " if (`$null -ne `$global:LASTEXITCODE -and `$global:LASTEXITCODE -ne 0) {", - " throw `"Command exited with code `$global:LASTEXITCODE`"", - " }", - " } catch {", - " `$epiphanyRunbookFailures.Add(`"`${Name}: `$(`$_.Exception.Message)`")", - " Write-Warning `"Epiphany runbook step failed: `${Name}: `$(`$_.Exception.Message)`"", - " }", - "}", - "", - "try {" - ) + ($clusterExecutionRunbookTryModes | ForEach-Object { - " " + (New-RunbookStepLine $_) - }) + @( - "} finally {", - " " + (New-RunbookStepLine $clusterExecutionRunbookFinalAuditMode), - "}", - 'if ($epiphanyRunbookFailures.Count -gt 0) {', - ' Write-Error ("Epiphany runbook completed with {0} failed step(s): {1}" -f $epiphanyRunbookFailures.Count, ($epiphanyRunbookFailures -join "; "))', - " exit 1", - "}" - ) - Set-Content -LiteralPath $executionRunbookPath -Value ($runbookLines -join [Environment]::NewLine) -Encoding UTF8 - - $runbookReceiptArgs = @( - "cluster-service-execution-runbook", - "--store", $localVerseStore, - "--runtime-id", $LocalVerseRuntimeId, - "--daemon-id", $DaemonId, - "--scheduler-id", $SchedulerId, - "--service-id", $clusterServiceId, - "--service-command", $wrapperScriptPath, - "--runbook-path", $executionRunbookPath, - "--artifact-ref", $executionRunbookPath, - "--reason", "Generated elevated cluster service execution runbook with final audit in finally." - ) - Invoke-Checked ` - -Label "record cluster daemon Windows service execution runbook receipt" ` - -FilePath $daemonSupervisorExe ` - -Arguments $runbookReceiptArgs ` - -WorkingDirectory $Root ` - -StdoutPath $resultPath ` - -StderrPath (Join-Path $artifactRoot "cluster-daemon-service-execution-runbook.stderr.log") -} - -if ($Mode -eq "cluster-service-execution-readiness") { - $resultPath = Join-Path $artifactRoot "cluster-daemon-service-execution-readiness.stdout.json" - $clusterServiceId = $ServiceId - if ($clusterServiceId -eq "epiphany-daemon-supervisor-service") { - $clusterServiceId = "epiphany-cluster-daemon-services" - } - $readinessArgs = @( - "cluster-service-execution-readiness", - "--store", $localVerseStore, - "--runtime-id", $LocalVerseRuntimeId, - "--daemon-id", $DaemonId, - "--scheduler-id", $SchedulerId, - "--service-id", $clusterServiceId - ) - if ($ServiceName -ne "") { - $readinessArgs += @("--service-name", $ServiceName) - } - if ($ServiceDisplayName -ne "") { - $readinessArgs += @("--service-display-name", $ServiceDisplayName) - } - Invoke-Checked ` - -Label "check cluster daemon Windows service execution readiness" ` - -FilePath $daemonSupervisorExe ` - -Arguments $readinessArgs ` - -WorkingDirectory $Root ` - -StdoutPath $resultPath ` - -StderrPath (Join-Path $artifactRoot "cluster-daemon-service-execution-readiness.stderr.log") -} - -if ($Mode -eq "cluster-service-execution-audit") { - $resultPath = Join-Path $artifactRoot "cluster-daemon-service-execution-audit.stdout.json" - $clusterServiceId = $ServiceId - if ($clusterServiceId -eq "epiphany-daemon-supervisor-service") { - $clusterServiceId = "epiphany-cluster-daemon-services" - } - $auditArgs = @( - "cluster-service-execution-audit", - "--store", $localVerseStore, - "--runtime-id", $LocalVerseRuntimeId, - "--daemon-id", $DaemonId, - "--scheduler-id", $SchedulerId, - "--service-id", $clusterServiceId - ) - Invoke-Checked ` - -Label "audit cluster daemon Windows service execution receipts" ` - -FilePath $daemonSupervisorExe ` - -Arguments $auditArgs ` - -WorkingDirectory $Root ` - -StdoutPath $resultPath ` - -StderrPath (Join-Path $artifactRoot "cluster-daemon-service-execution-audit.stderr.log") -} - -if ($Mode -eq "service-execution-runbook") { - $resultPath = Join-Path $artifactRoot "daemon-service-execution-runbook.stdout.json" - $executionRunbookPath = Join-Path $artifactRoot "epiphany-daemon-supervisor-execution-runbook.ps1" - $wrapperScriptPath = $PSCommandPath - if ($wrapperScriptPath -eq "") { - $wrapperScriptPath = Join-Path $Root "tools\epiphany_local_run.ps1" - } - $wrapperScriptPath = (Resolve-Path $wrapperScriptPath).Path - - $baseWrapperArgs = @( - "-SkipBuild", - "-Root", $Root, - "-Workspace", $Workspace, - "-CodexHome", $CodexHome, - "-TargetDir", $TargetDir, - "-DaemonId", $DaemonId, - "-ServiceId", $ServiceId, - "-ServiceStartType", $ServiceStartType, - "-SchedulerId", $SchedulerId, - "-LoopIntervalSeconds", "$LoopIntervalSeconds", - "-ServiceMaxIterations", "$ServiceMaxIterations" - ) - if ($ServiceName -ne "") { - $baseWrapperArgs += @("-ServiceName", $ServiceName) - } - if ($ServiceDisplayName -ne "") { - $baseWrapperArgs += @("-ServiceDisplayName", $ServiceDisplayName) - } - - function New-ServiceRunbookArgs { - param([string]$RunbookMode) - - return @("-Mode", $RunbookMode) + $baseWrapperArgs - } - - function New-ServiceRunbookStepLine { - param([string]$RunbookMode) - - return "Invoke-EpiphanyRunbookStep -Name " + (ConvertTo-PowerShellLiteral $RunbookMode) + " -ScriptPath " + (ConvertTo-PowerShellLiteral $wrapperScriptPath) + " -Arguments " + (ConvertTo-PowerShellArrayLiteral (New-ServiceRunbookArgs $RunbookMode)) - } - - $runbookLines = @( - "#Requires -RunAsAdministrator", - "Set-StrictMode -Version Latest", - '$ErrorActionPreference = "Stop"', - '$epiphanyRunbookFailures = New-Object System.Collections.Generic.List[string]', - "", - "# Elevated Epiphany daemon service execution rite.", - "# Run only under explicit operator authority. Each command writes typed operator and lifecycle receipts into the local Verse store.", - "# Individual steps continue after failure so the local Verse gets the fullest possible receipt trail.", - "# ServiceId: $ServiceId", - "# ServiceName: $ServiceName", - "", - "function Invoke-EpiphanyRunbookStep {", - " param(", - " [Parameter(Mandatory = `$true)][string]`$Name,", - " [Parameter(Mandatory = `$true)][string]`$ScriptPath,", - " [Parameter(Mandatory = `$true)][string[]]`$Arguments", - " )", - " Write-Host `"==> `$Name`"", - " try {", - " `$global:LASTEXITCODE = 0", - " & `$ScriptPath @Arguments", - " if (`$null -ne `$global:LASTEXITCODE -and `$global:LASTEXITCODE -ne 0) {", - " throw `"Command exited with code `$global:LASTEXITCODE`"", - " }", - " } catch {", - " `$epiphanyRunbookFailures.Add(`"`${Name}: `$(`$_.Exception.Message)`")", - " Write-Warning `"Epiphany runbook step failed: `${Name}: `$(`$_.Exception.Message)`"", - " }", - "}", - "", - "try {", - " " + (New-ServiceRunbookStepLine "service-execution-readiness"), - " " + (New-ServiceRunbookStepLine "service-install-execute"), - " " + (New-ServiceRunbookStepLine "service-start-execute"), - " " + (New-ServiceRunbookStepLine "service-status"), - " " + (New-ServiceRunbookStepLine "service-reconcile"), - " " + (New-ServiceRunbookStepLine "service-stop-execute"), - " " + (New-ServiceRunbookStepLine "service-status"), - "} finally {", - " " + (New-ServiceRunbookStepLine "service-execution-audit"), - "}", - 'if ($epiphanyRunbookFailures.Count -gt 0) {', - ' Write-Error ("Epiphany runbook completed with {0} failed step(s): {1}" -f $epiphanyRunbookFailures.Count, ($epiphanyRunbookFailures -join "; "))', - " exit 1", - "}" - ) - Set-Content -LiteralPath $executionRunbookPath -Value ($runbookLines -join [Environment]::NewLine) -Encoding UTF8 - - $runbookReceiptArgs = @( - "service-execution-runbook", - "--store", $localVerseStore, - "--runtime-id", $LocalVerseRuntimeId, - "--daemon-id", $DaemonId, - "--scheduler-id", $SchedulerId, - "--service-id", $ServiceId, - "--service-command", $wrapperScriptPath, - "--runbook-path", $executionRunbookPath, - "--artifact-ref", $executionRunbookPath, - "--reason", "Generated elevated daemon service execution runbook." - ) - if ($ServiceName -ne "") { - $runbookReceiptArgs += @("--service-name", $ServiceName) - } - if ($ServiceDisplayName -ne "") { - $runbookReceiptArgs += @("--service-display-name", $ServiceDisplayName) - } - Invoke-Checked ` - -Label "record daemon Windows service execution runbook receipt" ` - -FilePath $daemonSupervisorExe ` - -Arguments $runbookReceiptArgs ` - -WorkingDirectory $Root ` - -StdoutPath $resultPath ` - -StderrPath (Join-Path $artifactRoot "daemon-service-execution-runbook.stderr.log") -} - -if ($Mode -eq "service-install-plan" -or $Mode -eq "service-install-execute") { - $installModeName = if ($Mode -eq "service-install-execute") { "execute" } else { "plan" } - $resultPath = Join-Path $artifactRoot "daemon-service-install-$installModeName.stdout.json" - $installScriptPath = Join-Path $artifactRoot "epiphany-daemon-supervisor-install.ps1" - $installArgs = @( - "service-install-$installModeName", - "--store", $localVerseStore, - "--runtime-id", $LocalVerseRuntimeId, - "--daemon-id", $DaemonId, - "--scheduler-id", $SchedulerId, - "--service-id", $ServiceId, - "--service-start-type", $ServiceStartType, - "--loop-interval-seconds", "$LoopIntervalSeconds", - "--service-install-script-path", $installScriptPath - ) - if ($ServiceName -ne "") { - $installArgs += @("--service-name", $ServiceName) - } - if ($ServiceDisplayName -ne "") { - $installArgs += @("--service-display-name", $ServiceDisplayName) - } - if ($ServiceMaxIterations -gt 0) { - $installArgs += @("--max-iterations", "$ServiceMaxIterations") - } - Invoke-Checked ` - -Label "write daemon supervisor Windows service install $installModeName" ` - -FilePath $daemonSupervisorExe ` - -Arguments $installArgs ` - -WorkingDirectory $Root ` - -StdoutPath $resultPath ` - -StderrPath (Join-Path $artifactRoot "daemon-service-install-$installModeName.stderr.log") -} - -if ($Mode -eq "service-status") { - $resultPath = Join-Path $artifactRoot "daemon-service-status.stdout.json" - $statusArgs = @( - "windows-service-status", - "--store", $localVerseStore, - "--runtime-id", $LocalVerseRuntimeId, - "--daemon-id", $DaemonId, - "--scheduler-id", $SchedulerId, - "--service-id", $ServiceId - ) - if ($ServiceName -ne "") { - $statusArgs += @("--service-name", $ServiceName) - } - Invoke-Checked ` - -Label "query daemon supervisor Windows service status" ` - -FilePath $daemonSupervisorExe ` - -Arguments $statusArgs ` - -WorkingDirectory $Root ` - -StdoutPath $resultPath ` - -StderrPath (Join-Path $artifactRoot "daemon-service-status.stderr.log") -} - if ($Mode -eq "service-tick") { $resultPath = Join-Path $artifactRoot "daemon-service-tick.stdout.json" Invoke-Checked ` @@ -1973,100 +1221,6 @@ if ($Mode -eq "service-tick") { -StderrPath (Join-Path $artifactRoot "daemon-service-tick.stderr.log") } -if ($Mode -eq "service-reconcile") { - $resultPath = Join-Path $artifactRoot "daemon-service-reconcile.stdout.json" - $reconcileArgs = @( - "windows-service-reconcile", - "--store", $localVerseStore, - "--runtime-id", $LocalVerseRuntimeId, - "--daemon-id", $DaemonId, - "--scheduler-id", $SchedulerId, - "--service-id", $ServiceId, - "--service-start-type", $ServiceStartType - ) - if ($ServiceName -ne "") { - $reconcileArgs += @("--service-name", $ServiceName) - } - Invoke-Checked ` - -Label "reconcile daemon supervisor Windows service policy" ` - -FilePath $daemonSupervisorExe ` - -Arguments $reconcileArgs ` - -WorkingDirectory $Root ` - -StdoutPath $resultPath ` - -StderrPath (Join-Path $artifactRoot "daemon-service-reconcile.stderr.log") -} - -if ($Mode -eq "service-execution-readiness") { - $resultPath = Join-Path $artifactRoot "daemon-service-execution-readiness.stdout.json" - $readinessArgs = @( - "windows-service-execution-readiness", - "--store", $localVerseStore, - "--runtime-id", $LocalVerseRuntimeId, - "--daemon-id", $DaemonId, - "--scheduler-id", $SchedulerId, - "--service-id", $ServiceId - ) - if ($ServiceName -ne "") { - $readinessArgs += @("--service-name", $ServiceName) - } - Invoke-Checked ` - -Label "check daemon supervisor Windows service execution readiness" ` - -FilePath $daemonSupervisorExe ` - -Arguments $readinessArgs ` - -WorkingDirectory $Root ` - -StdoutPath $resultPath ` - -StderrPath (Join-Path $artifactRoot "daemon-service-execution-readiness.stderr.log") -} - -if ($Mode -eq "service-execution-audit") { - $resultPath = Join-Path $artifactRoot "daemon-service-execution-audit.stdout.json" - $auditArgs = @( - "windows-service-execution-audit", - "--store", $localVerseStore, - "--runtime-id", $LocalVerseRuntimeId, - "--daemon-id", $DaemonId, - "--scheduler-id", $SchedulerId, - "--service-id", $ServiceId - ) - if ($ServiceName -ne "") { - $auditArgs += @("--service-name", $ServiceName) - } - Invoke-Checked ` - -Label "audit daemon supervisor Windows service execution receipts" ` - -FilePath $daemonSupervisorExe ` - -Arguments $auditArgs ` - -WorkingDirectory $Root ` - -StdoutPath $resultPath ` - -StderrPath (Join-Path $artifactRoot "daemon-service-execution-audit.stderr.log") -} - -if ($Mode -eq "service-start-plan" -or $Mode -eq "service-stop-plan" -or $Mode -eq "service-start-execute" -or $Mode -eq "service-stop-execute") { - $control = if ($Mode -eq "service-start-plan" -or $Mode -eq "service-start-execute") { "start" } else { "stop" } - $controlModeName = if ($Mode -eq "service-start-execute" -or $Mode -eq "service-stop-execute") { "execute" } else { "plan" } - $resultPath = Join-Path $artifactRoot "daemon-service-$control-$controlModeName.stdout.json" - $controlArgs = @( - "windows-service-$control", - "--store", $localVerseStore, - "--runtime-id", $LocalVerseRuntimeId, - "--daemon-id", $DaemonId, - "--scheduler-id", $SchedulerId, - "--service-id", $ServiceId - ) - if ($ServiceName -ne "") { - $controlArgs += @("--service-name", $ServiceName) - } - if ($controlModeName -eq "execute") { - $controlArgs += @("--execute-control") - } - Invoke-Checked ` - -Label "$controlModeName daemon supervisor Windows service $control" ` - -FilePath $daemonSupervisorExe ` - -Arguments $controlArgs ` - -WorkingDirectory $Root ` - -StdoutPath $resultPath ` - -StderrPath (Join-Path $artifactRoot "daemon-service-$control-$controlModeName.stderr.log") -} - if ($Mode -eq "repo-persona-intake") { $resultPath = Join-Path $artifactRoot "repo-persona-intake.stdout.json" $repoWorkVerseStore = Join-Path $Workspace ".epiphany\local-verse.ccmp" @@ -2143,6 +1297,31 @@ if ($Mode -eq "repo-work-queue-run") { -StderrPath (Join-Path $artifactRoot "repo-work-queue-run.stderr.log") } +if (@("managed-service-task-plan", "managed-service-task-install", "managed-service-task-status", "managed-service-task-start", "managed-service-task-stop", "managed-service-task-uninstall") -contains $Mode) { + $taskCommand = $Mode + $resultPath = Join-Path $artifactRoot "$Mode.stdout.json" + $taskArgs = @( + $taskCommand, + "--store", $localVerseStore, + "--runtime-id", $LocalVerseRuntimeId, + "--service-id", $ServiceId, + "--service-name", $ServiceName, + "--service-command", $daemonSupervisorExe, + "--cwd", $Root, + "--loop-interval-seconds", "$LoopIntervalSeconds" + ) + if ($ServiceMaxIterations -gt 0) { + $taskArgs += @("--max-iterations", "$ServiceMaxIterations") + } + Invoke-Checked ` + -Label "$Mode Idunn managed-service scheduled task" ` + -FilePath $daemonSupervisorExe ` + -Arguments $taskArgs ` + -WorkingDirectory $Root ` + -StdoutPath $resultPath ` + -StderrPath (Join-Path $artifactRoot "$Mode.stderr.log") +} + if ($Mode -eq "repo-work-public-proof") { $resultPath = Join-Path $artifactRoot "repo-work-public-proof.stdout.json" $repoWorkVerseStore = Join-Path $Workspace ".epiphany\local-verse.ccmp" @@ -3012,20 +2191,6 @@ if ($resultPath -ne "" -and (Test-Path -LiteralPath $resultPath)) { $failed = (@($result.failedChecks) -join ",") } Write-Host "Repo work service audit: service=$($result.serviceId), status=$($result.status), receipt=$($result.receiptId), plan=$($result.planStatus), runbook=$($result.runbookStatus), runbookArtifact=$($result.runbookArtifactStatus), launch=$($result.launchStatus), exitCode=$($result.launchExitCode), missing=$missing, failed=$failed, owner=Idunn, hostedBody=repo-work, serviceManagerMutation=false, nextSafeMove=$($result.nextSafeMove), privateStateExposed=$($result.privateStateExposed)" - } elseif ($Mode -eq "swarm-online-runbook") { - $aftercare = "none" - if ($null -ne $result.aftercareCommands -and $result.aftercareCommands.Count -gt 0) { - $aftercare = ($result.aftercareCommands -join "; ") - } - $childRows = 0 - if ($null -ne $result.childRunbookRows) { - $childRows = @($result.childRunbookRows).Count - } - $aftercareRows = 0 - if ($null -ne $result.aftercareRows) { - $aftercareRows = @($result.aftercareRows).Count - } - Write-Host "Swarm online runbook: status=$($result.status), preflight=$($result.preflightStatus), owner=$($result.lifecycleOwner), hostedBody=$($result.hostedBody), commands=$($result.commandCount), childRows=$childRows, present=$($result.childArtifactPresentCount), hashVerified=$($result.childArtifactHashVerifiedCount), hashMismatch=$($result.childArtifactMismatchCount), missingArtifacts=$($result.childArtifactMissingCount), failedChecks=$($result.serviceExecutionFailedCheckCount), missingChecks=$($result.serviceExecutionMissingCheckCount), aftercareRows=$aftercareRows, artifactSha256=$($result.artifactSha256), elevatedCommand=$($result.elevatedCommand), verifiesChildArtifactSha256=$($result.verifiesChildArtifactSha256), usesExplicitAftercareArguments=$($result.usesExplicitAftercareArguments), aftercare=$aftercare, path=$($result.runbookPath), privateStateExposed=$($result.privateStateExposed)" } elseif ($Mode -eq "service-policy-directory") { $policyRows = "none" if ($null -ne $result.tuiRows -and $result.tuiRows.Count -gt 0) { @@ -3037,9 +2202,9 @@ if ($resultPath -ne "" -and (Test-Path -LiteralPath $resultPath)) { if ($null -ne $result.args -and $result.args.Count -gt 0) { $plannedArgs = ($result.args -join " ") } - Write-Host "Service plan: service=$($result.serviceId), status=$($result.status), receipt=$($result.receiptId), command=$($result.command), args=$plannedArgs, followUp=tools/epiphany_local_run.ps1 -Mode service-runbook, aftercare=tools/epiphany_local_run.ps1 -Mode service-status, privateStateExposed=$($result.privateStateExposed)" + Write-Host "Service plan: service=$($result.serviceId), status=$($result.status), receipt=$($result.receiptId), command=$($result.command), args=$plannedArgs, followUp=tools/epiphany_local_run.ps1 -Mode service-runbook, aftercare=tools/epiphany_local_run.ps1 -Mode managed-service-task-status, privateStateExposed=$($result.privateStateExposed)" } elseif ($Mode -eq "service-launch") { - Write-Host "Service launch: service=$($result.serviceId), status=$($result.status), processId=$($result.processId), exitCode=$($result.exitCode), receipt=$($result.receiptId), followUp=tools/epiphany_local_run.ps1 -Mode service-status, aftercare=tools/epiphany_local_run.ps1 -Mode service-tick, privateStateExposed=$($result.privateStateExposed)" + Write-Host "Service launch: service=$($result.serviceId), status=$($result.status), processId=$($result.processId), exitCode=$($result.exitCode), receipt=$($result.receiptId), followUp=tools/epiphany_local_run.ps1 -Mode managed-service-task-plan, aftercare=tools/epiphany_local_run.ps1 -Mode service-tick, privateStateExposed=$($result.privateStateExposed)" } elseif ($Mode -eq "swarm-poke-down") { $pokeRows = "none" if ($null -ne $result.pokeRows -and $result.pokeRows.Count -gt 0) { @@ -3143,114 +2308,11 @@ if ($resultPath -ne "" -and (Test-Path -LiteralPath $resultPath)) { Write-Host "Swarm triage: status=$($result.status), overview=$($result.overviewStatus), liveness=$($result.livenessStatus), recovery=$($result.recoveryStatus), clusters=$($result.clusterCount), privateVerses=$($result.privateVerseCount), recommended=$($result.recommendedWrapperMode), serviceRecommended=$($result.serviceLifecycleRecommendedWrapperMode), actionQueue=$actionQueue, actionRows=$actionRows, attention=$attention, daemonRows=$daemonRows, repoWorkMapRows=$repoWorkMapRows, repoWorkSemanticRows=$repoWorkSemanticRows, repoWorkFamilyLensRows=$repoWorkFamilyLensRows, repoWorkPathLensRows=$repoWorkPathLensRows, repoWorkBranchLensRows=$repoWorkBranchLensRows, repoWorkStageLensRows=$repoWorkStageLensRows, repoWorkGateLensRows=$repoWorkGateLensRows, repoWorkClosureRows=$repoWorkClosureRows, repoWorkAcceptanceRows=$repoWorkAcceptanceRows, repoWorkOverviewRows=$repoWorkOverviewRows, repoWorkPublicProofRows=$repoWorkPublicProofRows, repoWorkReadinessRows=$repoWorkReadinessRows, repoWorkReadinessReviewRows=$repoWorkReadinessReviewRows, toolHostAttention=$toolHostAttention, toolAttentionRows=$toolAttentionRows, serviceLifecycleAttention=$serviceLifecycleAttention, serviceAttentionRows=$serviceAttentionRows, serviceExecutionFailedChecks=$serviceExecutionFailedChecks, serviceFailedCheckRows=$serviceFailedCheckRows, poked=$($result.pokedDaemonCount), privateStateExposed=$($result.privateStateExposed)" } elseif ($Mode -eq "service-runbook") { $artifactSha256 = Get-LocalArtifactSha256 $result.runbookPath - Write-Host "Service runbook: service=$($result.serviceId), receipt=$($result.receiptId), artifactSha256=$artifactSha256, followUp=tools/epiphany_local_run.ps1 -Mode service-launch, aftercare=tools/epiphany_local_run.ps1 -Mode service-status, path=$($result.runbookPath)" - } elseif ($Mode -eq "cluster-service-runbook") { - $artifactSha256 = Get-LocalArtifactSha256 $result.runbookPath - Write-Host "Cluster daemon service runbook: service=$($result.serviceId), daemons=$($result.daemonCount), receipt=$($result.receiptId), artifactSha256=$artifactSha256, followUp=tools/epiphany_local_run.ps1 -Mode service-tick, aftercare=tools/epiphany_local_run.ps1 -Mode swarm-overview, path=$($result.runbookPath)" - } elseif ($Mode -eq "cluster-service-install-plan" -or $Mode -eq "cluster-service-install-execute") { - $serviceRows = Format-ClusterServiceRows $result.services - $artifactSha256 = Get-LocalArtifactSha256 $result.installScriptPath - Write-Host "Cluster daemon service install: service=$($result.serviceId), daemons=$($result.daemonCount), status=$($result.status), executed=$($result.executed), receipt=$($result.receiptId), artifactSha256=$artifactSha256, followUp=tools/epiphany_local_run.ps1 -Mode cluster-service-install-execute, serviceRows=$serviceRows, path=$($result.installScriptPath)" - } elseif ($Mode -eq "cluster-service-audit") { - $serviceRows = Format-ClusterServiceRows $result.services - Write-Host "Cluster daemon service audit: service=$($result.serviceId), daemons=$($result.daemonCount), status=$($result.status), missing=$($result.missingCount), running=$($result.runningCount), present=$($result.presentCount), queryFailed=$($result.queryFailedCount), receipt=$($result.receiptId), followUp=tools/epiphany_local_run.ps1 -Mode cluster-service-install-plan, aftercare=tools/epiphany_local_run.ps1 -Mode cluster-service-execution-audit, serviceRows=$serviceRows" - } elseif ($Mode -eq "cluster-service-start-plan" -or $Mode -eq "cluster-service-stop-plan" -or $Mode -eq "cluster-service-start-execute" -or $Mode -eq "cluster-service-stop-execute") { - $serviceRows = Format-ClusterServiceRows $result.services - Write-Host "Cluster daemon service control: service=$($result.serviceId), daemons=$($result.daemonCount), status=$($result.status), executeRequested=$($result.executeRequested), executed=$($result.executed), planned=$($result.plannedCount), requested=$($result.requestedCount), refused=$($result.refusedCount), failed=$($result.failedCount), receipt=$($result.receiptId), followUp=tools/epiphany_local_run.ps1 -Mode cluster-service-audit, aftercare=tools/epiphany_local_run.ps1 -Mode cluster-service-execution-audit, serviceRows=$serviceRows" - } elseif ($Mode -eq "cluster-service-execution-readiness") { - $serviceRows = Format-ClusterServiceRows $result.services $result.status - Write-Host "Cluster daemon service execution readiness: service=$($result.serviceId), daemons=$($result.daemonCount), services=$($result.serviceCount), status=$($result.status), elevated=$($result.elevated), receipt=$($result.receiptId), followUp=tools/epiphany_local_run.ps1 -Mode cluster-service-execution-runbook, aftercare=tools/epiphany_local_run.ps1 -Mode cluster-service-execution-audit, serviceRows=$serviceRows" - } elseif ($Mode -eq "cluster-service-execution-runbook") { - $elevatedCommand = Get-ElevatedRunbookCommand $result.runbookPath - $artifactSha256 = Get-LocalArtifactSha256 $result.runbookPath - Write-Host "Cluster daemon service execution runbook: service=$($result.serviceId), status=$($result.status), finalAuditInFinally=$($result.finalAuditRunsInFinally), continueAfterStepFailure=$($result.continueAfterStepFailure), nonzeroExitFailsStep=$($result.nonzeroExitFailsStep), exitsNonzeroAfterFinalAudit=$($result.exitsNonzeroAfterFinalAudit), artifactSha256=$artifactSha256, elevatedCommand=$elevatedCommand, aftercare=tools/epiphany_local_run.ps1 -Mode cluster-service-execution-audit, path=$($result.runbookPath)" - } elseif ($Mode -eq "cluster-service-execution-audit") { - Write-Host "Cluster daemon service execution audit: service=$($result.serviceId), status=$($result.status), missing=$($result.missingCount), failed=$($result.failedCount), receipt=$($result.receiptId), runbookSha256=$($result.runbookSha256), elevatedCommand=$($result.elevatedCommand), aftercare=$($result.aftercareCommand), requiresElevatedAuthority=$($result.requiresElevatedAuthority)" - $failedCheckRows = Format-ServiceExecutionFailedChecks @($result.checks | Where-Object { -not $_.ok }) - $runbookWitnessChecks = @($result.checks | Where-Object { $_.ok -and $null -ne $_.operatorArtifactRef -and $_.operatorArtifactRef -ne "" -and $_.operatorArtifactRef -ne "none" }) - if ($runbookWitnessChecks.Count -gt 0) { - $runbookWitnessSummary = ($runbookWitnessChecks | ForEach-Object { - $artifact = $_.operatorArtifactRef - $artifactSha256 = "none" - if (Test-Path -LiteralPath $artifact -PathType Leaf) { - $artifactSha256 = (Get-FileHash -LiteralPath $artifact -Algorithm SHA256).Hash.ToLowerInvariant() - } - $serviceId = $_.serviceId - if ($null -eq $serviceId -or $serviceId -eq "") { - $serviceId = "unknown-service" - } - "${serviceId}::$($_.action)=${artifact}:sha256=${artifactSha256}" - }) -join "; " - Write-Host "Cluster daemon service execution runbook witnesses: $runbookWitnessSummary" - } - $failedChecks = @($result.checks | Where-Object { -not $_.ok }) - if ($failedChecks.Count -gt 0) { - $failedSummary = ($failedChecks | ForEach-Object { - $observed = $_.observedStatus - if ($null -eq $observed -or $observed -eq "") { - $observed = "missing" - } - $serviceId = $_.serviceId - if ($null -eq $serviceId -or $serviceId -eq "") { - $serviceId = "unknown-service" - } - "${serviceId}::$($_.action)=$observed" - }) -join "; " - Write-Host "Cluster daemon service execution failed checks: $failedSummary" - } - Write-Host "Cluster daemon service execution failed check rows: $failedCheckRows" - } elseif ($Mode -eq "service-execution-runbook") { - $elevatedCommand = Get-ElevatedRunbookCommand $result.runbookPath - $artifactSha256 = Get-LocalArtifactSha256 $result.runbookPath - Write-Host "Service execution runbook: service=$($result.serviceId), name=$($result.serviceName), status=$($result.status), finalAuditInFinally=$($result.finalAuditRunsInFinally), continueAfterStepFailure=$($result.continueAfterStepFailure), nonzeroExitFailsStep=$($result.nonzeroExitFailsStep), exitsNonzeroAfterFinalAudit=$($result.exitsNonzeroAfterFinalAudit), artifactSha256=$artifactSha256, elevatedCommand=$elevatedCommand, aftercare=tools/epiphany_local_run.ps1 -Mode service-execution-audit, path=$($result.runbookPath)" - } elseif ($Mode -eq "service-install-plan" -or $Mode -eq "service-install-execute") { - $artifactSha256 = Get-LocalArtifactSha256 $result.installScriptPath - Write-Host "Service install: service=$($result.serviceId), name=$($result.serviceName), status=$($result.status), executed=$($result.executed), receipt=$($result.receiptId), artifactSha256=$artifactSha256, followUp=tools/epiphany_local_run.ps1 -Mode service-install-execute, path=$($result.installScriptPath)" - } elseif ($Mode -eq "service-status") { - Write-Host "Service status: service=$($result.serviceId), name=$($result.serviceName), status=$($result.status), receipt=$($result.receiptId), followUp=tools/epiphany_local_run.ps1 -Mode service-reconcile, aftercare=tools/epiphany_local_run.ps1 -Mode service-execution-audit" + Write-Host "Service runbook: service=$($result.serviceId), receipt=$($result.receiptId), artifactSha256=$artifactSha256, followUp=tools/epiphany_local_run.ps1 -Mode service-launch, aftercare=tools/epiphany_local_run.ps1 -Mode managed-service-task-plan, path=$($result.runbookPath)" } elseif ($Mode -eq "service-tick") { Write-Host "Scheduler tick: scheduler=$($result.schedulerId), daemonSelector=$($result.daemonId), status=$($result.status), outcomes=$($result.outcomeCount), restarted=$($result.restartedCount), refused=$($result.refusedCount), skipped=$($result.skippedCount), receipt=$($result.schedulerReceiptId), privateStateExposed=$($result.privateStateExposed)" - } elseif ($Mode -eq "service-reconcile") { - Write-Host "Service reconcile: service=$($result.serviceId), name=$($result.serviceName), status=$($result.status), receipt=$($result.receiptId), followUp=tools/epiphany_local_run.ps1 -Mode service-install-plan, aftercare=tools/epiphany_local_run.ps1 -Mode service-execution-audit" - } elseif ($Mode -eq "service-execution-readiness") { - Write-Host "Service execution readiness: service=$($result.serviceId), name=$($result.serviceName), status=$($result.status), elevated=$($result.elevated), receipt=$($result.receiptId), followUp=tools/epiphany_local_run.ps1 -Mode service-execution-runbook, aftercare=tools/epiphany_local_run.ps1 -Mode service-execution-audit" - } elseif ($Mode -eq "service-execution-audit") { - Write-Host "Service execution audit: service=$($result.serviceId), name=$($result.serviceName), status=$($result.status), missing=$($result.missingCount), failed=$($result.failedCount), receipt=$($result.receiptId), runbookSha256=$($result.runbookSha256), elevatedCommand=$($result.elevatedCommand), aftercare=$($result.aftercareCommand), requiresElevatedAuthority=$($result.requiresElevatedAuthority)" - $failedCheckRows = Format-ServiceExecutionFailedChecks @($result.checks | Where-Object { -not $_.ok }) - $runbookWitnessChecks = @($result.checks | Where-Object { $_.ok -and $null -ne $_.operatorArtifactRef -and $_.operatorArtifactRef -ne "" -and $_.operatorArtifactRef -ne "none" }) - if ($runbookWitnessChecks.Count -gt 0) { - $runbookWitnessSummary = ($runbookWitnessChecks | ForEach-Object { - $artifact = $_.operatorArtifactRef - $artifactSha256 = "none" - if (Test-Path -LiteralPath $artifact -PathType Leaf) { - $artifactSha256 = (Get-FileHash -LiteralPath $artifact -Algorithm SHA256).Hash.ToLowerInvariant() - } - $serviceId = $_.serviceId - if ($null -eq $serviceId -or $serviceId -eq "") { - $serviceId = "unknown-service" - } - "${serviceId}::$($_.action)=${artifact}:sha256=${artifactSha256}" - }) -join "; " - Write-Host "Service execution runbook witnesses: $runbookWitnessSummary" - } - $failedChecks = @($result.checks | Where-Object { -not $_.ok }) - if ($failedChecks.Count -gt 0) { - $failedSummary = ($failedChecks | ForEach-Object { - $observed = $_.observedStatus - if ($null -eq $observed -or $observed -eq "") { - $observed = "missing" - } - $serviceId = $_.serviceId - if ($null -eq $serviceId -or $serviceId -eq "") { - $serviceId = "unknown-service" - } - "${serviceId}::$($_.action)=$observed" - }) -join "; " - Write-Host "Service execution failed checks: $failedSummary" - } - Write-Host "Service execution failed check rows: $failedCheckRows" - } elseif ($Mode -eq "service-start-plan" -or $Mode -eq "service-stop-plan" -or $Mode -eq "service-start-execute" -or $Mode -eq "service-stop-execute") { - Write-Host "Service control: service=$($result.serviceId), name=$($result.serviceName), status=$($result.status), executeRequested=$($result.executeRequested), executed=$($result.executed), receipt=$($result.receiptId), followUp=tools/epiphany_local_run.ps1 -Mode service-status, aftercare=tools/epiphany_local_run.ps1 -Mode service-execution-audit" + } elseif (@("managed-service-task-plan", "managed-service-task-install", "managed-service-task-status", "managed-service-task-start", "managed-service-task-stop", "managed-service-task-uninstall") -contains $Mode) { + Write-Host "Idunn scheduled task: task=$($result.taskName), status=$($result.status), receipt=$($result.receiptId), exitCode=$($result.exitCode), privateStateExposed=$($result.privateStateExposed)" } else { Write-Host "Coordinator: thread=$($result.threadId), finalAction=$($result.finalAction.action), runtimePresent=$($result.runtimeSpine.present)" }