diff --git a/epiphany-core/Cargo.toml b/epiphany-core/Cargo.toml index 85bb8dcc..2a102a0d 100644 --- a/epiphany-core/Cargo.toml +++ b/epiphany-core/Cargo.toml @@ -37,3 +37,9 @@ pretty_assertions = "1.4.1" tempfile = "3" tokio = { version = "1", features = ["rt", "rt-multi-thread"] } wiremock = "0.6" + +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.59", features = ["Win32_Foundation", "Win32_System_Threading"] } + +[target.'cfg(unix)'.dependencies] +libc = "0.2" diff --git a/epiphany-core/src/bin/epiphany-daemon-supervisor.rs b/epiphany-core/src/bin/epiphany-daemon-supervisor.rs index cc32e968..33c6e499 100644 --- a/epiphany-core/src/bin/epiphany-daemon-supervisor.rs +++ b/epiphany-core/src/bin/epiphany-daemon-supervisor.rs @@ -53,6 +53,9 @@ fn main() -> Result<()> { "service-launch" | "launch-service" | "start-service" => service_launch(args), "managed-service-policy" | "service-desired-state" => managed_service_policy(args), "managed-service-read" | "service-desired-state-read" => managed_service_read(args), + "managed-service-reconcile" | "service-desired-state-reconcile" => { + managed_service_reconcile(args) + } "service-runbook" | "runbook-service" => service_runbook(args), "repo-work-service-audit" | "repo-work-service-readiness" @@ -373,10 +376,6 @@ fn managed_service_policy(args: Args) -> Result<()> { backoff_multiplier: args.backoff_multiplier, stdout_artifact: stdout_artifact.display().to_string(), stderr_artifact: stderr_artifact.display().to_string(), - last_lifecycle_receipt_id: latest - .as_ref() - .map(|receipt| receipt.receipt_id.clone()) - .unwrap_or_default(), updated_at_utc: Utc::now().to_rfc3339(), private_state_exposed: false, notes: vec![ @@ -403,7 +402,7 @@ fn managed_service_policy(args: Args) -> Result<()> { "args": written.args, "stdoutArtifact": written.stdout_artifact, "stderrArtifact": written.stderr_artifact, - "lastLifecycleReceiptId": written.last_lifecycle_receipt_id, + "latestLifecycleReceiptId": latest.as_ref().map(|receipt| receipt.receipt_id.as_str()), "privateStateExposed": written.private_state_exposed, }))?); Ok(()) @@ -423,6 +422,12 @@ fn managed_service_read(args: Args) -> Result<()> { .into_iter() .filter(|receipt| receipt.service_id == args.service_id) .max_by(|left, right| left.started_at_utc.cmp(&right.started_at_utc)); + let process_observation = latest + .as_ref() + .and_then(|receipt| receipt.process_id) + .map(observe_process) + .transpose()? + .unwrap_or(ProcessObservation::Missing); println!("{}", serde_json::to_string_pretty(&json!({ "schemaVersion": "epiphany.cultmesh.managed_service_readback.v0", "status": "desired-state-ready", @@ -445,12 +450,164 @@ fn managed_service_read(args: Args) -> Result<()> { "startedAtUtc": receipt.started_at_utc, "completedAtUtc": receipt.completed_at_utc, })), - "processObservation": "unknown-until-managed-service-reconcile", + "processObservation": process_observation.label(), "privateStateExposed": false, }))?); Ok(()) } +fn managed_service_reconcile(mut args: Args) -> Result<()> { + let policy = load_epiphany_cultmesh_managed_service_policy( + &args.store, + args.runtime_id.clone(), + &args.service_id, + )? + .with_context(|| format!("managed service policy missing for {}", args.service_id))?; + let latest = load_epiphany_cultmesh_daemon_service_lifecycle_receipts( + &args.store, + args.runtime_id.clone(), + )? + .into_iter() + .filter(|receipt| receipt.service_id == args.service_id) + .max_by(|left, right| left.started_at_utc.cmp(&right.started_at_utc)); + let observation = latest + .as_ref() + .and_then(|receipt| receipt.process_id) + .map(observe_process) + .transpose()? + .unwrap_or(ProcessObservation::Missing); + if !policy.enabled || policy.restart_mode == "never" { + println!("{}", serde_json::to_string_pretty(&json!({ + "schemaVersion": "epiphany.cultmesh.managed_service_reconcile.v0", + "status": "disabled", + "serviceId": policy.service_id, + "processObservation": observation.label(), + "restarted": false, + "privateStateExposed": false, + }))?); + return Ok(()); + } + if observation == ProcessObservation::Alive { + println!("{}", serde_json::to_string_pretty(&json!({ + "schemaVersion": "epiphany.cultmesh.managed_service_reconcile.v0", + "status": "observed-alive", + "serviceId": policy.service_id, + "processId": latest.as_ref().and_then(|receipt| receipt.process_id), + "processObservation": observation.label(), + "restarted": false, + "privateStateExposed": false, + }))?); + return Ok(()); + } + if policy.restart_mode == "on-failure" + && latest.as_ref().is_some_and(|receipt| { + receipt.status == "completed" && receipt.exit_code.unwrap_or_default() == 0 + }) + { + println!("{}", serde_json::to_string_pretty(&json!({ + "schemaVersion": "epiphany.cultmesh.managed_service_reconcile.v0", + "status": "completed-no-restart", + "serviceId": policy.service_id, + "processObservation": observation.label(), + "restarted": false, + "privateStateExposed": false, + }))?); + return Ok(()); + } + if !args.force && policy.cooldown_seconds > 0 { + if let Some(started) = latest + .as_ref() + .and_then(|receipt| DateTime::parse_from_rfc3339(&receipt.started_at_utc).ok()) + { + let elapsed = Utc::now().signed_duration_since(started.with_timezone(&Utc)); + if elapsed < Duration::seconds(policy.cooldown_seconds) { + println!("{}", serde_json::to_string_pretty(&json!({ + "schemaVersion": "epiphany.cultmesh.managed_service_reconcile.v0", + "status": "cooling-down", + "serviceId": policy.service_id, + "processObservation": observation.label(), + "cooldownSeconds": policy.cooldown_seconds, + "elapsedSeconds": elapsed.num_seconds(), + "restarted": false, + "privateStateExposed": false, + }))?); + return Ok(()); + } + } + } + args.service_command = Some(PathBuf::from(&policy.command)); + args.service_args = policy.args.clone(); + args.cwd = policy.cwd.as_ref().map(PathBuf::from); + args.stdout_artifact = Some(PathBuf::from(&policy.stdout_artifact)); + args.stderr_artifact = Some(PathBuf::from(&policy.stderr_artifact)); + args.reason = Some(format!( + "Idunn reconciled managed service {} after observing {}.", + policy.service_id, + observation.label() + )); + service_launch(args) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ProcessObservation { + Alive, + Dead, + Missing, +} + +impl ProcessObservation { + fn label(self) -> &'static str { + match self { + Self::Alive => "alive", + Self::Dead => "dead", + Self::Missing => "missing", + } + } +} + +#[cfg(windows)] +fn observe_process(process_id: u32) -> Result { + use windows_sys::Win32::Foundation::CloseHandle; + use windows_sys::Win32::System::Threading::{ + GetExitCodeProcess, OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION, + }; + const STILL_ACTIVE: u32 = 259; + unsafe { + let handle = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, process_id); + if handle.is_null() { + return Ok(ProcessObservation::Dead); + } + let mut exit_code = 0_u32; + let read = GetExitCodeProcess(handle, &mut exit_code); + CloseHandle(handle); + if read == 0 { + anyhow::bail!("failed to inspect managed service process {process_id}"); + } + Ok(if exit_code == STILL_ACTIVE { + ProcessObservation::Alive + } else { + ProcessObservation::Dead + }) + } +} + +#[cfg(unix)] +fn observe_process(process_id: u32) -> Result { + let result = unsafe { libc::kill(process_id as i32, 0) }; + if result == 0 { + Ok(ProcessObservation::Alive) + } else { + let error = std::io::Error::last_os_error(); + if error.raw_os_error() == Some(libc::ESRCH) { + Ok(ProcessObservation::Dead) + } else if error.raw_os_error() == Some(libc::EPERM) { + Ok(ProcessObservation::Alive) + } else { + Err(error.into()) + } + } +} + fn service_runbook(args: Args) -> Result<()> { seed_epiphany_local_verse_context( &args.store, @@ -3139,11 +3296,15 @@ fn service_lifecycle_receipt( operator_artifact_ref: Option, ) -> EpiphanyCultMeshDaemonServiceLifecycleReceiptEntry { let receipt_id = args.receipt_id.clone().unwrap_or_else(|| { - format!( + let base = format!( "daemon-service-lifecycle-receipt-{}-{}", - sanitize_id(&args.service_id), - sanitize_id(action) - ) + sanitize_id(&args.service_id), sanitize_id(action) + ); + if action == "launch" { + format!("{base}-{}", started_at.timestamp_millis()) + } else { + base + } }); EpiphanyCultMeshDaemonServiceLifecycleReceiptEntry { schema_version: EPIPHANY_CULTMESH_DAEMON_SERVICE_LIFECYCLE_RECEIPT_SCHEMA_VERSION @@ -3592,6 +3753,8 @@ impl Args { | "service-desired-state" | "managed-service-read" | "service-desired-state-read" + | "managed-service-reconcile" + | "service-desired-state-reconcile" | "service-runbook" | "runbook-service" | "repo-work-service-audit" diff --git a/epiphany-core/src/cultmesh_integration.rs b/epiphany-core/src/cultmesh_integration.rs index 573e7d2d..270802da 100644 --- a/epiphany-core/src/cultmesh_integration.rs +++ b/epiphany-core/src/cultmesh_integration.rs @@ -1511,8 +1511,6 @@ pub struct EpiphanyCultMeshManagedServicePolicyEntry { pub stdout_artifact: String, #[cultcache(key = 12)] pub stderr_artifact: String, - #[cultcache(key = 13)] - pub last_lifecycle_receipt_id: String, #[cultcache(key = 14)] pub updated_at_utc: String, #[cultcache(key = 15)] diff --git a/notes/epiphany-current-algorithmic-map.md b/notes/epiphany-current-algorithmic-map.md index cc5e8a9a..a6e46901 100644 --- a/notes/epiphany-current-algorithmic-map.md +++ b/notes/epiphany-current-algorithmic-map.md @@ -376,6 +376,17 @@ witness; `managed-service-read` projects desired state plus `latestLifecycle`. It deliberately reports process observation as unknown until a real reconcile probe exists, so an old launch receipt cannot claim the service is alive. +That probe now exists. `managed-service-reconcile` reads only the service policy +and lifecycle history, observes the latest PID through the native platform +process API, applies enabled/restart-mode/cooldown decisions, and delegates a +restart to `service_launch`. Launch receipt ids now carry attempt time, so +restarts preserve history. Live root +`C:\Users\Meta\AppData\Local\Temp\epiphany-managed-reconcile-8e3d7cc5fe0b4c5fa38b18b3d00cea0c` +proved restart receipt +`daemon-service-lifecycle-receipt-idunn-managed-heartbeat-launch-1783858125893` +and PID `19736` were observed alive after the prior process was killed. Next +fold this reconcile into Idunn's own serve loop and cut the full-context Verse +load from service launch. This map must change when ownership changes. Historical scars belong in git, evidence, or an explicitly archived note—not in the machine's proprioception. diff --git a/notes/fresh-workspace-handoff.md b/notes/fresh-workspace-handoff.md index e14df944..3c680e45 100644 --- a/notes/fresh-workspace-handoff.md +++ b/notes/fresh-workspace-handoff.md @@ -2217,6 +2217,18 @@ showing desired command/args/cwd/restart mode/log refs without routine payloads. Readback calls the historical evidence `latestLifecycle`, not `actual`, and reports `processObservation=unknown-until-managed-service-reconcile`. Next add that reconcile probe and delegate restarts to the existing lifecycle primitive. +The probe/reconcile path is now landed. It uses native platform process +observation, refuses duplicate launch while alive, honors policy disable/ +restart/cooldown, and restarts dead or missing services only through the typed +`service_launch` owner. Launch receipt ids now include attempt timestamps and +therefore remain immutable across restarts. Live heartbeat restart root +`C:\Users\Meta\AppData\Local\Temp\epiphany-managed-reconcile-8e3d7cc5fe0b4c5fa38b18b3d00cea0c` +proved the replacement PID `19736` alive under receipt +`daemon-service-lifecycle-receipt-idunn-managed-heartbeat-launch-1783858125893`; +the test harness then killed it deliberately. The first combined smoke exceeded +its harness window because `service_launch` loads the entire local Verse merely +to inspect the swarm brake. Next use the narrow brake loader and integrate +managed-service reconcile into Idunn's native serve scheduler. ## Immediate Re-entry Instruction diff --git a/notes/perfect-machine-audit-roadmap.md b/notes/perfect-machine-audit-roadmap.md index 30e710c2..cd1b435f 100644 --- a/notes/perfect-machine-audit-roadmap.md +++ b/notes/perfect-machine-audit-roadmap.md @@ -114,6 +114,14 @@ service command/args/cwd and sealed log refs used by lifecycle launch, and can read it beside the latest lifecycle receipt. Readback explicitly reports process observation as unknown until reconciliation probes reality; a past `launched` receipt is not allowed to impersonate a living process. +Managed-service reconciliation is now implemented with a native platform PID +probe (`OpenProcess`/`GetExitCodeProcess` on Windows and `kill(pid, 0)` on +Unix). It honors enabled/restart mode/cooldown, observes alive services without +duplicating them, and delegates dead/missing restarts to `service-launch`. +Launch receipts now include an attempt timestamp so restart history is +immutable instead of overwriting one service/action key. Live heartbeat proof +launched, observed alive, simulated a crash, relaunched with a distinct PID and +receipt, then observed the replacement alive on the same heartbeat store. ### Owner @@ -150,6 +158,10 @@ The persistence/publication half is complete. Next add reconciliation that probes the policy's last PID or an equivalent platform process witness, applies enabled/restart-mode/cooldown intent, and delegates any restart to the existing service lifecycle launch primitive. +Reconciliation is complete as an explicit Idunn command. The remaining +unattended boundary is to include managed-service reconciliation in Idunn's +native `serve` scheduler and replace `service-launch`'s full local-Verse context +load with the narrow swarm-brake query it actually needs. ### Forbidden writers diff --git a/state/ledgers.msgpack b/state/ledgers.msgpack index 7c60ff22..018a00cc 100644 Binary files a/state/ledgers.msgpack and b/state/ledgers.msgpack differ diff --git a/state/map.yaml b/state/map.yaml index 3ce4b299..b13d20cb 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -659,4 +659,4 @@ current_status: - Service lifecycle receipt history is now event-only: `load_epiphany_cultmesh_daemon_service_lifecycle_receipts` filters out the `epiphany-local/daemon-service-lifecycle-receipt/latest` mirror, while `load_latest_epiphany_cultmesh_daemon_service_lifecycle_receipt` remains the latest-state reader. Focused unit test `service_lifecycle_receipt_history_excludes_latest_mirror` proves two per-receipt writes return exactly two history records and latest points at the second. Fresh smoke `verse-query-lifecycle-history-filter-smoke` and live overview/triage `local-20260618-145632-497-efac6f0d` / `local-20260618-145632-321-867947af` prove compact swarm sights still expose service lifecycle attention plus the present elevated runbook artifact without daemon pokes or service mutation. Do not use a broad unscoped cargo test filter for this check on Windows; it fans out across bin harnesses and hit a broken pipe during this pass. - `epiphany-verse-query seed-compact` is the compact seed receipt for wrappers and read-only rites. The older `seed` command still prints the full local Verse context for explicit debugging, but compact wrapper modes now avoid full `seed`/final `query` readback after the combined overview proof exposed Windows debug stack overflow from full-context serialization pressure. last_completed_repo_swarm_cut: Huginn live-fire has crossed the first real repo Body from Persona pressure to branch-local Hands commit and Soul/Mind closure, and now has repeatable closure and redacted-public-proof mouths. On `E:\Projects\Huginn` branch `epiphany/huginn/first-awakening`, `tools/epiphany_local_run.ps1 -Mode repo-persona-intake` accepted item `huginn-inspection-readback-livefire`, Weksa/Persona intake remained sealed, and `epiphany-work derive-plan` lowered it to model-authored safe family `repo.status_section` for `README.md` only. `tools/epiphany_local_run.ps1 -Mode repo-swarm-run` advanced four bounded pulses: run-from-plan, adopt-from-plan, execute-from-plan, and close-from-execute. Huginn commit `e9b07c38fceabbdc89ea35c2156326c80c0393a2` updated only `README.md`; closure receipt `.epiphany\work\work-close-huginn-inspection-readback-livefire.json` records status `closed`, Soul verdict `passed`, Mind state commit `repo-work-close-huginn-inspection-readback-livefire-mind-commit`, safe family `repo.status_section`, path scope `README.md`, Bifrost publication gate, and `privateStateExposed=false`. Verification also ran Huginn's TypeScript compiler through concrete Node path: `C:\Program Files\nodejs\node.exe E:\Projects\Huginn\node_modules\typescript\bin\tsc -p tsconfig.json` passed. Native verifier `epiphany-repo-livefire-closure-smoke --workspace E:\Projects\Huginn --item huginn-inspection-readback-livefire --expected-path README.md --expected-family repo.status_section --expected-gate awaiting-publication --expected-blocker bifrost-publication-missing` checks closure receipt, Mind map entry, overview gate/blocker, authority seals, and `git show` changed paths, returning `status=ok`, `soulVerdict=passed`, `publicationGate=Bifrost`, and `privateStateExposed=false`. Wrapper `tools/epiphany_local_run.ps1 -Mode repo-livefire-closure -Workspace E:\Projects\Huginn -RepoWorkItem huginn-inspection-readback-livefire -SkipBuild` exposes the same read-only proof through the local operator mouth; artifact `local-20260621-190156-803-2f85ba15` printed commit, path, gate, blocker, Soul verdict, Mind commit receipt, and `privateStateExposed=False` without push, merge, publication, or target mutation. New wrapper `tools/epiphany_local_run.ps1 -Mode repo-work-public-proof -Workspace E:\Projects\Huginn -RepoWorkItem huginn-inspection-readback-livefire -SkipBuild` exports the redacted proof bundle and repo-local Verse index without publishing; artifact `local-20260621-190645-345-4555b1dc` wrote `.epiphany\public\proof-bundles\repo-work-public-proof-huginn-inspection-readback-livefire.json`, proof id `repo-work-public-proof-huginn-inspection-readback-livefire`, SHA-256 `857545b199f3e37eb06c2e757e5cfc201958e46667bb1f8403fd333a22620f63`, nine artifact rows, zero publication rows, and `privateStateExposed=False`. Gjallar readback `local-20260621-190701-140-b9519ec6` sees the `REPO-WORK-PUBLIC-PROOF` row and action row from Huginn's repo-local Verse with the same SHA-256 while Bifrost publication remains missing. Launcher compact modes are now named compact operator-context modes instead of read-only modes, and `bifrost-public-proof` summaries print item, target public Verse, room, proof SHA-256, review receipts, and credit receipts so publication receipts are auditable without claiming upstream sync. - next_action: Add Idunn managed-service reconciliation over the landed service-id desired-state policy. Probe the latest lifecycle PID or equivalent platform witness, keep historical lifecycle separate from current observation, honor enabled/restart-mode/cooldown intent, delegate every restart to the existing service-launch primitive, and prove heartbeat unattended restart from persisted state without duplicate pending turns. + next_action: Make managed heartbeat restart genuinely unattended. Replace `service_launch`'s full local-Verse context load with the narrow swarm-brake loader, then integrate managed-service policy reconciliation into Idunn's native `serve` scheduler. Prove the scheduler observes a live heartbeat without duplication, restarts it after death with a unique lifecycle receipt, and preserves the heartbeat store without duplicate pending turns.