Reconcile Idunn managed services
This commit is contained in:
Meta Vens
parent
1da96bd74f
commit
eeb1dce363
8 changed files with 215 additions and 13 deletions
|
|
@ -37,3 +37,9 @@ pretty_assertions = "1.4.1"
|
|||
tempfile = "3"
|
||||
tokio = { version = "1", features = ["rt", "rt-multi-thread"] }
|
||||
wiremock = "0.6"
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
windows-sys = { version = "0.59", features = ["Win32_Foundation", "Win32_System_Threading"] }
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
libc = "0.2"
|
||||
|
|
|
|||
|
|
@ -53,6 +53,9 @@ fn main() -> Result<()> {
|
|||
"service-launch" | "launch-service" | "start-service" => service_launch(args),
|
||||
"managed-service-policy" | "service-desired-state" => managed_service_policy(args),
|
||||
"managed-service-read" | "service-desired-state-read" => managed_service_read(args),
|
||||
"managed-service-reconcile" | "service-desired-state-reconcile" => {
|
||||
managed_service_reconcile(args)
|
||||
}
|
||||
"service-runbook" | "runbook-service" => service_runbook(args),
|
||||
"repo-work-service-audit"
|
||||
| "repo-work-service-readiness"
|
||||
|
|
@ -373,10 +376,6 @@ fn managed_service_policy(args: Args) -> Result<()> {
|
|||
backoff_multiplier: args.backoff_multiplier,
|
||||
stdout_artifact: stdout_artifact.display().to_string(),
|
||||
stderr_artifact: stderr_artifact.display().to_string(),
|
||||
last_lifecycle_receipt_id: latest
|
||||
.as_ref()
|
||||
.map(|receipt| receipt.receipt_id.clone())
|
||||
.unwrap_or_default(),
|
||||
updated_at_utc: Utc::now().to_rfc3339(),
|
||||
private_state_exposed: false,
|
||||
notes: vec![
|
||||
|
|
@ -403,7 +402,7 @@ fn managed_service_policy(args: Args) -> Result<()> {
|
|||
"args": written.args,
|
||||
"stdoutArtifact": written.stdout_artifact,
|
||||
"stderrArtifact": written.stderr_artifact,
|
||||
"lastLifecycleReceiptId": written.last_lifecycle_receipt_id,
|
||||
"latestLifecycleReceiptId": latest.as_ref().map(|receipt| receipt.receipt_id.as_str()),
|
||||
"privateStateExposed": written.private_state_exposed,
|
||||
}))?);
|
||||
Ok(())
|
||||
|
|
@ -423,6 +422,12 @@ fn managed_service_read(args: Args) -> Result<()> {
|
|||
.into_iter()
|
||||
.filter(|receipt| receipt.service_id == args.service_id)
|
||||
.max_by(|left, right| left.started_at_utc.cmp(&right.started_at_utc));
|
||||
let process_observation = latest
|
||||
.as_ref()
|
||||
.and_then(|receipt| receipt.process_id)
|
||||
.map(observe_process)
|
||||
.transpose()?
|
||||
.unwrap_or(ProcessObservation::Missing);
|
||||
println!("{}", serde_json::to_string_pretty(&json!({
|
||||
"schemaVersion": "epiphany.cultmesh.managed_service_readback.v0",
|
||||
"status": "desired-state-ready",
|
||||
|
|
@ -445,12 +450,164 @@ fn managed_service_read(args: Args) -> Result<()> {
|
|||
"startedAtUtc": receipt.started_at_utc,
|
||||
"completedAtUtc": receipt.completed_at_utc,
|
||||
})),
|
||||
"processObservation": "unknown-until-managed-service-reconcile",
|
||||
"processObservation": process_observation.label(),
|
||||
"privateStateExposed": false,
|
||||
}))?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn managed_service_reconcile(mut args: Args) -> Result<()> {
|
||||
let policy = load_epiphany_cultmesh_managed_service_policy(
|
||||
&args.store,
|
||||
args.runtime_id.clone(),
|
||||
&args.service_id,
|
||||
)?
|
||||
.with_context(|| format!("managed service policy missing for {}", args.service_id))?;
|
||||
let latest = load_epiphany_cultmesh_daemon_service_lifecycle_receipts(
|
||||
&args.store,
|
||||
args.runtime_id.clone(),
|
||||
)?
|
||||
.into_iter()
|
||||
.filter(|receipt| receipt.service_id == args.service_id)
|
||||
.max_by(|left, right| left.started_at_utc.cmp(&right.started_at_utc));
|
||||
let observation = latest
|
||||
.as_ref()
|
||||
.and_then(|receipt| receipt.process_id)
|
||||
.map(observe_process)
|
||||
.transpose()?
|
||||
.unwrap_or(ProcessObservation::Missing);
|
||||
if !policy.enabled || policy.restart_mode == "never" {
|
||||
println!("{}", serde_json::to_string_pretty(&json!({
|
||||
"schemaVersion": "epiphany.cultmesh.managed_service_reconcile.v0",
|
||||
"status": "disabled",
|
||||
"serviceId": policy.service_id,
|
||||
"processObservation": observation.label(),
|
||||
"restarted": false,
|
||||
"privateStateExposed": false,
|
||||
}))?);
|
||||
return Ok(());
|
||||
}
|
||||
if observation == ProcessObservation::Alive {
|
||||
println!("{}", serde_json::to_string_pretty(&json!({
|
||||
"schemaVersion": "epiphany.cultmesh.managed_service_reconcile.v0",
|
||||
"status": "observed-alive",
|
||||
"serviceId": policy.service_id,
|
||||
"processId": latest.as_ref().and_then(|receipt| receipt.process_id),
|
||||
"processObservation": observation.label(),
|
||||
"restarted": false,
|
||||
"privateStateExposed": false,
|
||||
}))?);
|
||||
return Ok(());
|
||||
}
|
||||
if policy.restart_mode == "on-failure"
|
||||
&& latest.as_ref().is_some_and(|receipt| {
|
||||
receipt.status == "completed" && receipt.exit_code.unwrap_or_default() == 0
|
||||
})
|
||||
{
|
||||
println!("{}", serde_json::to_string_pretty(&json!({
|
||||
"schemaVersion": "epiphany.cultmesh.managed_service_reconcile.v0",
|
||||
"status": "completed-no-restart",
|
||||
"serviceId": policy.service_id,
|
||||
"processObservation": observation.label(),
|
||||
"restarted": false,
|
||||
"privateStateExposed": false,
|
||||
}))?);
|
||||
return Ok(());
|
||||
}
|
||||
if !args.force && policy.cooldown_seconds > 0 {
|
||||
if let Some(started) = latest
|
||||
.as_ref()
|
||||
.and_then(|receipt| DateTime::parse_from_rfc3339(&receipt.started_at_utc).ok())
|
||||
{
|
||||
let elapsed = Utc::now().signed_duration_since(started.with_timezone(&Utc));
|
||||
if elapsed < Duration::seconds(policy.cooldown_seconds) {
|
||||
println!("{}", serde_json::to_string_pretty(&json!({
|
||||
"schemaVersion": "epiphany.cultmesh.managed_service_reconcile.v0",
|
||||
"status": "cooling-down",
|
||||
"serviceId": policy.service_id,
|
||||
"processObservation": observation.label(),
|
||||
"cooldownSeconds": policy.cooldown_seconds,
|
||||
"elapsedSeconds": elapsed.num_seconds(),
|
||||
"restarted": false,
|
||||
"privateStateExposed": false,
|
||||
}))?);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
args.service_command = Some(PathBuf::from(&policy.command));
|
||||
args.service_args = policy.args.clone();
|
||||
args.cwd = policy.cwd.as_ref().map(PathBuf::from);
|
||||
args.stdout_artifact = Some(PathBuf::from(&policy.stdout_artifact));
|
||||
args.stderr_artifact = Some(PathBuf::from(&policy.stderr_artifact));
|
||||
args.reason = Some(format!(
|
||||
"Idunn reconciled managed service {} after observing {}.",
|
||||
policy.service_id,
|
||||
observation.label()
|
||||
));
|
||||
service_launch(args)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum ProcessObservation {
|
||||
Alive,
|
||||
Dead,
|
||||
Missing,
|
||||
}
|
||||
|
||||
impl ProcessObservation {
|
||||
fn label(self) -> &'static str {
|
||||
match self {
|
||||
Self::Alive => "alive",
|
||||
Self::Dead => "dead",
|
||||
Self::Missing => "missing",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn observe_process(process_id: u32) -> Result<ProcessObservation> {
|
||||
use windows_sys::Win32::Foundation::CloseHandle;
|
||||
use windows_sys::Win32::System::Threading::{
|
||||
GetExitCodeProcess, OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION,
|
||||
};
|
||||
const STILL_ACTIVE: u32 = 259;
|
||||
unsafe {
|
||||
let handle = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, process_id);
|
||||
if handle.is_null() {
|
||||
return Ok(ProcessObservation::Dead);
|
||||
}
|
||||
let mut exit_code = 0_u32;
|
||||
let read = GetExitCodeProcess(handle, &mut exit_code);
|
||||
CloseHandle(handle);
|
||||
if read == 0 {
|
||||
anyhow::bail!("failed to inspect managed service process {process_id}");
|
||||
}
|
||||
Ok(if exit_code == STILL_ACTIVE {
|
||||
ProcessObservation::Alive
|
||||
} else {
|
||||
ProcessObservation::Dead
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn observe_process(process_id: u32) -> Result<ProcessObservation> {
|
||||
let result = unsafe { libc::kill(process_id as i32, 0) };
|
||||
if result == 0 {
|
||||
Ok(ProcessObservation::Alive)
|
||||
} else {
|
||||
let error = std::io::Error::last_os_error();
|
||||
if error.raw_os_error() == Some(libc::ESRCH) {
|
||||
Ok(ProcessObservation::Dead)
|
||||
} else if error.raw_os_error() == Some(libc::EPERM) {
|
||||
Ok(ProcessObservation::Alive)
|
||||
} else {
|
||||
Err(error.into())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn service_runbook(args: Args) -> Result<()> {
|
||||
seed_epiphany_local_verse_context(
|
||||
&args.store,
|
||||
|
|
@ -3139,11 +3296,15 @@ fn service_lifecycle_receipt(
|
|||
operator_artifact_ref: Option<String>,
|
||||
) -> EpiphanyCultMeshDaemonServiceLifecycleReceiptEntry {
|
||||
let receipt_id = args.receipt_id.clone().unwrap_or_else(|| {
|
||||
format!(
|
||||
let base = format!(
|
||||
"daemon-service-lifecycle-receipt-{}-{}",
|
||||
sanitize_id(&args.service_id),
|
||||
sanitize_id(action)
|
||||
)
|
||||
sanitize_id(&args.service_id), sanitize_id(action)
|
||||
);
|
||||
if action == "launch" {
|
||||
format!("{base}-{}", started_at.timestamp_millis())
|
||||
} else {
|
||||
base
|
||||
}
|
||||
});
|
||||
EpiphanyCultMeshDaemonServiceLifecycleReceiptEntry {
|
||||
schema_version: EPIPHANY_CULTMESH_DAEMON_SERVICE_LIFECYCLE_RECEIPT_SCHEMA_VERSION
|
||||
|
|
@ -3592,6 +3753,8 @@ impl Args {
|
|||
| "service-desired-state"
|
||||
| "managed-service-read"
|
||||
| "service-desired-state-read"
|
||||
| "managed-service-reconcile"
|
||||
| "service-desired-state-reconcile"
|
||||
| "service-runbook"
|
||||
| "runbook-service"
|
||||
| "repo-work-service-audit"
|
||||
|
|
|
|||
|
|
@ -1511,8 +1511,6 @@ pub struct EpiphanyCultMeshManagedServicePolicyEntry {
|
|||
pub stdout_artifact: String,
|
||||
#[cultcache(key = 12)]
|
||||
pub stderr_artifact: String,
|
||||
#[cultcache(key = 13)]
|
||||
pub last_lifecycle_receipt_id: String,
|
||||
#[cultcache(key = 14)]
|
||||
pub updated_at_utc: String,
|
||||
#[cultcache(key = 15)]
|
||||
|
|
|
|||
|
|
@ -376,6 +376,17 @@ witness; `managed-service-read` projects desired state plus
|
|||
`latestLifecycle`. It deliberately reports process observation as unknown
|
||||
until a real reconcile probe exists, so an old launch receipt cannot claim the
|
||||
service is alive.
|
||||
That probe now exists. `managed-service-reconcile` reads only the service policy
|
||||
and lifecycle history, observes the latest PID through the native platform
|
||||
process API, applies enabled/restart-mode/cooldown decisions, and delegates a
|
||||
restart to `service_launch`. Launch receipt ids now carry attempt time, so
|
||||
restarts preserve history. Live root
|
||||
`C:\Users\Meta\AppData\Local\Temp\epiphany-managed-reconcile-8e3d7cc5fe0b4c5fa38b18b3d00cea0c`
|
||||
proved restart receipt
|
||||
`daemon-service-lifecycle-receipt-idunn-managed-heartbeat-launch-1783858125893`
|
||||
and PID `19736` were observed alive after the prior process was killed. Next
|
||||
fold this reconcile into Idunn's own serve loop and cut the full-context Verse
|
||||
load from service launch.
|
||||
|
||||
This map must change when ownership changes. Historical scars belong in git,
|
||||
evidence, or an explicitly archived note—not in the machine's proprioception.
|
||||
|
|
|
|||
|
|
@ -2217,6 +2217,18 @@ showing desired command/args/cwd/restart mode/log refs without routine payloads.
|
|||
Readback calls the historical evidence `latestLifecycle`, not `actual`, and
|
||||
reports `processObservation=unknown-until-managed-service-reconcile`. Next add
|
||||
that reconcile probe and delegate restarts to the existing lifecycle primitive.
|
||||
The probe/reconcile path is now landed. It uses native platform process
|
||||
observation, refuses duplicate launch while alive, honors policy disable/
|
||||
restart/cooldown, and restarts dead or missing services only through the typed
|
||||
`service_launch` owner. Launch receipt ids now include attempt timestamps and
|
||||
therefore remain immutable across restarts. Live heartbeat restart root
|
||||
`C:\Users\Meta\AppData\Local\Temp\epiphany-managed-reconcile-8e3d7cc5fe0b4c5fa38b18b3d00cea0c`
|
||||
proved the replacement PID `19736` alive under receipt
|
||||
`daemon-service-lifecycle-receipt-idunn-managed-heartbeat-launch-1783858125893`;
|
||||
the test harness then killed it deliberately. The first combined smoke exceeded
|
||||
its harness window because `service_launch` loads the entire local Verse merely
|
||||
to inspect the swarm brake. Next use the narrow brake loader and integrate
|
||||
managed-service reconcile into Idunn's native serve scheduler.
|
||||
|
||||
## Immediate Re-entry Instruction
|
||||
|
||||
|
|
|
|||
|
|
@ -114,6 +114,14 @@ service command/args/cwd and sealed log refs used by lifecycle launch, and can
|
|||
read it beside the latest lifecycle receipt. Readback explicitly reports
|
||||
process observation as unknown until reconciliation probes reality; a past
|
||||
`launched` receipt is not allowed to impersonate a living process.
|
||||
Managed-service reconciliation is now implemented with a native platform PID
|
||||
probe (`OpenProcess`/`GetExitCodeProcess` on Windows and `kill(pid, 0)` on
|
||||
Unix). It honors enabled/restart mode/cooldown, observes alive services without
|
||||
duplicating them, and delegates dead/missing restarts to `service-launch`.
|
||||
Launch receipts now include an attempt timestamp so restart history is
|
||||
immutable instead of overwriting one service/action key. Live heartbeat proof
|
||||
launched, observed alive, simulated a crash, relaunched with a distinct PID and
|
||||
receipt, then observed the replacement alive on the same heartbeat store.
|
||||
|
||||
### Owner
|
||||
|
||||
|
|
@ -150,6 +158,10 @@ The persistence/publication half is complete. Next add reconciliation that
|
|||
probes the policy's last PID or an equivalent platform process witness, applies
|
||||
enabled/restart-mode/cooldown intent, and delegates any restart to the existing
|
||||
service lifecycle launch primitive.
|
||||
Reconciliation is complete as an explicit Idunn command. The remaining
|
||||
unattended boundary is to include managed-service reconciliation in Idunn's
|
||||
native `serve` scheduler and replace `service-launch`'s full local-Verse context
|
||||
load with the narrow swarm-brake query it actually needs.
|
||||
|
||||
### Forbidden writers
|
||||
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -659,4 +659,4 @@ current_status:
|
|||
- Service lifecycle receipt history is now event-only: `load_epiphany_cultmesh_daemon_service_lifecycle_receipts` filters out the `epiphany-local/daemon-service-lifecycle-receipt/latest` mirror, while `load_latest_epiphany_cultmesh_daemon_service_lifecycle_receipt` remains the latest-state reader. Focused unit test `service_lifecycle_receipt_history_excludes_latest_mirror` proves two per-receipt writes return exactly two history records and latest points at the second. Fresh smoke `verse-query-lifecycle-history-filter-smoke` and live overview/triage `local-20260618-145632-497-efac6f0d` / `local-20260618-145632-321-867947af` prove compact swarm sights still expose service lifecycle attention plus the present elevated runbook artifact without daemon pokes or service mutation. Do not use a broad unscoped cargo test filter for this check on Windows; it fans out across bin harnesses and hit a broken pipe during this pass.
|
||||
- `epiphany-verse-query seed-compact` is the compact seed receipt for wrappers and read-only rites. The older `seed` command still prints the full local Verse context for explicit debugging, but compact wrapper modes now avoid full `seed`/final `query` readback after the combined overview proof exposed Windows debug stack overflow from full-context serialization pressure.
|
||||
last_completed_repo_swarm_cut: Huginn live-fire has crossed the first real repo Body from Persona pressure to branch-local Hands commit and Soul/Mind closure, and now has repeatable closure and redacted-public-proof mouths. On `E:\Projects\Huginn` branch `epiphany/huginn/first-awakening`, `tools/epiphany_local_run.ps1 -Mode repo-persona-intake` accepted item `huginn-inspection-readback-livefire`, Weksa/Persona intake remained sealed, and `epiphany-work derive-plan` lowered it to model-authored safe family `repo.status_section` for `README.md` only. `tools/epiphany_local_run.ps1 -Mode repo-swarm-run` advanced four bounded pulses: run-from-plan, adopt-from-plan, execute-from-plan, and close-from-execute. Huginn commit `e9b07c38fceabbdc89ea35c2156326c80c0393a2` updated only `README.md`; closure receipt `.epiphany\work\work-close-huginn-inspection-readback-livefire.json` records status `closed`, Soul verdict `passed`, Mind state commit `repo-work-close-huginn-inspection-readback-livefire-mind-commit`, safe family `repo.status_section`, path scope `README.md`, Bifrost publication gate, and `privateStateExposed=false`. Verification also ran Huginn's TypeScript compiler through concrete Node path: `C:\Program Files\nodejs\node.exe E:\Projects\Huginn\node_modules\typescript\bin\tsc -p tsconfig.json` passed. Native verifier `epiphany-repo-livefire-closure-smoke --workspace E:\Projects\Huginn --item huginn-inspection-readback-livefire --expected-path README.md --expected-family repo.status_section --expected-gate awaiting-publication --expected-blocker bifrost-publication-missing` checks closure receipt, Mind map entry, overview gate/blocker, authority seals, and `git show` changed paths, returning `status=ok`, `soulVerdict=passed`, `publicationGate=Bifrost`, and `privateStateExposed=false`. Wrapper `tools/epiphany_local_run.ps1 -Mode repo-livefire-closure -Workspace E:\Projects\Huginn -RepoWorkItem huginn-inspection-readback-livefire -SkipBuild` exposes the same read-only proof through the local operator mouth; artifact `local-20260621-190156-803-2f85ba15` printed commit, path, gate, blocker, Soul verdict, Mind commit receipt, and `privateStateExposed=False` without push, merge, publication, or target mutation. New wrapper `tools/epiphany_local_run.ps1 -Mode repo-work-public-proof -Workspace E:\Projects\Huginn -RepoWorkItem huginn-inspection-readback-livefire -SkipBuild` exports the redacted proof bundle and repo-local Verse index without publishing; artifact `local-20260621-190645-345-4555b1dc` wrote `.epiphany\public\proof-bundles\repo-work-public-proof-huginn-inspection-readback-livefire.json`, proof id `repo-work-public-proof-huginn-inspection-readback-livefire`, SHA-256 `857545b199f3e37eb06c2e757e5cfc201958e46667bb1f8403fd333a22620f63`, nine artifact rows, zero publication rows, and `privateStateExposed=False`. Gjallar readback `local-20260621-190701-140-b9519ec6` sees the `REPO-WORK-PUBLIC-PROOF` row and action row from Huginn's repo-local Verse with the same SHA-256 while Bifrost publication remains missing. Launcher compact modes are now named compact operator-context modes instead of read-only modes, and `bifrost-public-proof` summaries print item, target public Verse, room, proof SHA-256, review receipts, and credit receipts so publication receipts are auditable without claiming upstream sync.
|
||||
next_action: Add Idunn managed-service reconciliation over the landed service-id desired-state policy. Probe the latest lifecycle PID or equivalent platform witness, keep historical lifecycle separate from current observation, honor enabled/restart-mode/cooldown intent, delegate every restart to the existing service-launch primitive, and prove heartbeat unattended restart from persisted state without duplicate pending turns.
|
||||
next_action: Make managed heartbeat restart genuinely unattended. Replace `service_launch`'s full local-Verse context load with the narrow swarm-brake loader, then integrate managed-service policy reconciliation into Idunn's native `serve` scheduler. Prove the scheduler observes a live heartbeat without duplication, restarts it after death with a unique lifecycle receipt, and preserves the heartbeat store without duplicate pending turns.
|
||||
|
|
|
|||
Reference in a new issue