From 7637b1cf0ecb12a7eacffc77a835b30046c2975b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 10 Aug 2026 14:56:02 +0100 Subject: [PATCH] Close failed worker process authority atomically --- epiphany-core/src/coordinator_launch.rs | 7 +- epiphany-core/src/resident_self.rs | 2 +- epiphany-core/src/runtime_spine.rs | 157 +++++++++++++++++++++- notes/epiphany-current-algorithmic-map.md | 31 +++-- notes/fresh-workspace-handoff.md | 32 +++++ state/ledgers.msgpack | Bin 605346 -> 605909 bytes state/map.yaml | 8 +- 7 files changed, 213 insertions(+), 24 deletions(-) diff --git a/epiphany-core/src/coordinator_launch.rs b/epiphany-core/src/coordinator_launch.rs index e7df62ef..973f8858 100644 --- a/epiphany-core/src/coordinator_launch.rs +++ b/epiphany-core/src/coordinator_launch.rs @@ -1216,7 +1216,7 @@ fn worker_process_attempt_is_terminal(cache: &CultCache, job_id: &str) -> Result .is_some_and(|claim| { matches!( claim.status.as_str(), - "terminal-death" | "terminal-unactivated" + "terminal-death" | "terminal-unactivated" | "terminal-failure" ) })) } @@ -1248,7 +1248,10 @@ fn terminal_runtime_link_for_binding( let Some(claim) = crate::runtime_worker_process_claim(runtime_store, &link.runtime_job_id)? else { return Ok(None); }; - if !matches!(claim.status.as_str(), "terminal-death" | "terminal-unactivated") { + if !matches!( + claim.status.as_str(), + "terminal-death" | "terminal-unactivated" | "terminal-failure" + ) { return Ok(None); } let terminal_authority_id = claim diff --git a/epiphany-core/src/resident_self.rs b/epiphany-core/src/resident_self.rs index 41ed2a01..450293ed 100644 --- a/epiphany-core/src/resident_self.rs +++ b/epiphany-core/src/resident_self.rs @@ -1348,7 +1348,7 @@ pub fn recover_dead_resident_typed_worker( return Ok(false); }; match claim.status.as_str() { - "terminal-death" | "terminal-unactivated" => return Ok(true), + "terminal-death" | "terminal-unactivated" | "terminal-failure" => return Ok(true), "terminal-result" => return Ok(false), "claimed" | "active" => {} _ => return Err(anyhow!("typed worker process claim has invalid status")), diff --git a/epiphany-core/src/runtime_spine.rs b/epiphany-core/src/runtime_spine.rs index 2f758860..3e99db05 100644 --- a/epiphany-core/src/runtime_spine.rs +++ b/epiphany-core/src/runtime_spine.rs @@ -10457,7 +10457,10 @@ pub fn runtime_typed_request_attempt_exists( .iter() .filter(|launch| { claims.get(&launch.job_id).is_none_or(|claim| { - !matches!(claim.status.as_str(), "terminal-death" | "terminal-unactivated") + !matches!( + claim.status.as_str(), + "terminal-death" | "terminal-unactivated" | "terminal-failure" + ) }) }) .any(|launch| match request { @@ -11128,6 +11131,19 @@ pub fn complete_runtime_job( options.result_id )); } + let event_id = format!("event-job-completed-{}", options.job_id); + if cache.get::(&event_id)?.is_some() { + return Err(anyhow!( + "runtime job completion event {:?} already exists", + event_id + )); + } + let snapshot = cache.snapshot_envelopes(); + let job_envelope = snapshot + .iter() + .find(|entry| entry.r#type == EpiphanyRuntimeJob::TYPE && entry.key == options.job_id) + .cloned() + .ok_or_else(|| anyhow!("runtime job lost its exact envelope"))?; let terminal_status = terminal_status_for_verdict(&options.verdict); job.status = terminal_status; job.updated_at = options.completed_at.clone(); @@ -11147,11 +11163,9 @@ pub fn complete_runtime_job( artifact_refs: options.artifact_refs, metadata: BTreeMap::new(), }; - cache.put(&job.job_id, &job)?; - cache.put(&result.result_id, &result)?; let event = EpiphanyRuntimeEvent { schema_version: RUNTIME_SPINE_SCHEMA_VERSION.to_string(), - event_id: format!("event-job-completed-{}", options.job_id), + event_id, occurred_at: options.completed_at, event_type: "job.completed".to_string(), source: "runtime-spine".to_string(), @@ -11163,7 +11177,56 @@ pub fn complete_runtime_job( ), metadata: BTreeMap::from([("resultId".to_string(), result.result_id.clone())]), }; - cache.put(&event.event_id, &event)?; + let mut expected = vec![job_envelope]; + let mut writes = vec![ + cache.prepare_entry(&job.job_id, &job)?.0, + cache.prepare_entry(&result.result_id, &result)?.0, + cache.prepare_entry(&event.event_id, &event)?.0, + ]; + let claim_id = worker_process_claim_id(&result.job_id); + if let Some(claim) = cache.get::(&claim_id)? { + match claim.status.as_str() { + "active" => { + let claim_envelope = snapshot + .iter() + .find(|entry| { + entry.r#type == EpiphanyRuntimeWorkerProcessClaim::TYPE + && entry.key == claim_id + }) + .cloned() + .ok_or_else(|| anyhow!("runtime worker completion lost its claim envelope"))?; + let mut terminal = claim; + if let Some(reorient) = + cache.get::(&result.job_id)? + { + terminal.status = "terminal-result".into(); + terminal.terminal_authority_id = Some(reorient.result_id); + } else { + terminal.status = "terminal-failure".into(); + terminal.terminal_authority_id = Some(result.result_id.clone()); + } + terminal.terminal_at = Some(result.completed_at.clone()); + expected.push(claim_envelope); + writes.push(cache.prepare_entry(&claim_id, &terminal)?.0); + } + "terminal-result" => {} + "claimed" => { + return Err(anyhow!( + "runtime worker job cannot complete before activation" + )); + } + status => { + return Err(anyhow!( + "runtime worker job completion found terminal process status {status:?}" + )); + } + } + } + if !runtime_spine_backing_store(store_path.as_ref())? + .compare_and_swap_batch(&expected, writes)? + { + return Err(anyhow!("runtime job completion lost its exact snapshot")); + } Ok(result) } @@ -12995,6 +13058,90 @@ pub(crate) mod tests { Ok(()) } + #[test] + fn generic_worker_failure_atomically_terminalizes_its_active_process_claim() -> Result<()> { + let root = tempdir()?; + let store = root.path().join("worker-process-failure.cc"); + initialize_runtime_spine( + &store, + RuntimeSpineInitOptions { + runtime_id: "worker-failure-runtime".into(), + display_name: "Worker failure runtime".into(), + created_at: "2026-08-10T13:30:00Z".into(), + }, + )?; + create_runtime_session( + &store, + RuntimeSpineSessionOptions { + session_id: "worker-failure-session".into(), + objective: "Prove generic failure closes exact worker process authority".into(), + created_at: "2026-08-10T13:30:01Z".into(), + coordinator_note: "Runtime Continuity fixture".into(), + }, + )?; + create_runtime_job( + &store, + RuntimeSpineJobOptions { + job_id: "worker-failure-job".into(), + session_id: "worker-failure-session".into(), + role: "epiphany-researcher".into(), + created_at: "2026-08-10T13:30:02Z".into(), + summary: "Await worker terminal authority".into(), + artifact_refs: Vec::new(), + }, + )?; + put_test_non_modeling_worker_launch(&store, "worker-failure-job", "research")?; + let mut cache = runtime_spine_cache(&store)?; + cache.pull_all_backing_stores()?; + let mut launch = cache + .get::("worker-failure-job")? + .expect("worker failure launch"); + launch.proposal_modeling_request_id = Some("worker-failure-request".into()); + cache.put("worker-failure-job", &launch)?; + let process = crate::capture_process_instance(std::process::id())?; + let token = "worker-failure-capability"; + claim_runtime_worker_process( + &store, + "worker-failure-job", + &process, + &format!("{:x}", Sha256::digest(token.as_bytes())), + "2026-08-10T13:30:03Z", + )?; + activate_runtime_worker_process( + &store, + "worker-failure-job", + &process, + token, + "2026-08-10T13:30:04Z", + )?; + let result = complete_runtime_job( + &store, + RuntimeSpineJobResultOptions { + result_id: "result-worker-failure-job".into(), + job_id: "worker-failure-job".into(), + completed_at: "2026-08-10T13:30:05Z".into(), + verdict: "failed".into(), + summary: "Provider failed before typed role output.".into(), + next_safe_move: "Retry only from this terminal process authority.".into(), + evidence_refs: Vec::new(), + artifact_refs: Vec::new(), + }, + )?; + let claim = runtime_worker_process_claim(&store, "worker-failure-job")? + .expect("terminal failure claim"); + assert_eq!(claim.status, "terminal-failure"); + assert_eq!( + claim.terminal_authority_id.as_deref(), + Some(result.result_id.as_str()) + ); + assert_eq!(claim.terminal_at.as_deref(), Some(result.completed_at.as_str())); + assert!(!runtime_typed_request_attempt_exists( + &store, + RuntimeTypedRequestRef::ProposalModeling("worker-failure-request"), + )?); + Ok(()) + } + #[test] fn worker_process_death_is_single_use_and_excludes_late_result() -> Result<()> { let root = tempdir()?; diff --git a/notes/epiphany-current-algorithmic-map.md b/notes/epiphany-current-algorithmic-map.md index 10f7a621..fdc1dff0 100644 --- a/notes/epiphany-current-algorithmic-map.md +++ b/notes/epiphany-current-algorithmic-map.md @@ -3168,8 +3168,9 @@ spawns `epiphany-openai-runtime run-worker` with an activation-capability digest then waits for the child-authored exact process claim before presenting the preimage. The worker cannot enter watchdog, provider, model, or tool paths before activation. Runtime result admission atomically replaces the active claim with -`terminal-result`; resident Continuity may replace it with `terminal-death` only -after exact native process observation. +`terminal-result`; generic runtime completion atomically replaces it with +`terminal-failure`; resident Continuity may replace it with `terminal-death` +only after exact native process observation. - Owner: the worker runtime owns claiming its exact native process incarnation before model or tool work. Runtime Continuity owns activation, terminalization, @@ -3179,9 +3180,10 @@ after exact native process observation. explicit activation bound to that claim, native exact-process observation, and authenticated terminal worker result. - Outputs: `claimed` before consequence-capable work, `active` only after one - explicit activation, then exactly one `terminal-result`, `terminal-death`, or - `terminal-unactivated`. A terminal-death/unactivated attempt may be superseded - by a distinct job/binding incarnation without deleting the prior launch family. + explicit activation, then exactly one `terminal-result`, `terminal-failure`, + `terminal-death`, or `terminal-unactivated`. A terminal failure/death/ + unactivated attempt may be superseded by a distinct job/binding incarnation + without deleting the prior launch family. - Derived state: coordinator artifact PID and detached `Child` handle are diagnostic only. Request identity remains retry-stable; launch, worker claim, process, session, and artifact directory are attempt-incarnation state. @@ -3197,7 +3199,9 @@ after exact native process observation. coordinator launcher, and resident Self. `roleFailureReview` no longer owns proposal retry. All three typed lanes consult exact process terminality, and dead runtime-link projections are demoted before planning a replacement. - The remaining cut is packaged Linux falsification, not another retry policy. + Generic job completion now owns one snapshot-fenced write of the runtime job, + generic result, completion event, and terminal-failure claim. The remaining + cut is packaged Linux falsification, not another retry policy. - Verification layer: kill before claim, after claim/before activation, after activation/before result, and after valid result/before coordinator receipt; prove no model/tool work before activation, no dual live attempts, same typed @@ -3205,10 +3209,13 @@ after exact native process observation. retention non-resurrection across all three typed lanes. Source evidence now includes exact process/result race tests, activation CLI ordering, typed retry for proposal/Imagination/model direction, stale-link demotion, canonical Self - settlement, core 660 passed/1 ignored, swarm 9, coordinator 17, and OpenAI - runtime 10. Exact 75b84180 package inspection authenticates 24 binaries, + settlement, core 661 passed/1 ignored, swarm 9, coordinator 17, and OpenAI + runtime 15+10+10. Exact 75b84180 package inspection authenticates 24 binaries, release sha256-3959bcb3, witness sha256-6e34eb0a, and zero warnings. Packaged - Linux proves terminal-unactivated without provider work plus durable claimed - and active orphan shapes after SIGKILL/no OOM. Resident terminal-death, - canonical requeue/retry, valid-result-before-receipt, and retention replay - remain the open packaged boundaries. + Linux proves terminal-unactivated without provider work, durable claimed and + active orphan shapes after SIGKILL/no OOM, native terminal-death, canonical + requeue, and one same-request distinct retry without dual live authority. The + retry exposed an activated generic-failure path that left its claim active; + source now closes that as terminal-failure atomically. Packaged + terminal-failure, valid-result-before-receipt, and retention replay remain the + open boundaries. diff --git a/notes/fresh-workspace-handoff.md b/notes/fresh-workspace-handoff.md index eece549d..24041e22 100644 --- a/notes/fresh-workspace-handoff.md +++ b/notes/fresh-workspace-handoff.md @@ -1,5 +1,37 @@ # Fresh workspace handoff +## Worker terminal-failure ownership exposed live and repaired in source — 2026-08-10 + +Exact packaged `75b84180` copied-state runs now prove both claimed and active +SIGKILL orphans become `terminal-death` only after Linux exact-process +observation. Replaying the claimed recovery returned success without changing +the runtime-store SHA-256. A persisted active resident lease then settled +through production `settle_resident_self_exited_coordinator`: the lease cleared, +the exact grant became `process-failed`, one cancellation ack appeared, and the +pressure returned to pending once. Heartbeat issued attempt two for that same +pressure. A distinct packaged job/process became active while the prior claim +remained terminal-death, proving no dual live attempt. Receipts are under +`.epiphany-run/worker-continuity-75b84180/typed-claimed-sigkill/` and +`typed-active-sigkill-3/`. + +The retry worker then exited zero after a typed model-transport failure, with a +generic failed runtime result but an incorrectly surviving `active` process +claim. This is a real Continuity fault: generic completion and process +terminality had split writers. Current source makes `complete_runtime_job` one +snapshot-fenced transaction for job, result, completion event, and exact active +claim. A generic worker completion becomes `terminal-failure`; an admitted +reorient result remains `terminal-result`. Proposal Modeling, Imagination, and +admitted-model-direction retry predicates accept terminal-failure as exact +supersession authority. No cleanup loop was added. + +Core passes 661 tests with one intentional ignore. The OpenAI runtime passes 15 +library tests, 10 model-runtime tests, and 10 openai-runtime tests. The cut is +not yet committed or packaged. Next review/commit/push it, package exact source +once on Starfire, then replay the packaged activated transport failure and +require atomic terminal-failure plus canonical resident settlement/retry and +retention non-resurrection. Valid-result-before-coordinator-receipt remains +open. Actual c006 remains braked and stopped. Yggdrasil never builds. + ## Packaged worker start gate accepted — recovery/retry still open — 2026-08-10 Exact pushed source `75b841808bb5f693acfe4513befd430fbaf65b75` diff --git a/state/ledgers.msgpack b/state/ledgers.msgpack index a77912d7da54db7629855399d40d0d440384ba1d..3a7d60058af466d4cd055e5b8184a939572c9996 100644 GIT binary patch delta 610 zcmX|8v2GJV5ak>vjtjSlawO7$&pC)qWGh7!Ma04qNOW}L%fw#1x4YS$wPP9=DQIXd z_k00@fAaT%zN|Re*JUw`|r`W$4TpD|D@N?vJbx=HP62Pc$(ZM z$(Q?e?en^p2+H@Kpf1=K90(eM!=LL~v(?`;(q^lXZ8_e_m6?6?~1UaiY!TaIsH^b2gtanp10hoND3~KdB`mE7Q zz#I7nR7n`DhU_?ypUc^r0jzeco}9)dI8WtY8TV173(>ebZ@=jF20Kkn7zhF}Yw2>{ z<`hEv231)u6i-E%qws0Imof$yZaELbz5zd`b+#YHB~Y z+h&l$kb{^fG%z7UUgf}E+&b$SO5Ny596B3HD)X>syD|ncHcR0mdQYZgk3N^>*cK{B zE%|Jon}fsEj`yP-%hn-k?$Ep(sQm0wtxT3;74-meE9YJ>>tvuNMvCp-yme&GhyU}- ey3ev>dB9-^DZG