From 497df4586cd59411e51c75c245bbb2c32ac30d91 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 10 Aug 2026 08:16:20 +0100 Subject: [PATCH] Close coordinator error exits atomically --- .../src/bin/epiphany-mvp-coordinator.rs | 203 ++++++++++++------ epiphany-core/src/lib.rs | 1 + epiphany-core/src/runtime_spine.rs | 183 ++++++++++++++++ notes/epiphany-current-algorithmic-map.md | 37 ++++ notes/fresh-workspace-handoff.md | 39 ++++ state/ledgers.msgpack | Bin 599930 -> 600722 bytes state/map.yaml | 8 +- 7 files changed, 407 insertions(+), 64 deletions(-) diff --git a/epiphany-core/src/bin/epiphany-mvp-coordinator.rs b/epiphany-core/src/bin/epiphany-mvp-coordinator.rs index 778102b9..a213604a 100644 --- a/epiphany-core/src/bin/epiphany-mvp-coordinator.rs +++ b/epiphany-core/src/bin/epiphany-mvp-coordinator.rs @@ -13,14 +13,11 @@ use epiphany_core::HandsActionIntent; use epiphany_core::REPO_FRONTIER_HANDS_AUTHORITY_CONTRACT; use epiphany_core::REPO_FRONTIER_HANDS_AUTHORITY_SCHEMA_VERSION; use epiphany_core::RepoFrontierHandsAuthority; -use epiphany_core::RuntimeSpineEventOptions; use epiphany_core::RuntimeSpineInitOptions; -use epiphany_core::RuntimeSpineSessionOptions; -use epiphany_core::append_runtime_event; -use epiphany_core::create_runtime_session; use epiphany_core::finalize_coordinator_run; use epiphany_core::hands_action_review_for_intent; use epiphany_core::initialize_runtime_spine; +use epiphany_core::open_coordinator_run; use epiphany_core::load_epiphany_cultmesh_swarm_brake; use epiphany_core::put_hands_action_intent; use epiphany_core::put_hands_action_review; @@ -312,41 +309,13 @@ fn run_coordinator(args: &Args) -> Result { now(), ), )?; - let runtime_session = ensure_runtime_session( + let (runtime_session, runtime_event) = open_coordinator_run( &runtime_store, &runtime_session_id, + &thread_id, "Coordinate the Epiphany MVP lanes with native runtime job receipts.", + &now(), )?; - let runtime_event = append_runtime_event( - &runtime_store, - RuntimeSpineEventOptions { - event_id: format!("event-coordinator-started-{}", sanitize_id(&thread_id)), - occurred_at: now(), - event_type: "coordinator.started".to_string(), - source: "epiphany-mvp-coordinator".to_string(), - session_id: Some(runtime_session_id.clone()), - job_id: None, - summary: "Native coordinator session opened.".to_string(), - }, - ) - .or_else(|_| { - append_runtime_event( - &runtime_store, - RuntimeSpineEventOptions { - event_id: format!( - "event-coordinator-started-{}-{}", - sanitize_id(&thread_id), - uuid::Uuid::new_v4() - ), - occurred_at: now(), - event_type: "coordinator.started".to_string(), - source: "epiphany-mvp-coordinator".to_string(), - session_id: Some(runtime_session_id.clone()), - job_id: None, - summary: "Native coordinator session opened.".to_string(), - }, - ) - })?; startup_events.push(json!({ "type": "runtimeSpineSession", "store": runtime_store, @@ -354,6 +323,7 @@ fn run_coordinator(args: &Args) -> Result { "sessionId": runtime_session.session_id, "eventId": runtime_event.event_id, })); + let run_result = (|| -> Result { if let Some(objective) = args.objective.as_deref() { let intake = intake_operator_objective(&runtime_store, &thread_id, objective)?; startup_events.push(json!({ @@ -1164,6 +1134,73 @@ fn run_coordinator(args: &Args) -> Result { }), )?; Ok(summary) + })(); + match run_result { + Ok(summary) => Ok(summary), + Err(run_error) => { + let failure_receipt = failed_coordinator_run_receipt( + args, + &runtime_session_id, + &thread_id, + &runtime_store, + &run_error, + ); + match finalize_coordinator_run(&runtime_store, &failure_receipt) { + Ok(_) => Err(anyhow!( + "coordinator run failed after opening and was terminalized by receipt {:?}: {run_error:#}", + failure_receipt.receipt_id + )), + Err(finalize_error) => Err(anyhow!( + "coordinator run failed after opening ({run_error:#}); failure terminalization also failed: {finalize_error:#}" + )), + } + } + } +} + +fn failed_coordinator_run_receipt( + args: &Args, + session_id: &str, + thread_id: &str, + runtime_store: &Path, + error: &anyhow::Error, +) -> EpiphanyCoordinatorRunReceipt { + let created_at = now(); + EpiphanyCoordinatorRunReceipt { + schema_version: COORDINATOR_RUN_RECEIPT_SCHEMA_VERSION.to_string(), + receipt_id: format!( + "coordinator-run-{}-failed-{}", + sanitize_id(thread_id), + chrono::Utc::now().timestamp_millis() + ), + session_id: session_id.to_string(), + thread_id: thread_id.to_string(), + mode: args.mode.clone(), + status: "failed".to_string(), + final_action: "failed".to_string(), + final_reason: Some(format!("{error:#}")), + step_count: 0, + created_at, + model_provider: Some(args.model_provider.clone()), + runtime_store: runtime_store.display().to_string(), + artifact_refs: Vec::new(), + sealed_artifact_refs: Vec::new(), + metadata: BTreeMap::from([( + "terminalizationOwner".to_string(), + "epiphany-mvp-coordinator-error-boundary".to_string(), + )]), + resident_grant_id: args.resident_binding.get("grant-id").cloned(), + resident_launch_digest: args.resident_binding.get("launch-digest").cloned(), + resident_policy_digest: args.resident_binding.get("policy-digest").cloned(), + resident_argv_digest: args.resident_binding.get("argv-digest").cloned(), + resident_objective_digest: args.resident_binding.get("objective-digest").cloned(), + resident_release_commit: args.resident_binding.get("release-commit").cloned(), + resident_release_manifest_digest: args + .resident_binding + .get("release-manifest-digest") + .cloned(), + resident_executable_digest: args.resident_binding.get("executable-digest").cloned(), + } } fn intake_operator_objective( @@ -2079,31 +2116,6 @@ fn maybe_apply_role_self_patch(accepted: &Value, agent_memory_dir: &Path) -> Res Ok(Some(output)) } -fn ensure_runtime_session( - runtime_store: &Path, - session_id: &str, - objective: &str, -) -> Result { - match create_runtime_session( - runtime_store, - RuntimeSpineSessionOptions { - session_id: session_id.to_string(), - objective: objective.to_string(), - created_at: now(), - coordinator_note: - "Coordinator owns native runtime receipts before Codex execution dies.".to_string(), - }, - ) { - Ok(session) => Ok(session), - Err(err) if err.to_string().contains("already exists") => { - let mut cache = epiphany_core::runtime_spine_cache(runtime_store)?; - cache.pull_all_backing_stores()?; - cache.get_required::(session_id) - } - Err(err) => Err(err), - } -} - fn first_string_at(value: &Value, paths: &[&[&str]]) -> Option { 'paths: for path in paths { let mut cursor = value; @@ -2349,6 +2361,77 @@ mod tests { assert_eq!(options.display_name, "Epiphany epiphany-starfire"); } + #[test] + fn coordinator_error_after_opening_terminalizes_the_exact_session() -> Result<()> { + let smoke_root = env::current_dir()?.join(".epiphany-smoke"); + fs::create_dir_all(&smoke_root)?; + let temp = tempfile::tempdir_in(smoke_root)?; + let cwd = temp.path().join("workspace"); + fs::create_dir_all(&cwd)?; + let current_exe = env::current_exe()?; + let runtime_store = temp.path().join("runtime.cc"); + let args = Args { + model_runtime_bin: current_exe.clone(), + tool_adapter_bin: current_exe, + model_provider: "test".to_string(), + thread_id: Some("error-terminalization".to_string()), + objective: Some("Exercise the coordinator error boundary.".to_string()), + cwd: cwd.clone(), + codex_home: temp.path().join("codex-home"), + mcp_config: cwd.join(".epiphany/mcp.toml"), + artifact_dir: cwd.join(".epiphany-smoke/error-terminalization"), + agent_memory_dir: temp.path().join("mind.cc"), + runtime_store: runtime_store.clone(), + local_verse_store: temp.path().join("local-verse.cc"), + mode: "plan".to_string(), + max_steps: 1, + poll_seconds: 0.01, + timeout_seconds: 1, + max_runtime_seconds: 1, + ephemeral: true, + auto_review: false, + supersede_failed_results: false, + approve_manual_regather: false, + auto_tools: false, + proposal_modeling_request_id: None, + imagination_consideration_request_id: Some("conflicting-request".to_string()), + admitted_model_direction_consideration_request_id: None, + resident_binding: BTreeMap::new(), + resident_state_store: None, + resident_preparation_id: None, + runtime_id: Some("epiphany-error-terminalization".to_string()), + }; + + let error = run_coordinator(&args) + .expect_err("conflicting typed and operator intake must fail after opening"); + assert!( + error.to_string().contains("was terminalized by receipt"), + "unexpected coordinator error: {error:#}" + ); + let mut cache = epiphany_core::runtime_spine_cache(&runtime_store)?; + cache.pull_all_backing_stores()?; + let session = cache.get_required::( + "coordinator-error-terminalization", + )?; + assert_eq!( + session.status, + epiphany_core::EpiphanyRuntimeSessionStatus::Completed + ); + let receipts = cache.get_all::()?; + assert_eq!(receipts.len(), 1); + assert_eq!(receipts[0].status, "failed"); + assert_eq!(receipts[0].session_id, session.session_id); + let events = cache + .get_all::()? + .into_iter() + .filter(|event| event.session_id.as_deref() == Some(session.session_id.as_str())) + .collect::>(); + assert_eq!(events.len(), 2); + assert!(events.iter().any(|event| event.event_type == "coordinator.started")); + assert!(events.iter().any(|event| event.event_type == "session.completed")); + Ok(()) + } + #[test] fn coordinator_binary_has_no_codex_host_or_epiphany_json_rpc_dependency() { let source = include_str!("epiphany-mvp-coordinator.rs"); diff --git a/epiphany-core/src/lib.rs b/epiphany-core/src/lib.rs index 11f5d849..019793f2 100644 --- a/epiphany-core/src/lib.rs +++ b/epiphany-core/src/lib.rs @@ -1062,6 +1062,7 @@ pub use runtime_spine::ensure_runtime_repo_model; pub use runtime_spine::ensure_runtime_session; pub use runtime_spine::finalize_coordinator_run; pub use runtime_spine::initialize_runtime_spine; +pub use runtime_spine::open_coordinator_run; pub use runtime_spine::intake_user_repo_frontier_proposal; pub use runtime_spine::migrate_legacy_repo_model_projection_obligation; pub use runtime_spine::open_runtime_model_execution; diff --git a/epiphany-core/src/runtime_spine.rs b/epiphany-core/src/runtime_spine.rs index 67457801..c9971329 100644 --- a/epiphany-core/src/runtime_spine.rs +++ b/epiphany-core/src/runtime_spine.rs @@ -10125,6 +10125,98 @@ fn coordinator_completion_event(receipt: &EpiphanyCoordinatorRunReceipt) -> Epip } } +pub fn open_coordinator_run( + store_path: impl AsRef, + session_id: &str, + thread_id: &str, + objective: &str, + started_at: &str, +) -> Result<(EpiphanyRuntimeSession, EpiphanyRuntimeEvent)> { + open_coordinator_run_with_before_commit( + store_path, + session_id, + thread_id, + objective, + started_at, + || Ok(()), + ) +} + +fn open_coordinator_run_with_before_commit( + store_path: impl AsRef, + session_id: &str, + thread_id: &str, + objective: &str, + started_at: &str, + before_commit: F, +) -> Result<(EpiphanyRuntimeSession, EpiphanyRuntimeEvent)> +where + F: FnOnce() -> Result<()>, +{ + validate_non_empty(session_id, "coordinator run session id")?; + validate_non_empty(thread_id, "coordinator run thread id")?; + validate_non_empty(objective, "coordinator run objective")?; + chrono::DateTime::parse_from_rfc3339(started_at) + .map_err(|error| anyhow!("coordinator run start timestamp is invalid: {error}"))?; + if session_id != format!("coordinator-{thread_id}") { + return Err(anyhow!( + "coordinator run session id is not bound to its thread" + )); + } + let store_path = store_path.as_ref(); + let mut cache = runtime_spine_cache(store_path)?; + cache.pull_all_backing_stores()?; + require_identity(&cache)?; + let event_id = format!("event-coordinator-started-{session_id}"); + if cache.get::(session_id)?.is_some() + || cache.get::(&event_id)?.is_some() + || cache + .get_all::()? + .iter() + .any(|receipt| receipt.session_id == session_id || receipt.thread_id == thread_id) + { + return Err(anyhow!( + "coordinator run session or thread authority already exists" + )); + } + let session = EpiphanyRuntimeSession { + schema_version: RUNTIME_SPINE_SCHEMA_VERSION.to_string(), + session_id: session_id.to_string(), + objective: objective.to_string(), + status: EpiphanyRuntimeSessionStatus::Active, + created_at: started_at.to_string(), + updated_at: started_at.to_string(), + coordinator_note: "Coordinator owns native runtime receipts before process exit." + .to_string(), + metadata: BTreeMap::new(), + }; + let event = EpiphanyRuntimeEvent { + schema_version: RUNTIME_SPINE_SCHEMA_VERSION.to_string(), + event_id, + occurred_at: started_at.to_string(), + event_type: "coordinator.started".to_string(), + source: "epiphany-mvp-coordinator".to_string(), + session_id: Some(session_id.to_string()), + job_id: None, + summary: "Native coordinator session opened.".to_string(), + metadata: BTreeMap::new(), + }; + let snapshot = cache.snapshot_envelopes(); + let replacements = vec![ + cache.prepare_entry(session_id, &session)?.0, + cache.prepare_entry(&event.event_id, &event)?.0, + ]; + before_commit()?; + if !runtime_spine_backing_store(store_path)? + .replace_and_append_if_snapshot_unchanged(&snapshot, replacements)? + { + return Err(anyhow!( + "coordinator run opening lost its full snapshot fence" + )); + } + Ok((session, event)) +} + pub fn finalize_coordinator_run( store_path: impl AsRef, receipt: &EpiphanyCoordinatorRunReceipt, @@ -15826,6 +15918,97 @@ pub(crate) mod tests { Ok(()) } + #[test] + fn coordinator_run_opening_is_atomic_single_use_and_snapshot_fenced() -> Result<()> { + let temp = tempdir()?; + let store = temp.path().join("runtime.msgpack"); + initialize_runtime_spine( + &store, + RuntimeSpineInitOptions { + runtime_id: "epiphany-coordinator-opening-test".to_string(), + display_name: "Epiphany Coordinator Opening Test".to_string(), + created_at: "2026-08-10T10:00:00Z".to_string(), + }, + )?; + + let (session, event) = open_coordinator_run( + &store, + "coordinator-thread-open", + "thread-open", + "Open one coordinator authority family.", + "2026-08-10T10:00:01Z", + )?; + assert_eq!(session.status, EpiphanyRuntimeSessionStatus::Active); + assert_eq!(event.session_id.as_deref(), Some(session.session_id.as_str())); + assert_eq!(event.event_type, "coordinator.started"); + let exact_snapshot = fs::read(&store)?; + assert!( + open_coordinator_run( + &store, + "coordinator-thread-open", + "thread-open", + "Open one coordinator authority family.", + "2026-08-10T10:00:01Z", + ) + .is_err() + ); + assert_eq!(fs::read(&store)?, exact_snapshot); + assert!( + open_coordinator_run( + &store, + "coordinator-swapped", + "thread-bound", + "Refuse split identity.", + "2026-08-10T10:00:02Z", + ) + .is_err() + ); + + let raced = open_coordinator_run_with_before_commit( + &store, + "coordinator-thread-raced-open", + "thread-raced-open", + "Lose the opening snapshot.", + "2026-08-10T10:00:03Z", + || { + append_runtime_event( + &store, + RuntimeSpineEventOptions { + event_id: "event-concurrent-opening-observation".to_string(), + occurred_at: "2026-08-10T10:00:03Z".to_string(), + event_type: "test.concurrent".to_string(), + source: "test".to_string(), + session_id: None, + job_id: None, + summary: "Invalidate the opening snapshot.".to_string(), + }, + )?; + Ok(()) + }, + ); + assert!(raced.is_err()); + let mut cache = runtime_spine_cache(&store)?; + cache.pull_all_backing_stores()?; + assert!( + cache + .get::("coordinator-thread-raced-open")? + .is_none() + ); + assert!( + cache + .get::( + "event-coordinator-started-coordinator-thread-raced-open" + )? + .is_none() + ); + assert!( + cache + .get::("event-concurrent-opening-observation")? + .is_some() + ); + Ok(()) + } + #[test] fn coordinator_session_retention_repairs_legacy_and_owns_the_exact_family() -> Result<()> { let temp = tempdir()?; diff --git a/notes/epiphany-current-algorithmic-map.md b/notes/epiphany-current-algorithmic-map.md index be96f230..2bf11c16 100644 --- a/notes/epiphany-current-algorithmic-map.md +++ b/notes/epiphany-current-algorithmic-map.md @@ -2807,6 +2807,43 @@ core library passes 649 tests with one intentional ignore; both swarm binary suites and the coordinator release target are green. Exact package and copied live plateau proof remain the acceptance boundary. +Copied-live execution exposed a surviving opening/exit split after that map. +The coordinator binary still created `EpiphanyRuntimeSession` and +`coordinator.started` in separate calls, reused an existing Active session on +the same thread, and invoked `finalize_coordinator_run` only at the bottom of +the success path. Any ordinary error after opening could therefore leave false +Active authority; repeated use of the thread could append start events and +produce the ambiguous receipt family observed in copied c005 state. + +The corrected authority map is: + +- Owner: runtime spine owns both boundaries of a normal coordinator run. +- Inputs: exact runtime identity, unique thread/session pair, objective, start + timestamp, and later either the success receipt or the captured run error. +- Outputs: opening atomically appends the Active session plus deterministic + start event; every normal exit atomically appends one terminal receipt plus + completion event while replacing the session as Completed. +- Derived state: CLI errors and process exit codes report the terminalized + failure; they do not own session status. +- Forbidden writers: the coordinator binary no longer separately creates a + session, appends retry start events, or reuses an existing thread authority. + A catch-all error wrapper may construct the failed receipt but can only close + through `finalize_coordinator_run`. +- Shared paths: successful plan/default execution and every ordinary post-open + error share the runtime finalizer. Failure before atomic opening writes no + session family. Hard kill after opening remains Continuity recovery work. +- Cut line: remove `ensure_runtime_session`, the fallback UUID start event, and + success-only ownership of terminalization. +- Verification: opening is single-use and full-snapshot fenced; an unrelated + concurrent event leaves neither proposed session nor start event; conflicting + typed/operator intake returns an error only after one failed receipt, + completion event, and Completed session exist. + +Corrected source verification passes 59 focused coordinator tests, 17 native +coordinator-binary tests, both 8-test swarm suites, and 650 core library tests +with one intentional ignore. Exact corrected packaging and copied-live replay +remain open. + ### Semantic logical lifecycle implementation `retain_memory_semantic_projection_lifecycles` now realizes the safe unattempted diff --git a/notes/fresh-workspace-handoff.md b/notes/fresh-workspace-handoff.md index f16982ff..0b88b392 100644 --- a/notes/fresh-workspace-handoff.md +++ b/notes/fresh-workspace-handoff.md @@ -1999,3 +1999,42 @@ exercised on copied live state. Active c005 was untouched. Next: commit and push this exact source, package it once with the persistent Starfire cache, then prove copied-live repair, archival, non-resurrection, and bounded idle/active session counts. Yggdrasil never builds. + +## Copied-live coordinator lifecycle falsification and corrected owner — 2026-08-10 + +Exact pushed `5a479522` packaged successfully as +`sha256-7aadee7aeb651c480a7ac82248f03329e5ebded9a64d5332169b9d5f310a6619` +with witness +`sha256-89802be96d223c60b176cd69d144e691899f12110bc7747b0e79c5e67cc389bc`, +24 binaries, no warnings, and authenticated inspection. The first changed-core +package took 8m21s; an identical warm replay took 12.13s wall with Cargo at +0.92s and reproduced the exact release and witness. + +Copied c005 state then falsified the source-complete claim. Its historical +`coordinator-linux-eyes-tools-c005` has multiple terminal receipts, so exact +repair correctly refused ambiguity without mutation. A cleaner exact copy from +the accepted b8138a6b shutdown proof repaired one legacy Active session to +Completed under the packaged binary. While creating a newer terminal session +for archival proof, a deliberate coordinator input failure left a new Active +session: coordinator startup still wrote session and start event separately, +and ordinary `?` exits bypassed the success-only finalizer. + +The corrected source moves coordinator opening into runtime spine. Session and +deterministic `coordinator.started` event now commit together under one full +snapshot fence. Existing session, event, receipt, or thread authority refuses +reuse rather than appending another start event. Once opening succeeds, every +ordinary error is converted into a failed coordinator receipt and passed +through the same atomic terminalizer before the error returns. Hard process +death remains a distinct Continuity recovery boundary; retention does not +invent a receipt from age. + +Verification is green: 59 focused coordinator tests; 17 coordinator-binary +tests including deliberate conflicting intake and exact failed terminalization; +both swarm binary suites at 8/8; and the full core library at 650 passed with +one intentional ignore. The corrected source is not yet committed or packaged. +Receipts for the falsification and first package are under +`.epiphany-run/coordinator-session-live-5a479522*` and +`.epiphany-run/linux-package-5a479522*`. Active c005 was untouched. + +Next: commit/push the corrected owner, package it exactly, then repeat the +copied-live error closure, repair, archival, replay, and plateau proof. diff --git a/state/ledgers.msgpack b/state/ledgers.msgpack index f2b99c5d71d3808aae9bfa575a8a211a2945bfbe..91c0b40251a35b7493ef2e4de5ba718543592154 100644 GIT binary patch delta 815 zcmZ9Kv5wO~5QcFG$3+Orqo5gyG)QsI3HWlC>d=V>5ET{OW<8D<*1OB@I=LWD0a4H+ zqPzePfJA|K0ZKX=ItpHbido0e=_p2C+dDJ=eE#EiH(_O#0*}2gUCDzBE+I#DV-Q8=; z(SuXQJ*4d3=qMcyfomf$i=~($7^`&-7M~;cn9(n7GohH*7Np3LFStbdS{jKS6*@Nt zR!+pU(v^ckIJzh}lfaWCr7|3a@s198<*1xvIm+dt2TWzaAVVo8Xql(J+WA-;R2Nzb zWhxJ7ZD^0VryPZI$eDB)-{(Fkb|gY*1ry|qnm{Nb)D-5-fiqH25;gN^OPWK9SM0G5 z*ImUqWyV^`MIWAMAS4ZdtfFJM;)AkK$Loq0nDsw?8z#epI5~_5$`SRKC~Ha)77AGmO$)-S>q>M&K%6 z5N6+S^tjy4hLkizOD$^G=y1+;Ly=18YFdC+NEn4AQyQ0gTU*2Ejgmm_q%oV^+bHfs zdIY)VwWIZdte3jA^STFv(FOM=1l*FyiS`8?#vYvqH)pma_=4UxYmirx)