From 0b733c910104baf2bbdc1fdd5ffaa9558c2abdb0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 28 May 2026 19:19:32 +0100 Subject: [PATCH] Seal inner runtime jobs on worker timeout --- .../src/bin/epiphany-runtime-spine.rs | 28 +++++++++++- .../src/bin/epiphany-openai-runtime.rs | 43 ++++++++++++++++-- epiphany-openai-runtime/src/lib.rs | 7 +-- notes/fresh-workspace-handoff.md | 2 +- state/ledgers.msgpack | Bin 161684 -> 162044 bytes state/map.yaml | 4 +- 6 files changed, 74 insertions(+), 10 deletions(-) diff --git a/epiphany-core/src/bin/epiphany-runtime-spine.rs b/epiphany-core/src/bin/epiphany-runtime-spine.rs index 499209c8..00cd112d 100644 --- a/epiphany-core/src/bin/epiphany-runtime-spine.rs +++ b/epiphany-core/src/bin/epiphany-runtime-spine.rs @@ -2,6 +2,8 @@ use anyhow::Context; use anyhow::Result; use anyhow::anyhow; use chrono::SecondsFormat; +use epiphany_core::EpiphanyRuntimeJob; +use epiphany_core::EpiphanyRuntimeJobResult; use epiphany_core::RuntimeSpineEventOptions; use epiphany_core::RuntimeSpineInitOptions; use epiphany_core::RuntimeSpineJobOptions; @@ -12,6 +14,7 @@ use epiphany_core::complete_runtime_job; use epiphany_core::create_runtime_job; use epiphany_core::create_runtime_session; use epiphany_core::initialize_runtime_spine; +use epiphany_core::runtime_spine_cache; use epiphany_core::runtime_spine_status; use epiphany_core::write_runtime_hello_frame; use epiphany_core::write_runtime_schema_catalog_json; @@ -74,6 +77,27 @@ fn main() -> Result<()> { println!("documents: {}", status.supported_document_types.join(", ")); } } + Command::ListJobs => { + let mut cache = runtime_spine_cache(&args.store)?; + cache.pull_all_backing_stores()?; + let mut jobs = cache.get_all::()?; + let results = cache.get_all::()?; + jobs.sort_by(|left, right| left.created_at.cmp(&right.created_at)); + println!("runtime jobs"); + for job in jobs { + let result = results.iter().find(|result| result.job_id == job.job_id); + println!( + "{}\t{:?}\t{}\t{}\t{}", + job.job_id, + job.status, + job.role, + result + .map(|result| result.result_id.as_str()) + .unwrap_or("no-result"), + job.summary.replace(['\r', '\n', '\t'], " ") + ); + } + } Command::OpenSession { session_id, objective, @@ -198,6 +222,7 @@ enum Command { display_name: String, }, Status, + ListJobs, OpenSession { session_id: String, objective: String, @@ -278,6 +303,7 @@ fn parse_command(mut args: Vec) -> Result { }) } "status" => Ok(Command::Status), + "list-jobs" => Ok(Command::ListJobs), "open-session" => { let mut session_id = format!("session-{}", Uuid::new_v4()); let mut objective = String::new(); @@ -523,5 +549,5 @@ fn now() -> String { } fn usage() -> &'static str { - "usage: epiphany-runtime-spine [--store path] " + "usage: epiphany-runtime-spine [--store path] " } diff --git a/epiphany-openai-runtime/src/bin/epiphany-openai-runtime.rs b/epiphany-openai-runtime/src/bin/epiphany-openai-runtime.rs index 38faa65f..22bfe2db 100644 --- a/epiphany-openai-runtime/src/bin/epiphany-openai-runtime.rs +++ b/epiphany-openai-runtime/src/bin/epiphany-openai-runtime.rs @@ -1,5 +1,6 @@ use std::env; use std::fs; +use std::path::Path; use std::path::PathBuf; use std::process; use std::process::Command; @@ -94,7 +95,7 @@ async fn main() -> Result<()> { Ok(result) => result?, Err(_) => { let summary = format!("Worker runtime timed out after {seconds} seconds."); - let result = fail_worker_job( + let result = fail_worker_and_openai_jobs( &timeout_store, &timeout_job_id, summary.clone(), @@ -486,10 +487,10 @@ fn start_run_worker_timeout_watchdog(options: &RunWorkerCliOptions) -> Arc Arc Result { + let result = fail_worker_job(store_path, job_id, summary.clone(), next_safe_move)?; + let openai_job_id = format!("openai-worker-{job_id}"); + let _ = fail_worker_job_with_retry( + store_path, + &openai_job_id, + format!("{summary} Inner OpenAI transport job was sealed by the worker timeout."), + "Inspect provider stream/request observability before relaunching the worker.".to_string(), + ); + Ok(result) +} + +fn fail_worker_job_with_retry( + store_path: &Path, + job_id: &str, + summary: String, + next_safe_move: String, +) -> Result<()> { + let mut last_error = None; + for _ in 0..20 { + match fail_worker_job(store_path, job_id, summary.clone(), next_safe_move.clone()) { + Ok(_) => return Ok(()), + Err(err) => { + last_error = Some(err); + thread::sleep(Duration::from_millis(100)); + } + } + } + Err(last_error.unwrap_or_else(|| anyhow!("failed to seal runtime job {job_id:?}"))) +} + async fn run_worker_launch_with_tool_continuation( options: RunWorkerCliOptions, ) -> Result { diff --git a/epiphany-openai-runtime/src/lib.rs b/epiphany-openai-runtime/src/lib.rs index 0c0dc0e6..4022b22f 100644 --- a/epiphany-openai-runtime/src/lib.rs +++ b/epiphany-openai-runtime/src/lib.rs @@ -106,9 +106,6 @@ pub async fn run_openai_model_turn( ) -> Result { ensure_openai_runtime_ready(&options)?; let auth_manager = auth_manager(options.codex_home.clone()); - let status = status_from_auth_manager(&auth_manager, options.default_model.clone(), true).await; - store_openai_status(&options.store_path, &status)?; - store_model_status(&options.store_path, &status, DEFAULT_MODEL_PROVIDER)?; store_model_request( &options.store_path, &model_request_from_openai_request(DEFAULT_MODEL_PROVIDER, &request), @@ -147,6 +144,10 @@ pub async fn run_openai_model_turn( }, )?; + let status = status_from_auth_manager(&auth_manager, options.default_model.clone(), true).await; + store_openai_status(&options.store_path, &status)?; + store_model_status(&options.store_path, &status, DEFAULT_MODEL_PROVIDER)?; + let transport = EpiphanyCodexOpenAiTransport::openai(auth_manager); let events = match transport.collect_model_events(request.clone()).await { Ok(events) => events, diff --git a/notes/fresh-workspace-handoff.md b/notes/fresh-workspace-handoff.md index fd3e506e..6c132339 100644 --- a/notes/fresh-workspace-handoff.md +++ b/notes/fresh-workspace-handoff.md @@ -34,7 +34,7 @@ remembers doctrine; the branch remembers the blade. - Do not copy exact branch or HEAD from this note. Run `git status --short --branch` and `git log --oneline -5`. - The repo now has a one-command local operator path: `.\tools\epiphany_local_run.ps1`. Default mode is `smoke`; it builds the retained Codex app-server compatibility edge, builds native Epiphany operator binaries, runs the coordinator smoke, and writes launcher artifacts under `.epiphany-run/` plus coordinator artifacts under `.epiphany-dogfood/`. `-Mode status` is now Epiphany-native: it builds only Epiphany operator binaries, reads `state/thread-state.msgpack` when present, derives status through `epiphany-core`, and does not start or require `codex-app-server`. Use `-Mode plan` or `-Mode run -MaxSteps 4` for the raw bridge-equipped coordinator entrypoints; use `-Mode mvp -FaceInput "" -MaxSteps 4` for the local product cycle that projects Face's character turn, writes a Face/Aquarium bubble, bootstraps a minimal local checkpoint for fresh threads, runs the coordinator with auto-tools, then runs heartbeat sleep/dream maintenance. Live plan/run/mvp modes use the workspace `state/runtime-spine.msgpack`, because CodexThread launch writes worker requests there; the run bundle remains artifact evidence, not runtime lifecycle authority. Add `-ThreadId ` and `-Workspace ` to inspect or continue an existing thread. `status`, `plan`, and `smoke` do not spend model calls; `run` and `mvp` build and use `epiphany-openai-runtime`. Coordinator workers now launch detached and are polled through runtime-spine instead of blocking the coordinator. MVP runs are non-ephemeral by default so `-ThreadId ` can resume and poll later. `-MaxRuntimeSeconds ` is passed into worker runtime; timeout writes a failed worker receipt and exits. Failed modeling now blocks downstream verification until the failed result is reviewed. -- Current Mom-ready blocker: the normal 180-second MVP pass reaches Face -> coordinator -> detached runtime -> heartbeat sleep, but the modeling worker produces no typed result, no tool intents, and empty stdout/stderr before the runtime watchdog writes `Worker runtime timed out after 180 seconds.` Native status correctly projects that failure through role result, jobs, roles, and coordinator source signals. Next cut should diagnose provider stream/request observability or prompt/request size in `epiphany-model-runtime run-worker --provider openai-codex`, not coordinator scheduling. +- Current Mom-ready blocker: the normal 180-second MVP pass reaches Face -> coordinator -> detached runtime -> heartbeat sleep, but the modeling worker produces no typed result, no tool intents, and empty stdout/stderr before the runtime timeout writes `Worker runtime timed out after 180 seconds.` Native status correctly projects that failure through role result, jobs, roles, and coordinator source signals. Timeout ownership now seals the inner `openai-worker-*` job as well as the outer role worker; use `epiphany-runtime-spine list-jobs` to verify lifecycle invariants. Next cut should diagnose provider stream/request observability or prompt/request size in `epiphany-model-runtime run-worker --provider openai-codex`, not coordinator scheduling. - Local operator runs now write typed CultMesh intent/receipt documents to `.epiphany-run/cultmesh/operator-runs.ccmp` through `epiphany-operator-run`. The wrapper writes intent before acting and a completion receipt after success; this is the local consent/trace rail, not a hidden scheduler queue. - Status-mode local runs now also write a native CultMesh operator snapshot to `.epiphany-run/cultmesh/operator-snapshots.ccmp` through `epiphany-operator-snapshot`. That snapshot distills the operator-safe MVP status artifact into typed fields such as thread id, state status, coordinator action, CRRC action, reorient action, available actions, and artifact refs; it does not preserve the raw JSON status blob as internal state. - `epiphany-mvp-status --source native` is the current provider-neutral status boundary. It projects scene, pressure, jobs, roles, planning, reorient, CRRC, coordinator status, and runtime-linked worker lifecycle from native core surfaces. It reads runtime-spine job/result receipts for linked jobs so failed workers show as failed in role results, jobs, roles, and coordinator source signals, while successful semantic findings still require typed role/reorient worker result documents. Keep future pluggable model-provider work behind launch/result/runtime documents; do not let provider choice leak into status, prompt authority, scheduler policy, or durable `EpiphanyThreadState`. diff --git a/state/ledgers.msgpack b/state/ledgers.msgpack index 05aaab1bf0c46c9307ca85a00aa29837140cb705..598c9faa0dd93c2a9856e930c3168ca396761057 100644 GIT binary patch delta 387 zcmX|+Jx;?w5QVY36qNjwBT+j)mI9Fxcc4f`H;yNGh`noe$FWkSpx^)`2jB)s6o?CO z2gD%|JrazQKs)>G_ugB5cD}zlPlv(&C>fk3!~WIVUf8@pp9Hr-@Nu^e@7H1QvJS%^ zIas~M{rEKM52JWA9_;vs$D`wAGUk$OEx=}uj?oA>s*qg*t~OC)-HaR<+d%EfgEa)Q z`!Ol3C#W2kEIcQeS_i3>9K_Dp{$pq|Xiz43X0M?_$^AJlJ~qg!cu)K9$K>}ER<}x%~h*T(Z1W-UAT1CPWA3cEm@`@?%sOa m+zV~QZe!B9?Xt*MtRubPqQzxn^;oJ}OGRzL6g?FRb$&JALWyrx#BCRV0KQO6jV$_h6tGZr#3-fEu8)IODoar;yzrvD`XQBw~2 diff --git a/state/map.yaml b/state/map.yaml index 38818023..b468a5ca 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -72,7 +72,7 @@ active_subgoals: - Use `notes/epiphany-cultmesh-dreaming-roadmap.md` as the design authority for distributed Epiphany dreaming. Public dreams are separately authored typed documents distributed through CultMesh/CultNet; private state remains local and cannot be shared by flipping a visibility flag. Foreign dreams are external thought weather until a reviewed local adoption receipt digests them into memory, planning, doctrine, or governance. - CultMesh is now the preferred ergonomic Rust abstraction over CultCache and CultNet for Epiphany-owned local database surfaces. The Rust substrate is repo-contained under `vendor/cultcache-rs`, `vendor/cultnet-rs`, and `vendor/cultmesh-rs`; the old `E:\Projects\CultLib\crates\*` path is dead in this checkout and must not be reintroduced as a compile-time dependency. Future local Verse/dream/status work should start at CultMesh unless it is explicitly polishing CultMesh internals. Preserve the three-Verse boundary: `epiphany-internal` for private sub-agent typed state, `gamecult-local` for trusted LAN/Yggdrasil-tunnel GameCult project sharing, and `epiphany-global` for untrusted public dreams plus topic-specific threaded public rooms for Face posts. - The active priority is Codex starvation and CultNet liberation. Use `notes/codex-starvation-and-cultnet-liberation-plan.md` as the directive and `notes/codex-auth-spine-inventory.md` as the source-grounded keeper list for the retained OpenAI subscription auth/model transport bridge. Classify all Epiphany-relevant JSON contamination, define the native CultNet runtime boundary, keep worker launch/result/selfPatch cargo as typed documents, evacuate Epiphany operator/state/process authority from Codex JSON-RPC, wrap MCP's unavoidable JSON protocol edge in typed Epiphany intent/result/receipt documents, and keep cutting until Epiphany can run its own state and processes without `codex_message_processor.rs` except for bridge compatibility. - - The next Codex-starvation slice is provider-transport diagnosis after the Mom-ready normal pass. The quarantined Codex MCP spine may execute typed tool intents and emit typed receipts; the model runtime can build and run provider-neutral follow-up turns from those receipts; coordinator run mode passes the quarantined tool adapter into worker runtime with auto-tools enabled by default; worker runtime now enforces max-runtime with a failed lifecycle receipt; native status/roles/jobs/coordinator surfaces project failed runtime-linked jobs as terminal without Codex prompt or scheduler authority; and the normal 180-second MVP pass now fails cleanly at the provider/runtime stream seam instead of blocking coordinator or hiding as pending state. + - The next Codex-starvation slice is provider-transport diagnosis after the Mom-ready normal pass. The quarantined Codex MCP spine may execute typed tool intents and emit typed receipts; the model runtime can build and run provider-neutral follow-up turns from those receipts; coordinator run mode passes the quarantined tool adapter into worker runtime with auto-tools enabled by default; worker runtime now enforces max-runtime with one timeout close primitive for both tokio timeout and watchdog paths; inner `openai-worker-*` jobs are opened before async auth/stream waits and sealed with their outer worker on timeout; native status/roles/jobs/coordinator surfaces project failed runtime-linked jobs as terminal without Codex prompt or scheduler authority; and the normal 180-second MVP pass now fails cleanly at the provider/runtime stream seam instead of blocking coordinator or hiding as pending state. - The May 2026 teardown/rebuild migration in `notes/epiphany-ideal-architecture-rebuild-plan.md` is closed unless a concrete regression reopens it. Preserve the new foundation: Epiphany view/coordinator policy lives in `epiphany-core`, runtime-spine owns job/result lifecycle identity through `runtime_links`, duplicate acceptance is receipt-keyed, public findings stay typed/sealed, view expansion requires a core contract, and app-server remains routing/adaptation rather than a second policy throne. - Treat Phase 5 update/distill/propose/promote/read/notification as complete enough unless a concrete regression appears. - Preserve read-only boundaries for retrieve, distill, propose, freshness, context, graphQuery, roleResult, reorientResult, and the canonical view lenses for scene, jobs, roles, planning, pressure, reorient, crrc, and coordinator. @@ -569,4 +569,4 @@ current_status: materializes complete MCP-shaped model tool calls as typed tool invocation intents, and the Codex MCP spine can consume those intents from the mixed runtime-spine store and emit typed success/failure receipts. - next_action: Diagnose why `epiphany-model-runtime run-worker --provider openai-codex` produces no typed worker result, no tool intents, and empty worker stdout/stderr before the 180-second watchdog fires in the normal Mom-ready MVP pass. Keep the timeout/status projection repair; the next likely cut is provider stream observability or prompt/request sizing, not coordinator scheduling. + next_action: Diagnose why `epiphany-model-runtime run-worker --provider openai-codex` produces no typed worker result, no tool intents, and empty worker stdout/stderr before the 180-second timeout fires in the normal Mom-ready MVP pass. The timeout ownership leak is repaired; use `epiphany-runtime-spine list-jobs` to check job lifecycle invariants while investigating provider stream observability or prompt/request sizing.