Seal inner runtime jobs on worker timeout

epiphany-core/src/bin/epiphany-runtime-spine.rs

@@ -2,6 +2,8 @@ use anyhow::Context;
 use anyhow::Result;
 use anyhow::anyhow;
 use chrono::SecondsFormat;
+use epiphany_core::EpiphanyRuntimeJob;
+use epiphany_core::EpiphanyRuntimeJobResult;
 use epiphany_core::RuntimeSpineEventOptions;
 use epiphany_core::RuntimeSpineInitOptions;
 use epiphany_core::RuntimeSpineJobOptions;
@@ -12,6 +14,7 @@ use epiphany_core::complete_runtime_job;
 use epiphany_core::create_runtime_job;
 use epiphany_core::create_runtime_session;
 use epiphany_core::initialize_runtime_spine;
+use epiphany_core::runtime_spine_cache;
 use epiphany_core::runtime_spine_status;
 use epiphany_core::write_runtime_hello_frame;
 use epiphany_core::write_runtime_schema_catalog_json;
@@ -74,6 +77,27 @@ fn main() -> Result<()> {
                 println!("documents: {}", status.supported_document_types.join(", "));
             }
         }
+        Command::ListJobs => {
+            let mut cache = runtime_spine_cache(&args.store)?;
+            cache.pull_all_backing_stores()?;
+            let mut jobs = cache.get_all::<EpiphanyRuntimeJob>()?;
+            let results = cache.get_all::<EpiphanyRuntimeJobResult>()?;
+            jobs.sort_by(|left, right| left.created_at.cmp(&right.created_at));
+            println!("runtime jobs");
+            for job in jobs {
+                let result = results.iter().find(|result| result.job_id == job.job_id);
+                println!(
+                    "{}\t{:?}\t{}\t{}\t{}",
+                    job.job_id,
+                    job.status,
+                    job.role,
+                    result
+                        .map(|result| result.result_id.as_str())
+                        .unwrap_or("no-result"),
+                    job.summary.replace(['\r', '\n', '\t'], " ")
+                );
+            }
+        }
         Command::OpenSession {
             session_id,
             objective,
@@ -198,6 +222,7 @@ enum Command {
         display_name: String,
     },
     Status,
+    ListJobs,
     OpenSession {
         session_id: String,
         objective: String,
@@ -278,6 +303,7 @@ fn parse_command(mut args: Vec<String>) -> Result<Command> {
             })
         }
         "status" => Ok(Command::Status),
+        "list-jobs" => Ok(Command::ListJobs),
         "open-session" => {
             let mut session_id = format!("session-{}", Uuid::new_v4());
             let mut objective = String::new();
@@ -523,5 +549,5 @@ fn now() -> String {
 }
 
 fn usage() -> &'static str {
-    "usage: epiphany-runtime-spine [--store path] <init|status|open-session|open-job|complete-job|record-event|hello-frame|schema-catalog>"
+    "usage: epiphany-runtime-spine [--store path] <init|status|list-jobs|open-session|open-job|complete-job|record-event|hello-frame|schema-catalog>"
 }

epiphany-openai-runtime/src/bin/epiphany-openai-runtime.rs

@@ -1,5 +1,6 @@
 use std::env;
 use std::fs;
+use std::path::Path;
 use std::path::PathBuf;
 use std::process;
 use std::process::Command;
@@ -94,7 +95,7 @@ async fn main() -> Result<()> {
                     Ok(result) => result?,
                     Err(_) => {
                         let summary = format!("Worker runtime timed out after {seconds} seconds.");
-                        let result = fail_worker_job(
+                        let result = fail_worker_and_openai_jobs(
                             &timeout_store,
                             &timeout_job_id,
                             summary.clone(),
@@ -486,10 +487,10 @@ fn start_run_worker_timeout_watchdog(options: &RunWorkerCliOptions) -> Arc<Atomi
             return;
         }
         let summary = format!("Worker runtime timed out after {seconds} seconds.");
-        let _ = fail_worker_job(
+        let _ = fail_worker_and_openai_jobs(
             &store_path,
             &job_id,
-            summary,
+            summary.clone(),
             "Inspect provider/tool transport before relaunching the worker.".to_string(),
         );
         process::exit(124);
@@ -497,6 +498,42 @@ fn start_run_worker_timeout_watchdog(options: &RunWorkerCliOptions) -> Arc<Atomi
     completed
 }
 
+fn fail_worker_and_openai_jobs(
+    store_path: &Path,
+    job_id: &str,
+    summary: String,
+    next_safe_move: String,
+) -> Result<epiphany_core::EpiphanyRuntimeJobResult> {
+    let result = fail_worker_job(store_path, job_id, summary.clone(), next_safe_move)?;
+    let openai_job_id = format!("openai-worker-{job_id}");
+    let _ = fail_worker_job_with_retry(
+        store_path,
+        &openai_job_id,
+        format!("{summary} Inner OpenAI transport job was sealed by the worker timeout."),
+        "Inspect provider stream/request observability before relaunching the worker.".to_string(),
+    );
+    Ok(result)
+}
+
+fn fail_worker_job_with_retry(
+    store_path: &Path,
+    job_id: &str,
+    summary: String,
+    next_safe_move: String,
+) -> Result<()> {
+    let mut last_error = None;
+    for _ in 0..20 {
+        match fail_worker_job(store_path, job_id, summary.clone(), next_safe_move.clone()) {
+            Ok(_) => return Ok(()),
+            Err(err) => {
+                last_error = Some(err);
+                thread::sleep(Duration::from_millis(100));
+            }
+        }
+    }
+    Err(last_error.unwrap_or_else(|| anyhow!("failed to seal runtime job {job_id:?}")))
+}
+
 async fn run_worker_launch_with_tool_continuation(
     options: RunWorkerCliOptions,
 ) -> Result<serde_json::Value> {

epiphany-openai-runtime/src/lib.rs

@@ -106,9 +106,6 @@ pub async fn run_openai_model_turn(
 ) -> Result<EpiphanyOpenAiRuntimeRunSummary> {
     ensure_openai_runtime_ready(&options)?;
     let auth_manager = auth_manager(options.codex_home.clone());
-    let status = status_from_auth_manager(&auth_manager, options.default_model.clone(), true).await;
-    store_openai_status(&options.store_path, &status)?;
-    store_model_status(&options.store_path, &status, DEFAULT_MODEL_PROVIDER)?;
     store_model_request(
         &options.store_path,
         &model_request_from_openai_request(DEFAULT_MODEL_PROVIDER, &request),
@@ -147,6 +144,10 @@ pub async fn run_openai_model_turn(
         },
     )?;
 
+    let status = status_from_auth_manager(&auth_manager, options.default_model.clone(), true).await;
+    store_openai_status(&options.store_path, &status)?;
+    store_model_status(&options.store_path, &status, DEFAULT_MODEL_PROVIDER)?;
+
     let transport = EpiphanyCodexOpenAiTransport::openai(auth_manager);
     let events = match transport.collect_model_events(request.clone()).await {
         Ok(events) => events,

notes/fresh-workspace-handoff.md

@@ -34,7 +34,7 @@ remembers doctrine; the branch remembers the blade.
 
 - Do not copy exact branch or HEAD from this note. Run `git status --short --branch` and `git log --oneline -5`.
 - The repo now has a one-command local operator path: `.\tools\epiphany_local_run.ps1`. Default mode is `smoke`; it builds the retained Codex app-server compatibility edge, builds native Epiphany operator binaries, runs the coordinator smoke, and writes launcher artifacts under `.epiphany-run/` plus coordinator artifacts under `.epiphany-dogfood/`. `-Mode status` is now Epiphany-native: it builds only Epiphany operator binaries, reads `state/thread-state.msgpack` when present, derives status through `epiphany-core`, and does not start or require `codex-app-server`. Use `-Mode plan` or `-Mode run -MaxSteps 4` for the raw bridge-equipped coordinator entrypoints; use `-Mode mvp -FaceInput "<operator request>" -MaxSteps 4` for the local product cycle that projects Face's character turn, writes a Face/Aquarium bubble, bootstraps a minimal local checkpoint for fresh threads, runs the coordinator with auto-tools, then runs heartbeat sleep/dream maintenance. Live plan/run/mvp modes use the workspace `state/runtime-spine.msgpack`, because CodexThread launch writes worker requests there; the run bundle remains artifact evidence, not runtime lifecycle authority. Add `-ThreadId <id>` and `-Workspace <path>` to inspect or continue an existing thread. `status`, `plan`, and `smoke` do not spend model calls; `run` and `mvp` build and use `epiphany-openai-runtime`. Coordinator workers now launch detached and are polled through runtime-spine instead of blocking the coordinator. MVP runs are non-ephemeral by default so `-ThreadId <id>` can resume and poll later. `-MaxRuntimeSeconds <n>` is passed into worker runtime; timeout writes a failed worker receipt and exits. Failed modeling now blocks downstream verification until the failed result is reviewed.
-- Current Mom-ready blocker: the normal 180-second MVP pass reaches Face -> coordinator -> detached runtime -> heartbeat sleep, but the modeling worker produces no typed result, no tool intents, and empty stdout/stderr before the runtime watchdog writes `Worker runtime timed out after 180 seconds.` Native status correctly projects that failure through role result, jobs, roles, and coordinator source signals. Next cut should diagnose provider stream/request observability or prompt/request size in `epiphany-model-runtime run-worker --provider openai-codex`, not coordinator scheduling.
+- Current Mom-ready blocker: the normal 180-second MVP pass reaches Face -> coordinator -> detached runtime -> heartbeat sleep, but the modeling worker produces no typed result, no tool intents, and empty stdout/stderr before the runtime timeout writes `Worker runtime timed out after 180 seconds.` Native status correctly projects that failure through role result, jobs, roles, and coordinator source signals. Timeout ownership now seals the inner `openai-worker-*` job as well as the outer role worker; use `epiphany-runtime-spine list-jobs` to verify lifecycle invariants. Next cut should diagnose provider stream/request observability or prompt/request size in `epiphany-model-runtime run-worker --provider openai-codex`, not coordinator scheduling.
 - Local operator runs now write typed CultMesh intent/receipt documents to `.epiphany-run/cultmesh/operator-runs.ccmp` through `epiphany-operator-run`. The wrapper writes intent before acting and a completion receipt after success; this is the local consent/trace rail, not a hidden scheduler queue.
 - Status-mode local runs now also write a native CultMesh operator snapshot to `.epiphany-run/cultmesh/operator-snapshots.ccmp` through `epiphany-operator-snapshot`. That snapshot distills the operator-safe MVP status artifact into typed fields such as thread id, state status, coordinator action, CRRC action, reorient action, available actions, and artifact refs; it does not preserve the raw JSON status blob as internal state.
 - `epiphany-mvp-status --source native` is the current provider-neutral status boundary. It projects scene, pressure, jobs, roles, planning, reorient, CRRC, coordinator status, and runtime-linked worker lifecycle from native core surfaces. It reads runtime-spine job/result receipts for linked jobs so failed workers show as failed in role results, jobs, roles, and coordinator source signals, while successful semantic findings still require typed role/reorient worker result documents. Keep future pluggable model-provider work behind launch/result/runtime documents; do not let provider choice leak into status, prompt authority, scheduler policy, or durable `EpiphanyThreadState`.

state/map.yaml

@@ -72,7 +72,7 @@ active_subgoals:
   - Use `notes/epiphany-cultmesh-dreaming-roadmap.md` as the design authority for distributed Epiphany dreaming. Public dreams are separately authored typed documents distributed through CultMesh/CultNet; private state remains local and cannot be shared by flipping a visibility flag. Foreign dreams are external thought weather until a reviewed local adoption receipt digests them into memory, planning, doctrine, or governance.
   - CultMesh is now the preferred ergonomic Rust abstraction over CultCache and CultNet for Epiphany-owned local database surfaces. The Rust substrate is repo-contained under `vendor/cultcache-rs`, `vendor/cultnet-rs`, and `vendor/cultmesh-rs`; the old `E:\Projects\CultLib\crates\*` path is dead in this checkout and must not be reintroduced as a compile-time dependency. Future local Verse/dream/status work should start at CultMesh unless it is explicitly polishing CultMesh internals. Preserve the three-Verse boundary: `epiphany-internal` for private sub-agent typed state, `gamecult-local` for trusted LAN/Yggdrasil-tunnel GameCult project sharing, and `epiphany-global` for untrusted public dreams plus topic-specific threaded public rooms for Face posts.
   - The active priority is Codex starvation and CultNet liberation. Use `notes/codex-starvation-and-cultnet-liberation-plan.md` as the directive and `notes/codex-auth-spine-inventory.md` as the source-grounded keeper list for the retained OpenAI subscription auth/model transport bridge. Classify all Epiphany-relevant JSON contamination, define the native CultNet runtime boundary, keep worker launch/result/selfPatch cargo as typed documents, evacuate Epiphany operator/state/process authority from Codex JSON-RPC, wrap MCP's unavoidable JSON protocol edge in typed Epiphany intent/result/receipt documents, and keep cutting until Epiphany can run its own state and processes without `codex_message_processor.rs` except for bridge compatibility.
-  - The next Codex-starvation slice is provider-transport diagnosis after the Mom-ready normal pass. The quarantined Codex MCP spine may execute typed tool intents and emit typed receipts; the model runtime can build and run provider-neutral follow-up turns from those receipts; coordinator run mode passes the quarantined tool adapter into worker runtime with auto-tools enabled by default; worker runtime now enforces max-runtime with a failed lifecycle receipt; native status/roles/jobs/coordinator surfaces project failed runtime-linked jobs as terminal without Codex prompt or scheduler authority; and the normal 180-second MVP pass now fails cleanly at the provider/runtime stream seam instead of blocking coordinator or hiding as pending state.
+  - The next Codex-starvation slice is provider-transport diagnosis after the Mom-ready normal pass. The quarantined Codex MCP spine may execute typed tool intents and emit typed receipts; the model runtime can build and run provider-neutral follow-up turns from those receipts; coordinator run mode passes the quarantined tool adapter into worker runtime with auto-tools enabled by default; worker runtime now enforces max-runtime with one timeout close primitive for both tokio timeout and watchdog paths; inner `openai-worker-*` jobs are opened before async auth/stream waits and sealed with their outer worker on timeout; native status/roles/jobs/coordinator surfaces project failed runtime-linked jobs as terminal without Codex prompt or scheduler authority; and the normal 180-second MVP pass now fails cleanly at the provider/runtime stream seam instead of blocking coordinator or hiding as pending state.
   - The May 2026 teardown/rebuild migration in `notes/epiphany-ideal-architecture-rebuild-plan.md` is closed unless a concrete regression reopens it. Preserve the new foundation: Epiphany view/coordinator policy lives in `epiphany-core`, runtime-spine owns job/result lifecycle identity through `runtime_links`, duplicate acceptance is receipt-keyed, public findings stay typed/sealed, view expansion requires a core contract, and app-server remains routing/adaptation rather than a second policy throne.
   - Treat Phase 5 update/distill/propose/promote/read/notification as complete enough unless a concrete regression appears.
   - Preserve read-only boundaries for retrieve, distill, propose, freshness, context, graphQuery, roleResult, reorientResult, and the canonical view lenses for scene, jobs, roles, planning, pressure, reorient, crrc, and coordinator.
@@ -569,4 +569,4 @@ current_status:
     materializes complete MCP-shaped model tool calls as typed tool invocation
     intents, and the Codex MCP spine can consume those intents from the mixed
     runtime-spine store and emit typed success/failure receipts.
-  next_action: Diagnose why `epiphany-model-runtime run-worker --provider openai-codex` produces no typed worker result, no tool intents, and empty worker stdout/stderr before the 180-second watchdog fires in the normal Mom-ready MVP pass. Keep the timeout/status projection repair; the next likely cut is provider stream observability or prompt/request sizing, not coordinator scheduling.
+  next_action: Diagnose why `epiphany-model-runtime run-worker --provider openai-codex` produces no typed worker result, no tool intents, and empty worker stdout/stderr before the 180-second timeout fires in the normal Mom-ready MVP pass. The timeout ownership leak is repaired; use `epiphany-runtime-spine list-jobs` to check job lifecycle invariants while investigating provider stream observability or prompt/request sizing.