Recover typed review after coordinator failure

This commit is contained in:
Meta Vens 2026-08-11 15:46:32 +02:00
commit 2e1ccaf7d0
5 changed files with 373 additions and 18 deletions

View file

@ -655,6 +655,7 @@ pub fn resident_cognitive_runtime_id(runtime_store: &Path) -> Result<String> {
}
fn resident_coordinator_binding_for_grant(
resident_store: &Path,
policy: &ResidentSelfPolicy,
grant: &ResidentSelfHeartbeatGrant,
) -> Result<ResidentCoordinatorBinding> {
@ -672,13 +673,18 @@ fn resident_coordinator_binding_for_grant(
let receipt = cache
.get::<crate::EpiphanyCoordinatorRunReceipt>(receipt_id)?
.ok_or_else(|| anyhow!("coordinator continuation receipt is missing"))?;
if !matches!(receipt.status.as_str(), "planned" | "needsReview" | "completed") {
return Err(anyhow!("coordinator continuation receipt is not successful"));
let required_action = if matches!(
receipt.status.as_str(),
"planned" | "needsReview" | "completed"
) {
resident_self_safe_continuation_action(&policy.runtime_store, &receipt)?
} else {
resident_self_failed_receipt_typed_continuation_action(
resident_store,
&policy.runtime_store,
&receipt,
)?
}
let required_action = resident_self_safe_continuation_action(
&policy.runtime_store,
&receipt,
)?
.ok_or_else(|| anyhow!("coordinator receipt no longer owns a safe continuation"))?;
return Ok(ResidentCoordinatorBinding {
runtime_id: runtime_identity.runtime_id,
@ -1275,6 +1281,106 @@ fn resident_self_safe_continuation_action(
Ok(action.map(str::to_string))
}
fn resident_self_failed_receipt_typed_continuation_action(
resident_store: &Path,
runtime_store: &Path,
receipt: &crate::EpiphanyCoordinatorRunReceipt,
) -> Result<Option<String>> {
if matches!(
receipt.status.as_str(),
"planned" | "needsReview" | "completed"
) {
return Ok(None);
}
let Some(grant_id) = receipt.resident_grant_id.as_deref() else {
return Ok(None);
};
let resident = state_cache(resident_store)?;
let Some(grant) = resident.get::<ResidentSelfHeartbeatGrant>(grant_id)? else {
return Ok(None);
};
if grant.consumed_at_millis.is_none()
|| grant.terminal_at_millis.is_none()
|| grant.private_state_exposed
{
return Ok(None);
}
let Some(crate::RuntimeTypedRequestRef::ProposalModeling(request_id)) =
resident_self_typed_request_ref(&grant)?
else {
return Ok(None);
};
let acknowledgements = resident
.get_all::<ResidentSelfTerminalAck>()?
.into_iter()
.filter(|ack| ack.grant_id == grant.grant_id)
.collect::<Vec<_>>();
if acknowledgements.len() != 1 {
return Err(anyhow!(
"failed coordinator receipt typed continuation requires one exact terminal acknowledgement"
));
}
let acknowledgement = &acknowledgements[0];
if acknowledgement.coordinator_receipt_id != receipt.receipt_id
|| receipt.resident_launch_digest.as_deref()
!= Some(acknowledgement.launch_digest.as_str())
{
return Err(anyhow!(
"failed coordinator receipt typed continuation lost its exact resident binding"
));
}
let Some(evidence) = crate::runtime_typed_request_fulfillment(
runtime_store,
crate::RuntimeTypedRequestRef::ProposalModeling(request_id),
)?
else {
return Ok(None);
};
let mut runtime = crate::runtime_spine_cache(runtime_store)?;
runtime.pull_all_backing_stores()?;
let thread = runtime
.get::<crate::EpiphanyThreadStateEntry>(crate::THREAD_STATE_KEY)?
.ok_or_else(|| anyhow!("typed continuation requires authoritative thread state"))?;
let state = thread.state()?;
if thread.thread_id != receipt.thread_id {
return Err(anyhow!(
"failed coordinator receipt typed continuation crossed thread authority"
));
}
let links = state
.runtime_links
.iter()
.filter(|link| {
link.binding_id == crate::EPIPHANY_MODELING_ROLE_BINDING_ID
&& link.runtime_job_id == evidence.job_id
&& link.runtime_result_id.as_deref() == Some(evidence.result_id.as_str())
})
.count();
if links != 1 {
return Err(anyhow!(
"typed proposal fulfillment does not own exactly one current Modeling result link"
));
}
let acceptances = state
.acceptance_receipts
.iter()
.filter(|accepted| {
accepted.result_id == evidence.result_id
&& accepted.job_id == evidence.job_id
&& accepted.binding_id == crate::EPIPHANY_MODELING_ROLE_BINDING_ID
&& accepted.surface == "roleAccept"
&& accepted.role_id == "modeling"
&& accepted.status == "accepted"
})
.count();
if acceptances > 1 {
return Err(anyhow!(
"typed proposal fulfillment has multiple Modeling acceptance authorities"
));
}
Ok((acceptances == 0).then(|| "reviewModelingResult".to_string()))
}
pub fn ingest_resident_self_coordinator_continuation_pressure(
resident_store: &Path,
runtime_store: &Path,
@ -1296,10 +1402,19 @@ pub fn ingest_resident_self_coordinator_continuation_pressure(
"resident continuation lost its last exact coordinator receipt"
));
};
if !matches!(receipt.status.as_str(), "planned" | "needsReview" | "completed") {
return Ok(false);
}
let Some(action) = resident_self_safe_continuation_action(runtime_store, &receipt)? else {
let action = if matches!(
receipt.status.as_str(),
"planned" | "needsReview" | "completed"
) {
resident_self_safe_continuation_action(runtime_store, &receipt)?
} else {
resident_self_failed_receipt_typed_continuation_action(
resident_store,
runtime_store,
&receipt,
)?
};
let Some(action) = action else {
return Ok(false);
};
let pressure_id = format!("resident-self-continuation-{receipt_id}-{action}");
@ -1987,7 +2102,7 @@ pub fn settle_resident_self_exited_coordinator(
resident_store,
lease,
&receipt.receipt_id,
&receipt.status,
if typed { "fulfilled" } else { &receipt.status },
now_millis,
cooldown_seconds,
false,
@ -2175,7 +2290,7 @@ pub fn prepare_resident_self_launch(
let Some(mut grant) = pending_resident_self_grant(path)? else {
return Ok(None);
};
let binding = resident_coordinator_binding_for_grant(policy, &grant)?;
let binding = resident_coordinator_binding_for_grant(path, policy, &grant)?;
let wake = ResidentSelfWake::Explicit {
objective: grant.objective.clone(),
};
@ -3939,7 +4054,7 @@ mod tests {
.get::<ResidentSelfHeartbeatGrant>(&grant.grant_id)?
.expect("completed grant");
assert_eq!(stale_grant.terminal_at_millis, Some(10));
assert_eq!(stale_grant.terminal_status.as_deref(), Some("planned"));
assert_eq!(stale_grant.terminal_status.as_deref(), Some("fulfilled"));
let expected_grant = cache
.snapshot_envelopes()
.into_iter()

View file

@ -14453,6 +14453,199 @@ pub(crate) mod tests {
Ok(())
}
#[test]
fn failed_coordinator_receipt_cannot_strand_authenticated_proposal_fulfillment()
-> Result<()> {
let root = tempfile::tempdir()?;
let (runtime_store, result, _) =
proposal_admission_fixture(root.path(), "failed-receipt-continuation")?;
let request_id = result
.proposal_modeling_request_id
.clone()
.expect("proposal request echo");
put_runtime_role_worker_result(&runtime_store, &result)?;
let mut runtime = runtime_spine_cache(&runtime_store)?;
runtime.pull_all_backing_stores()?;
let thread_entry = runtime
.get::<crate::EpiphanyThreadStateEntry>(crate::THREAD_STATE_KEY)?
.expect("proposal fixture thread state");
let mut thread_state = thread_entry.state()?;
let modeling_link = thread_state
.runtime_links
.iter_mut()
.find(|link| {
link.binding_id == crate::EPIPHANY_MODELING_ROLE_BINDING_ID
&& link.runtime_job_id == result.job_id
})
.expect("proposal fixture Modeling runtime link");
modeling_link.surface = "runtimeResult".into();
modeling_link.runtime_result_id = Some(result.result_id.clone());
runtime.put(
crate::THREAD_STATE_KEY,
&crate::EpiphanyThreadStateEntry::from_state(
&thread_entry.thread_id,
&thread_state,
)?,
)?;
let resident_store = root.path().join("resident.cc");
crate::enqueue_resident_self_pressure(
&resident_store,
&crate::ResidentSelfPressure {
schema_version: crate::RESIDENT_SELF_PRESSURE_SCHEMA_VERSION.into(),
pressure_id: "failed-receipt-proposal-pressure".into(),
kind: "repo-frontier-proposal-modeling".into(),
provenance_ref: format!(
"cultcache://repo-frontier-proposal-modeling/{request_id}"
),
objective: "Settle exact proposal Modeling fulfillment despite process failure."
.into(),
created_at_millis: 1,
status: "pending".into(),
consumed_by_grant_id: None,
private_state_exposed: false,
},
)?;
crate::heartbeat_issue_resident_self_grant(
&resident_store,
"failed-receipt-heartbeat",
"failed-receipt-action",
2,
)?
.expect("proposal Modeling grant");
let coordinator = root.path().join("epiphany-mvp-coordinator");
std::fs::write(&coordinator, b"witnessed executable")?;
let policy = crate::ResidentSelfPolicy {
workspace: root.path().to_path_buf(),
coordinator_bin: coordinator.clone(),
model_runtime_bin: root.path().join("epiphany-model-runtime"),
tool_adapter_bin: root.path().join("epiphany-tool-mcp-runtime"),
runtime_store: runtime_store.clone(),
local_verse_store: root.path().join("local-verse.cc"),
agent_memory_store: root.path().join("mind.cc"),
artifact_root: root.path().join("artifacts"),
codex_home: root.path().join("codex-home"),
mcp_config: root.path().join("mcp.toml"),
model_provider: "local".into(),
max_steps: 1,
turn_timeout_seconds: 60,
cooldown_seconds: 1,
idle_sleep_seconds: 5,
failure_backoff_seconds: 20,
release_commit: "failed-receipt-test-release".into(),
release_manifest_digest: "sha256:failed-receipt-test-manifest".into(),
release_store: root.path().join("release.cc"),
release_runtime_id: "deployment-runtime".into(),
release_id: "failed-receipt-test-release".into(),
release_witness_sha256: "sha256:failed-receipt-test-witness".into(),
};
let prepared = crate::prepare_resident_self_launch(&resident_store, &policy, 3)?
.expect("proposal Modeling preparation");
let process = crate::LaunchedCoordinator {
process_id: u32::MAX - 17,
process_creation_token: 83,
process_executable_path: coordinator,
};
crate::claim_resident_self_preparation_as_child(
&resident_store,
&prepared.preparation_id,
&process,
4,
)?;
let lease = crate::acknowledge_resident_self_launch(
&resident_store,
&prepared.preparation_id,
&process,
5,
)?;
let receipt = EpiphanyCoordinatorRunReceipt {
schema_version: COORDINATOR_RUN_RECEIPT_SCHEMA_VERSION.into(),
receipt_id: "failed-receipt-with-valid-proposal-result".into(),
session_id: coordinator_run_session_id(
&lease.turn_id,
Some(&lease.launch_digest),
)?,
thread_id: lease.turn_id.clone(),
mode: "execute".into(),
status: "failed".into(),
final_action: "waitForModelingResult".into(),
final_reason: Some("redundant proposal launch refused".into()),
step_count: 0,
created_at: "2026-08-11T13:00:00Z".into(),
model_provider: Some("local".into()),
runtime_store: runtime_store.display().to_string(),
artifact_refs: Vec::new(),
sealed_artifact_refs: Vec::new(),
metadata: BTreeMap::new(),
resident_grant_id: Some(lease.grant_id.clone()),
resident_launch_digest: Some(lease.launch_digest.clone()),
resident_policy_digest: Some(lease.policy_digest.clone()),
resident_argv_digest: Some(lease.argv_digest.clone()),
resident_objective_digest: Some(lease.objective_digest.clone()),
resident_release_commit: Some(lease.release_commit.clone()),
resident_release_manifest_digest: Some(lease.release_manifest_digest.clone()),
resident_executable_digest: Some(lease.coordinator_executable_digest.clone()),
};
runtime_spine_cache(&runtime_store)?.put(&receipt.receipt_id, &receipt)?;
assert_eq!(
crate::settle_resident_self_exited_coordinator(
&resident_store,
&runtime_store,
&lease,
&receipt,
false,
false,
false,
6,
policy.cooldown_seconds,
)?,
crate::ResidentSelfOutcome::Completed
);
let terminal_grant = crate::resident_self_grant_lifecycle_projection(
&resident_store,
Some(&lease.grant_id),
1,
)?
.pop()
.expect("terminal proposal grant");
assert_eq!(terminal_grant.terminal_status.as_deref(), Some("fulfilled"));
assert!(crate::ingest_resident_self_coordinator_continuation_pressure(
&resident_store,
&runtime_store,
7,
)?);
assert!(!crate::ingest_resident_self_coordinator_continuation_pressure(
&resident_store,
&runtime_store,
8,
)?);
let continuation = crate::resident_self_pressures(&resident_store)?
.into_iter()
.find(|pressure| {
pressure.kind == crate::RESIDENT_SELF_COORDINATOR_CONTINUATION_PRESSURE_KIND
})
.expect("typed fulfillment continuation pressure");
assert!(continuation.pressure_id.ends_with("-reviewModelingResult"));
assert_eq!(
continuation.provenance_ref,
format!("cultcache://coordinator-run-receipt/{}", receipt.receipt_id)
);
crate::heartbeat_issue_resident_self_grant(
&resident_store,
"failed-receipt-review-heartbeat",
"failed-receipt-review-action",
1_007,
)?
.expect("typed review continuation grant");
let review = crate::prepare_resident_self_launch(&resident_store, &policy, 1_008)?
.expect("typed review continuation preparation");
assert!(review.argv.windows(2).any(|pair| {
pair == ["--required-action", "reviewModelingResult"]
}));
assert!(!review.argv.iter().any(|arg| arg == "--objective"));
Ok(())
}
fn claim_repair_admission_fixture(
root: &Path,
suffix: &str,

View file

@ -1418,6 +1418,19 @@ the lease once and return its exact pressure to Heartbeat; invalid or
substituted evidence closes as `unfulfilled`. Coordinator process closure is
therefore evidence, never cognitive completion authority.
A failed coordinator receipt cannot erase a valid typed completion. This
matters when a continuation launch claims a proposal immediately before an
already-enqueued proposal grant is delivered: the second coordinator correctly
refuses the duplicate launch, while the first detached worker may still finish
the exact request. Resident settlement records that grant as `fulfilled`, not
as the failed process status. Continuation ingress may derive
`reviewModelingResult` only when the failed receipt, its exact resident grant
and launch digest, one terminal acknowledgement, the authenticated proposal
fulfillment, and the current Modeling runtime link all agree, and no acceptance
receipt already exists. The failed process receipt remains process evidence;
the typed result owns cognitive review readiness. This route cannot launch a
second worker or continue into Hands.
Exact packaged `6dd9b132` proves that ownership cut live. One autonomous
proposal Modeling coordinator exited while its detached worker continued; Self
held the same lease as `awaiting-fulfillment`. Runtime later admitted one exact

View file

@ -3678,6 +3678,40 @@ The live cascade exposed an ownership gap: admitted Imagination
runtime validation. Scheduler FIFO and one-at-a-time promotion cannot bound
the durable queue a single worker result may mint. The active source cut adds
shared `MAX_OPTION_DRAFTS = 3`, emits that as schema `maxItems`, and rejects
larger admitted results in Rust. Both focused tests pass. Commit/push and exact
package/warm replay remain next; c007d stays braked until the bounded body is
authenticated. Yggdrasil never builds.
larger admitted results in Rust. Commit `bc0806b5` is pushed. A current Linux
packager bootstrap (SHA-256
`b14405aeacb008b0a075cf409ccf7239b2a48c8b5af5d380e8aed64ba2e8a34d`)
then restored the already-designed stable `/source/repo-live` forge path. The
one-time path migration compiled fb873927 in 4m36s; the distinct bc0806b5 core
change completed in 1m34s, down from the stale per-commit clone path's 13m40s.
An identical warm replay took 8.32s wall / 0.77s Cargo and reproduced release
`sha256-3214699820704481268f3f52de9a9dd86948d424ff556177eb60e45330ad414f`
and witness
`sha256-6d03e787da348d9901aae72fa6fe94e1cbe31b4db97d95e8bf05303bc65029cc`
across 24 binaries with zero warnings. Receipts are
`.epiphany-run/forge-crosscommit-*` and
`.epiphany-run/packager-bootstrap-bc0806b5`.
c007d stopped both owners with exit zero. The workspace advanced to exact
bc0806b5, the release was published, and c007e restarted the exact authenticated
body with bounded Docker logs. After brake release, a stored safe continuation
launched the pending proposal's Modeling worker. The already-enqueued typed
proposal grant then correctly refused a duplicate unsuperseded launch, retained
the same active lease as `awaiting-fulfillment`, authenticated worker result
`result-worker-8ddf57c1-96b4-4a60-ae8b-b24404e9ce2f`, terminalized once, and
was acknowledged by Heartbeat. The apparent duplicate-launch failure was
therefore fail-closure, not corrupt state. The true wound was that the failed
coordinator receipt could not derive review pressure for the valid unaccepted
typed result. c007e is braked at resident revision 302 with no active turn;
status derives `reviewModelingResult`, `modelingResultProposalBound=true`, and
no pending proposal request.
The active source cut preserves the failed receipt as process evidence but lets
the exact fulfilled proposal result own review readiness. It requires the
failed receipt, its grant and launch digest, one terminal acknowledgement, the
runtime-owned typed fulfillment, the current Modeling result link, and absence
of prior acceptance before deriving only `reviewModelingResult`. Typed
completion now records terminal status `fulfilled` instead of copying the
process receipt's status. Thirty-one resident tests, nine swarm tests, and the
new hostile failed-receipt/valid-result lifecycle test pass. Commit/package and
exact c007e continuation replay remain next. Yggdrasil never builds.

View file

@ -36,7 +36,7 @@ invariants:
current_status:
summary: Epiphany is a supervised engineering alpha on Starfire, not deployment-ready. Exact b8138a6b active-child SIGTERM and restart, exact 9f369528 deterministic warm packaging, exact c930f505 cross-container ownership, exact ed15f83a resident lifecycle retention, exact 16c4b19f coordinator receipt and Heartbeat artifact retention, exact cce2b8ac narrow model-generation retention, exact eea15e82 model-turn JSON ingress, exact b1e8c89a semantic unattempted-generation retention, exact 6bc3ec3f semantic physical-epoch retirement, and exact 44230a05 Persona conversation retention are proven. Gamecult-ops 059821f owns Yggdrasil stream retention and exact 6ee08a4 owns stable Starfire Persona-permit firewall ingress. Exact 16c4b19f genuine idle and exact a987d2f7 active cognition completed 900-pulse bounded plateaus. Runtime-owned typed fulfillment is accepted live under exact 6dd9b132. Yggdrasil is the small public crossing body, never the release forge.
latest_phase: Exact fb873927 root recovery and resumed Modeling/Imagination proof remains accepted. c007c stopped both owners exit0; authenticated fb873927 was published, the mounted workspace advanced to current 0a3ca20d, and c007d restarted exact fb873927 under bounded local logs. Resident production serve passed its git Body preflight and both owners remained braked/healthy. A fresh Soul-frontier user proposal then completed exact typed Modeling fulfillment. Native Imagination's admitted-direction result produced several autonomous options, which were promoted through authenticated proposal bindings and exposed one at a time. The first autonomous proposal Modeling fulfillment completed once before the brake settled at resident revision 294; another selection repo-frontier-proposal-modeling-ef8245a7349c5cf78e3c2fbd6d4b570912aac522edb93dc695dd257ab448efe1 remains pending. Source inspection found the admitted Imagination option_drafts contract had no structural fan-out bound. The active cut adds one shared maximum of three to model output schema and runtime validation; two focused tests pass. This cut is not yet committed or packaged. Yggdrasil never builds.
latest_phase: Exact bc0806b5 bounds one admitted Imagination result to three option drafts and is pushed, packaged, inspected, and live as release sha256-3214699820704481268f3f52de9a9dd86948d424ff556177eb60e45330ad414f with witness sha256-6d03e787da348d9901aae72fa6fe94e1cbe31b4db97d95e8bf05303bc65029cc. The current packager plus stable /source/repo-live reduced a distinct core-changing package from the stale per-commit clone path's 13m40s to 1m34s; identical warm replay was 8.32s wall and 0.77s Cargo with identical 24 binaries and zero warnings. c007d stopped exit0 and c007e runs this exact body braked at resident revision 302. Its pending autonomous proposal launched one detached Modeling worker; an already-enqueued typed grant then refused a duplicate unsuperseded launch, waited on the exact existing worker, authenticated result-worker-8ddf57c1-96b4-4a60-ae8b-b24404e9ce2f, terminalized once, and was acknowledged. No pending proposal remains and status derives reviewModelingResult, but the failed redundant coordinator receipt cannot currently seed that review continuation. The active source cut lets only the exact fulfilled typed result bound through that receipt/grant/link derive reviewModelingResult; 31 resident, 9 swarm, and one focused lifecycle test pass. The cut is uncommitted and unpackaged. Yggdrasil never builds.
operational_state:
local_worktree: exact_acd91e6e_imagination_runtime_owned_causality_live_accepted
operator_wireguard: restored
@ -70,7 +70,7 @@ current_status:
yggdrasil_epiphany_service_contract: repaired_in_gamecult_ops_812b427_not_installed
resident_physiology: c006_exact_c9a2a979_stopped_cleanly_braked_revision_34_after_36700_pulse_endurance
unresolved_authority: Coordinator receipt, Heartbeat pulse-artifact, genuine-idle and active resource plateaus, exact-grant terminality, restart-persistent cooldown refusal, schedule/action issuance cardinality, runtime-owned typed fulfillment, all three packaged fulfilled-before-receipt lanes, genuine packaged Imagination option generation, narrow model-generation retention, published model-turn JSON ingress, semantic logical/physical retention, Persona conversation retention, stable Starfire firewall ownership, completed model-adapter session archival, normal coordinator lifecycle opening/error closure/legacy repair/archival, co-resident resident lifecycle retention, authenticated outer-worker attempt retention, deployed supervisor/projector journald ownership, Docker named-volume atomic snapshot visibility, and deterministic incremental release construction are accepted. Starfire's Docker Desktop F: bind is rejected for mutable authority after an exact packaged lockless reader observed an empty published snapshot during writer interruption. Historical migrated semantic attempts without recorded physical namespace remain durable and non-retirable. One completed Discord consequence remains blocked by the absent Yggdrasil credential. Exact aa841873 also makes legacy coordinator repair one full-snapshot transaction so a later hostile family cannot leave an earlier partial repair.
next_action: Keep exact fb873927 c007d braked at revision 294 with the pending autonomous proposal intact. Finish the bounded Imagination fan-out Soul pass, commit/push, and package exact source through Starfire; require exact inspection and an identical warm replay. Restart c007d only onto that authenticated release. Then drain at most the already-authenticated finite autonomous proposal queue one grant at a time, preserving receipt-bound review continuations, and prove the chain converges to an Imagination planning or Soul verification action without Hands/public consequence. Reengage the brake after each active fulfillment. Cold per-source compilation remains an open architecture cost; Yggdrasil never builds.
next_action: Keep exact bc0806b5 c007e braked at resident revision 302. Review the failed-receipt typed-fulfillment continuation cut, commit/push, package it through the stable Starfire forge, and require exact inspection plus identical warm replay. Restart only onto that authenticated body. Release one grant and prove result-worker-8ddf57c1-96b4-4a60-ae8b-b24404e9ce2f is accepted exactly once without worker relaunch or Hands/public consequence, then inspect the next typed action and reengage the brake. Yggdrasil never builds.
active_subgoals:
- Prove Persona, Self, Eyes, Modeling, Imagination, Hands, Soul, Mind, Continuity, and nervous-system behavior individually.