From 3d5d729ae45bd899d44967d0a04cc3519230b874 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 16 Jul 2026 02:22:56 +0100 Subject: [PATCH] Recover exact workspace coverage authority --- .../src/bin/epiphany-daemon-supervisor.rs | 341 +++++++++- epiphany-core/src/lib.rs | 6 + .../workspace_coverage_process_documents.rs | 407 ++++++++++- .../src/workspace_coverage_projector.rs | 637 +++++++++++++++++- notes/fresh-workspace-handoff.md | 34 + state/ledgers.msgpack | Bin 519056 -> 519611 bytes state/map.yaml | 5 +- state/scratch.md | 20 + 8 files changed, 1406 insertions(+), 44 deletions(-) diff --git a/epiphany-core/src/bin/epiphany-daemon-supervisor.rs b/epiphany-core/src/bin/epiphany-daemon-supervisor.rs index 244f85fd..785b3e73 100644 --- a/epiphany-core/src/bin/epiphany-daemon-supervisor.rs +++ b/epiphany-core/src/bin/epiphany-daemon-supervisor.rs @@ -49,10 +49,16 @@ use epiphany_core::write_epiphany_cultmesh_semantic_projector_service_policy; use epiphany_core::write_epiphany_cultmesh_workspace_coverage_projector_service_policy; use epiphany_core::{ WORKSPACE_COVERAGE_PROCESS_LAUNCH_SCHEMA_VERSION, WorkspaceCoverageManagedProcessLaunchEntry, - WorkspaceCoverageProcessBootstrap, capture_process_instance, native_boot_identity, - open_default_host_identity, sign_workspace_coverage_launch, - workspace_coverage_host_identity_record_digest, + WorkspaceCoverageProcessBootstrap, authenticate_workspace_coverage_provider_heartbeat, + authenticate_workspace_coverage_termination_with_envelope_digest, capture_process_instance, + current_workspace_coverage_recovery_target, + load_latest_workspace_coverage_managed_process_launch, + load_latest_workspace_coverage_provider_heartbeat, + load_workspace_coverage_process_termination_observation, native_boot_identity, + open_default_host_identity, recover_workspace_coverage_projection, + sign_workspace_coverage_launch, workspace_coverage_host_identity_record_digest, write_workspace_coverage_managed_process_launch, write_workspace_coverage_process_bootstrap, + write_workspace_coverage_process_termination_observation, }; use rand_core::{OsRng, RngCore}; use serde_json::Value; @@ -64,6 +70,8 @@ use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; use std::process::Stdio; +use std::thread; +use std::time::Duration as StdDuration; use uuid::Uuid; use zeroize::Zeroize; @@ -798,7 +806,26 @@ fn service_plan(args: Args) -> Result<()> { Ok(()) } +#[derive(Clone)] +struct CoverageReplacementEvidence { + old_launch_id: String, + termination_id: String, + termination_envelope_digest: String, +} + +struct ServiceLaunchOutcome { + coverage_launch: Option, +} + fn service_launch(args: Args) -> Result<()> { + service_launch_internal(args, None, true).map(|_| ()) +} + +fn service_launch_internal( + args: Args, + replacement: Option, + emit_output: bool, +) -> Result { let brake = load_epiphany_cultmesh_swarm_brake(&args.store, args.runtime_id.clone())?; assert_swarm_brake_allows_service_lifecycle_entry(brake.as_ref())?; let started_at = Utc::now(); @@ -982,6 +1009,15 @@ fn service_launch(args: Args) -> Result<()> { supervisor_id: "epiphany-daemon-supervisor".to_string(), identity_captured_at_utc: Utc::now().to_rfc3339(), signature_algorithm: "ed25519".to_string(), + replaces_launch_id: replacement + .as_ref() + .map(|evidence| evidence.old_launch_id.clone()), + replaces_termination_id: replacement + .as_ref() + .map(|evidence| evidence.termination_id.clone()), + replaces_termination_envelope_digest: replacement + .as_ref() + .map(|evidence| evidence.termination_envelope_digest.clone()), }; sign_workspace_coverage_launch(&mut launch, &host)?; write_workspace_coverage_managed_process_launch( @@ -994,19 +1030,23 @@ fn service_launch(args: Args) -> Result<()> { seed.zeroize(); match persist_result { Ok(written) => { - println!( - "{}", - serde_json::to_string_pretty(&json!({ - "status": "launched", - "store": args.store, - "runtimeId": args.runtime_id, - "serviceId": written.service_id, - "launchId": written.launch_id, - "processId": written.process_id, - "privateStateExposed": false, - }))? - ); - return Ok(()); + if emit_output { + println!( + "{}", + serde_json::to_string_pretty(&json!({ + "status": "launched", + "store": args.store, + "runtimeId": args.runtime_id, + "serviceId": written.service_id, + "launchId": written.launch_id, + "processId": written.process_id, + "privateStateExposed": false, + }))? + ); + } + return Ok(ServiceLaunchOutcome { + coverage_launch: Some(written), + }); } Err(error) => { let _ = child.kill(); @@ -1070,9 +1110,10 @@ fn service_launch(args: Args) -> Result<()> { return Err(error).context("failed to persist service launch receipt"); } }; - println!( - "{}", - serde_json::to_string_pretty(&json!({ + if emit_output { + println!( + "{}", + serde_json::to_string_pretty(&json!({ "status": written.status, "store": args.store, "runtimeId": args.runtime_id, @@ -1083,9 +1124,12 @@ fn service_launch(args: Args) -> Result<()> { "stdoutArtifact": args.stdout_artifact, "stderrArtifact": args.stderr_artifact, "privateStateExposed": written.private_state_exposed, - }))? - ); - Ok(()) + }))? + ); + } + Ok(ServiceLaunchOutcome { + coverage_launch: None, + }) } fn managed_service_policy(args: Args) -> Result<()> { @@ -1427,6 +1471,9 @@ fn managed_service_reconcile(mut args: Args) -> Result<()> { &args.service_id, )? .with_context(|| format!("managed service policy missing for {}", args.service_id))?; + if policy.service_id == WORKSPACE_COVERAGE_PROJECTOR_SERVICE_ID { + return reconcile_workspace_coverage_projector(args, policy); + } let latest = load_epiphany_cultmesh_daemon_service_lifecycle_receipts( &args.store, args.runtime_id.clone(), @@ -1524,6 +1571,230 @@ fn managed_service_reconcile(mut args: Args) -> Result<()> { service_launch(args) } +fn reconcile_workspace_coverage_projector( + mut args: Args, + policy: EpiphanyCultMeshManagedServicePolicyEntry, +) -> Result<()> { + if !policy.enabled || policy.restart_mode == "never" { + println!( + "{}", + serde_json::to_string_pretty(&json!({ + "schemaVersion": "epiphany.workspace_coverage_reconcile.v0", + "status": "disabled", + "serviceId": policy.service_id, + "restarted": false, + "privateStateExposed": false, + }))? + ); + return Ok(()); + } + let runtime_store = args + .runtime_store + .clone() + .context("workspace coverage reconciliation requires --runtime-store")?; + let Some(target) = current_workspace_coverage_recovery_target(&runtime_store)? else { + let latest = load_latest_workspace_coverage_managed_process_launch( + &args.store, + args.runtime_id.clone(), + )?; + if let Some(latest) = latest { + let host = open_default_host_identity() + .context("workspace coverage reconciliation requires an enrolled host identity")?; + if load_workspace_coverage_process_termination_observation( + &args.store, + args.runtime_id.clone(), + &latest.launch_id, + )? + .is_none() + && let Err(error) = write_workspace_coverage_process_termination_observation( + &args.store, + args.runtime_id.clone(), + &latest.launch_id, + &host, + ) + { + println!( + "{}", + serde_json::to_string_pretty(&json!({ + "schemaVersion": "epiphany.workspace_coverage_reconcile.v0", + "status": "awaiting-first-claim", + "serviceId": policy.service_id, + "launchId": latest.launch_id, + "reason": error.to_string(), + "restarted": false, + "privateStateExposed": false, + }))? + ); + return Ok(()); + } + let (termination, termination_digest) = + authenticate_workspace_coverage_termination_with_envelope_digest( + &args.store, + args.runtime_id.clone(), + &latest.launch_id, + host.entry(), + )?; + if load_latest_workspace_coverage_managed_process_launch( + &args.store, + args.runtime_id.clone(), + )? + .is_some_and(|candidate| { + candidate.replaces_termination_id.as_deref() + == Some(termination.termination_id.as_str()) + }) { + println!( + "{}", + serde_json::to_string_pretty(&json!({ + "schemaVersion": "epiphany.workspace_coverage_reconcile.v0", + "status": "awaiting-first-claim", + "serviceId": policy.service_id, + "restarted": false, + "privateStateExposed": false, + }))? + ); + return Ok(()); + } + args.service_command = Some(PathBuf::from(&policy.command)); + args.service_args = policy.args; + args.cwd = policy.cwd.map(PathBuf::from); + args.stdout_artifact = Some(PathBuf::from(policy.stdout_artifact)); + args.stderr_artifact = Some(PathBuf::from(policy.stderr_artifact)); + let replacement = CoverageReplacementEvidence { + old_launch_id: latest.launch_id, + termination_id: termination.termination_id, + termination_envelope_digest: termination_digest, + }; + let _ = service_launch_internal(args, Some(replacement), true)?; + return Ok(()); + } else { + args.service_command = Some(PathBuf::from(&policy.command)); + args.service_args = policy.args; + args.cwd = policy.cwd.map(PathBuf::from); + args.stdout_artifact = Some(PathBuf::from(policy.stdout_artifact)); + args.stderr_artifact = Some(PathBuf::from(policy.stderr_artifact)); + let _ = service_launch_internal(args, None, true)?; + return Ok(()); + } + }; + + let host = open_default_host_identity() + .context("workspace coverage reconciliation requires an enrolled host identity")?; + if load_workspace_coverage_process_termination_observation( + &args.store, + args.runtime_id.clone(), + &target.managed_process_launch_id, + )? + .is_none() + { + if let Err(error) = write_workspace_coverage_process_termination_observation( + &args.store, + args.runtime_id.clone(), + &target.managed_process_launch_id, + &host, + ) { + println!( + "{}", + serde_json::to_string_pretty(&json!({ + "schemaVersion": "epiphany.workspace_coverage_reconcile.v0", + "status": "awaiting-exact-termination", + "serviceId": policy.service_id, + "claimId": target.claim_id, + "reason": error.to_string(), + "restarted": false, + "privateStateExposed": false, + }))? + ); + return Ok(()); + } + } + let (termination, termination_digest) = + authenticate_workspace_coverage_termination_with_envelope_digest( + &args.store, + args.runtime_id.clone(), + &target.managed_process_launch_id, + host.entry(), + )?; + let replacement_evidence = CoverageReplacementEvidence { + old_launch_id: target.managed_process_launch_id.clone(), + termination_id: termination.termination_id.clone(), + termination_envelope_digest: termination_digest, + }; + let existing_replacement = load_latest_workspace_coverage_managed_process_launch( + &args.store, + args.runtime_id.clone(), + )? + .filter(|launch| { + launch.replaces_launch_id.as_deref() == Some(target.managed_process_launch_id.as_str()) + && launch.replaces_termination_id.as_deref() + == Some(termination.termination_id.as_str()) + }); + let replacement = if let Some(existing) = existing_replacement { + existing + } else { + args.service_command = Some(PathBuf::from(&policy.command)); + args.service_args = policy.args; + args.cwd = policy.cwd.map(PathBuf::from); + args.stdout_artifact = Some(PathBuf::from(policy.stdout_artifact)); + args.stderr_artifact = Some(PathBuf::from(policy.stderr_artifact)); + service_launch_internal(args.clone(), Some(replacement_evidence), false)? + .coverage_launch + .context("workspace coverage replacement launch returned no specialized identity")? + }; + + let ready = (0..100).find_map(|_| { + let observed = load_latest_workspace_coverage_provider_heartbeat( + &args.store, + args.runtime_id.clone(), + &replacement.launch_id, + ) + .ok() + .flatten(); + if let Some(heartbeat) = observed + && heartbeat.status == "ready" + && authenticate_workspace_coverage_provider_heartbeat( + &args.store, + args.runtime_id.clone(), + &heartbeat.heartbeat_id, + host.entry(), + ) + .is_ok() + { + return Some(heartbeat); + } + thread::sleep(StdDuration::from_millis(100)); + None + }); + let Some(ready) = ready else { + anyhow::bail!("workspace coverage replacement did not publish signed readiness in time"); + }; + let recovered = recover_workspace_coverage_projection( + &runtime_store, + &args.store, + &args.runtime_id, + host.entry(), + &target.managed_process_launch_id, + &replacement.launch_id, + &ready.heartbeat_id, + &target.claim_id, + )?; + println!( + "{}", + serde_json::to_string_pretty(&json!({ + "schemaVersion": "epiphany.workspace_coverage_reconcile.v0", + "status": "recovered", + "serviceId": policy.service_id, + "oldClaimId": target.claim_id, + "newClaimId": recovered.claim_id, + "claimEpoch": recovered.claim_epoch, + "replacementLaunchId": replacement.launch_id, + "recoveryReceiptId": recovered.recovery_receipt_id, + "restarted": true, + "privateStateExposed": false, + }))? + ); + Ok(()) +} + fn service_runbook(args: Args) -> Result<()> { require_supervisor_bootstrap(&args)?; @@ -2960,7 +3231,7 @@ mod semantic_projector_authority_tests { #[test] fn reserved_projector_launch_seals_spawn_before_the_child_may_publish() { let source = include_str!("epiphany-daemon-supervisor.rs"); - let start = source.find("fn service_launch(args: Args)").unwrap(); + let start = source.find("fn service_launch_internal(").unwrap(); let tail = &source[start..]; let end = tail.find("\nfn ").unwrap(); let body = &tail[..end]; @@ -2983,7 +3254,7 @@ mod semantic_projector_authority_tests { #[test] fn workspace_coverage_launch_uses_only_specialized_process_authority() { let source = include_str!("epiphany-daemon-supervisor.rs"); - let start = source.find("fn service_launch(args: Args)").unwrap(); + let start = source.find("fn service_launch_internal(").unwrap(); let tail = &source[start..]; let end = tail.find("\nfn ").unwrap(); let body = &tail[..end]; @@ -3004,6 +3275,28 @@ mod semantic_projector_authority_tests { assert!(body.contains("let _ = child.wait()")); } + #[test] + fn workspace_coverage_reconcile_orders_termination_launch_readiness_and_body_transfer() { + let source = include_str!("epiphany-daemon-supervisor.rs"); + let start = source + .find("fn reconcile_workspace_coverage_projector(") + .unwrap(); + let tail = &source[start..]; + let end = tail.find("\nfn service_runbook").unwrap(); + let body = &tail[..end]; + let termination = body + .rfind("authenticate_workspace_coverage_termination_with_envelope_digest") + .unwrap(); + let launch = body.rfind("service_launch_internal").unwrap(); + let readiness = body + .rfind("load_latest_workspace_coverage_provider_heartbeat") + .unwrap(); + let recovery = body.rfind("recover_workspace_coverage_projection").unwrap(); + assert!(termination < launch && launch < readiness && readiness < recovery); + assert!(body.contains("write_workspace_coverage_process_termination_observation")); + assert!(body.contains("status == \"ready\"")); + } + #[test] fn reserved_projector_policy_has_no_generic_or_command_override_path() { let source = include_str!("epiphany-daemon-supervisor.rs"); diff --git a/epiphany-core/src/lib.rs b/epiphany-core/src/lib.rs index ee9052e1..3355ccf5 100644 --- a/epiphany-core/src/lib.rs +++ b/epiphany-core/src/lib.rs @@ -1168,6 +1168,7 @@ pub use workspace_coverage_process_documents::{ authenticate_workspace_coverage_managed_process_launch, authenticate_workspace_coverage_process_termination_observation, authenticate_workspace_coverage_provider_heartbeat, + authenticate_workspace_coverage_termination_with_envelope_digest, load_latest_workspace_coverage_managed_process_launch, load_latest_workspace_coverage_provider_heartbeat, load_workspace_coverage_managed_process_launch, @@ -1181,6 +1182,11 @@ pub use workspace_coverage_process_documents::{ write_workspace_coverage_process_termination_observation, write_workspace_coverage_provider_heartbeat, }; +pub use workspace_coverage_projector::{ + WorkspaceCoverageRecoveryOutcome, WorkspaceCoverageRecoveryTarget, + authenticate_workspace_coverage_recovery_receipt, current_workspace_coverage_recovery_target, + recover_workspace_coverage_projection, +}; pub use workspace_coverage_projector_service::WorkspaceCoverageProjectorConfig; pub use workspace_coverage_projector_service::WorkspaceCoverageProjectorPulseStatus; pub use workspace_coverage_projector_service::WorkspaceCoverageProjectorServiceBody; diff --git a/epiphany-core/src/workspace_coverage_process_documents.rs b/epiphany-core/src/workspace_coverage_process_documents.rs index f3a2f37d..a3b63b65 100644 --- a/epiphany-core/src/workspace_coverage_process_documents.rs +++ b/epiphany-core/src/workspace_coverage_process_documents.rs @@ -18,7 +18,7 @@ use std::path::Path; pub const WORKSPACE_COVERAGE_PROCESS_LAUNCH_TYPE: &str = "epiphany.workspace_coverage.managed_process_launch"; pub const WORKSPACE_COVERAGE_PROCESS_LAUNCH_SCHEMA_VERSION: &str = - "epiphany.workspace_coverage.managed_process_launch.v0"; + "epiphany.workspace_coverage.managed_process_launch.v1"; pub const WORKSPACE_COVERAGE_PROCESS_LAUNCH_LATEST_KEY: &str = "epiphany-local/workspace-coverage/managed-process-launch/latest"; pub const WORKSPACE_COVERAGE_PROVIDER_HEARTBEAT_TYPE: &str = @@ -99,6 +99,12 @@ pub struct WorkspaceCoverageManagedProcessLaunchEntry { pub identity_captured_at_utc: String, #[cultcache(key = 26)] pub signature_algorithm: String, + #[cultcache(key = 27, default)] + pub replaces_launch_id: Option, + #[cultcache(key = 28, default)] + pub replaces_termination_id: Option, + #[cultcache(key = 29, default)] + pub replaces_termination_envelope_digest: Option, } #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] @@ -252,6 +258,9 @@ struct LaunchStatement<'a> { supervisor_id: &'a str, identity_captured_at_utc: &'a str, signature_algorithm: &'a str, + replaces_launch_id: &'a Option, + replaces_termination_id: &'a Option, + replaces_termination_envelope_digest: &'a Option, } #[derive(Serialize)] @@ -349,6 +358,9 @@ pub fn workspace_coverage_launch_statement( supervisor_id: &entry.supervisor_id, identity_captured_at_utc: &entry.identity_captured_at_utc, signature_algorithm: &entry.signature_algorithm, + replaces_launch_id: &entry.replaces_launch_id, + replaces_termination_id: &entry.replaces_termination_id, + replaces_termination_envelope_digest: &entry.replaces_termination_envelope_digest, })?) } @@ -504,6 +516,43 @@ pub fn write_workspace_coverage_managed_process_launch( policy_envelope, node.cache().prepare_entry(&identity_key, &entry)?.0, ]; + if let (Some(old_launch_id), Some(termination_id), Some(termination_digest)) = ( + entry.replaces_launch_id.as_deref(), + entry.replaces_termination_id.as_deref(), + entry.replaces_termination_envelope_digest.as_deref(), + ) { + let termination_envelope = node + .cache() + .get_envelope::(&termination_key( + old_launch_id, + ))? + .ok_or_else(|| anyhow!("workspace coverage replacement termination is absent"))?; + let termination: WorkspaceCoverageProcessTerminationObservationEntry = + rmp_serde::from_slice(&termination_envelope.payload)?; + authenticate_workspace_coverage_process_termination_observation( + store_path, + entry.runtime_id.clone(), + old_launch_id, + host_identity, + )?; + if termination.termination_id != termination_id + || envelope_digest(&termination_envelope) != termination_digest + { + bail!("workspace coverage replacement launch disagrees with exact termination"); + } + let replacement_key = + format!("epiphany-local/workspace-coverage/replacement-for/{old_launch_id}"); + if node + .cache() + .get_envelope::(&replacement_key)? + .is_some() + { + bail!("workspace coverage termination already has a replacement launch"); + } + expected.push(termination_envelope.clone()); + replacements.push(termination_envelope); + replacements.push(node.cache().prepare_entry(&replacement_key, &entry)?.0); + } if let Some(latest) = node .cache() .get_envelope::( @@ -717,6 +766,46 @@ pub fn authenticate_workspace_coverage_provider_heartbeat( Ok(heartbeat) } +pub(crate) fn authenticate_workspace_coverage_managed_process_launch_with_envelope_digest( + store_path: impl AsRef, + runtime_id: impl Into, + launch_id: &str, + host_identity: &HostIncarnationIdentityEntry, +) -> Result<(WorkspaceCoverageManagedProcessLaunchEntry, String)> { + let runtime_id = runtime_id.into(); + let launch = authenticate_workspace_coverage_managed_process_launch( + store_path.as_ref(), + runtime_id.clone(), + launch_id, + host_identity, + )?; + let envelope = open_epiphany_cultmesh_node(store_path, runtime_id)? + .cache() + .get_envelope::(&launch_key(launch_id))? + .ok_or_else(|| anyhow!("workspace coverage launch envelope disappeared"))?; + Ok((launch, envelope_digest(&envelope))) +} + +pub(crate) fn authenticate_workspace_coverage_provider_heartbeat_with_envelope_digest( + store_path: impl AsRef, + runtime_id: impl Into, + heartbeat_id: &str, + host_identity: &HostIncarnationIdentityEntry, +) -> Result<(WorkspaceCoverageProviderHeartbeatEntry, String)> { + let runtime_id = runtime_id.into(); + let heartbeat = authenticate_workspace_coverage_provider_heartbeat( + store_path.as_ref(), + runtime_id.clone(), + heartbeat_id, + host_identity, + )?; + let envelope = open_epiphany_cultmesh_node(store_path, runtime_id)? + .cache() + .get_envelope::(&heartbeat_key(heartbeat_id))? + .ok_or_else(|| anyhow!("workspace coverage heartbeat envelope disappeared"))?; + Ok((heartbeat, envelope_digest(&envelope))) +} + pub fn write_workspace_coverage_process_termination_observation( store_path: impl AsRef, runtime_id: impl Into, @@ -926,6 +1015,29 @@ pub fn authenticate_workspace_coverage_process_termination_observation( Ok(entry) } +pub fn authenticate_workspace_coverage_termination_with_envelope_digest( + store_path: impl AsRef, + runtime_id: impl Into, + launch_id: &str, + host: &HostIncarnationIdentityEntry, +) -> Result<(WorkspaceCoverageProcessTerminationObservationEntry, String)> { + let store_path = store_path.as_ref(); + let runtime_id = runtime_id.into(); + let entry = authenticate_workspace_coverage_process_termination_observation( + store_path, + runtime_id.clone(), + launch_id, + host, + )?; + let envelope = open_epiphany_cultmesh_node(store_path, runtime_id)? + .cache() + .get_envelope::(&termination_key( + launch_id, + ))? + .ok_or_else(|| anyhow!("workspace coverage termination envelope disappeared"))?; + Ok((entry, envelope_digest(&envelope))) +} + fn validate_launch( entry: &WorkspaceCoverageManagedProcessLaunchEntry, host: &HostIncarnationIdentityEntry, @@ -937,6 +1049,19 @@ fn validate_launch( { bail!("workspace coverage launch violates its reserved schema or authority"); } + match ( + entry.replaces_launch_id.as_deref(), + entry.replaces_termination_id.as_deref(), + entry.replaces_termination_envelope_digest.as_deref(), + ) { + (None, None, None) => {} + (Some(launch_id), Some(termination_id), Some(digest)) + if !launch_id.trim().is_empty() && !termination_id.trim().is_empty() => + { + validate_digest("replacement termination", digest)?; + } + _ => bail!("workspace coverage replacement launch has a partial causal edge"), + } for (name, value) in [ ("launch id", entry.launch_id.as_str()), ("runtime id", entry.runtime_id.as_str()), @@ -1282,7 +1407,9 @@ mod tests { EPIPHANY_CULTMESH_MANAGED_SERVICE_POLICY_SCHEMA_VERSION, enroll_host_identity_at, write_epiphany_cultmesh_workspace_coverage_projector_service_policy, }; + use cultcache_rs::CacheBackingStore; use rand_core::{OsRng, RngCore}; + use std::process::Command; use uuid::Uuid; struct FakeObservation { @@ -1392,6 +1519,9 @@ mod tests { supervisor_id: "epiphany-daemon-supervisor".to_string(), identity_captured_at_utc: now, signature_algorithm: "ed25519".to_string(), + replaces_launch_id: None, + replaces_termination_id: None, + replaces_termination_envelope_digest: None, }; sign_workspace_coverage_launch(&mut entry, host)?; Ok(entry) @@ -1685,4 +1815,279 @@ mod tests { } Ok(()) } + + #[test] + fn body_recovery_is_one_exact_evidence_fenced_transaction() -> Result<()> { + let temp = tempfile::tempdir()?; + let (verse, host, old_launch) = persisted_chain(temp.path())?; + + let repo = temp.path().join("repo"); + std::fs::create_dir_all(&repo)?; + Command::new("git") + .args(["init"]) + .current_dir(&repo) + .output()?; + std::fs::write(repo.join("source.rs"), "fn awake() {}\n")?; + Command::new("git") + .args(["add", "."]) + .current_dir(&repo) + .output()?; + let runtime = temp.path().join("runtime.ccmp"); + let agents = temp.path().join("agents.ccmp"); + let body_store = temp.path().join("body.ccmp"); + crate::initialize_runtime_spine( + &runtime, + crate::RuntimeSpineInitOptions { + runtime_id: "local".into(), + display_name: "recovery-test".into(), + created_at: "2026-07-16T00:00:00Z".into(), + }, + )?; + crate::ensure_agent_memory_swarm_identity(&agents, "recovery-swarm")?; + crate::bind_runtime_to_agent_memory_swarm(&runtime, &agents, "2026-07-16T00:00:01Z")?; + crate::bind_repository_body(&repo, &body_store, &runtime, "recovery-workspace")?; + let basis = crate::observe_runtime_repository_body_basis(&runtime)?; + let session = crate::RepositoryBodyReadSession::open(&runtime, &basis)?; + let prepared = crate::workspace_coverage_projector::prepare_workspace_coverage_projection( + &session, + "test-provider", + "test-model", + 3, + )?; + let acquired = + match crate::workspace_coverage_projector::acquire_workspace_coverage_projection( + &prepared, + EPIPHANY_WORKSPACE_COVERAGE_PROJECTOR_DAEMON_ID, + &old_launch.provider_incarnation_id, + &old_launch.launch_id, + )? { + crate::workspace_coverage_projector::WorkspaceCoverageAcquireResult::Acquired( + value, + ) => value, + _ => bail!("fixture did not acquire old claim"), + }; + + let source = FakeObservation { + boot: Some(old_launch.boot_identity.clone()), + process: ProcessInstanceObservation::Missing, + }; + let termination = write_workspace_coverage_process_termination_observation_with_source( + &verse, + "local", + &old_launch.launch_id, + &host, + &source, + )?; + let (_, termination_digest) = + authenticate_workspace_coverage_termination_with_envelope_digest( + &verse, + "local", + &old_launch.launch_id, + host.entry(), + )?; + + let node = open_epiphany_cultmesh_node(&verse, "local")?; + let policy_envelope = node + .cache() + .get_envelope::(&managed_policy_key())? + .context("policy missing")?; + let policy: EpiphanyCultMeshManagedServicePolicyEntry = + rmp_serde::from_slice(&policy_envelope.payload)?; + let body_before_refusal = std::fs::read(&body_store)?; + let out_of_order_provider = provider_key(); + let mut out_of_order = launch( + &policy, + envelope_digest(&policy_envelope), + &host, + &out_of_order_provider, + )?; + out_of_order.launched_at_utc = termination.observed_at_utc.clone(); + out_of_order.identity_captured_at_utc = termination.observed_at_utc.clone(); + sign_workspace_coverage_launch(&mut out_of_order, &host)?; + write_workspace_coverage_managed_process_launch( + &verse, + "local", + out_of_order.clone(), + host.entry(), + )?; + let out_of_order_envelope = open_epiphany_cultmesh_node(&verse, "local")? + .cache() + .get_envelope::(&launch_key( + &out_of_order.launch_id, + ))? + .context("out-of-order launch missing")?; + let out_of_order_ready = heartbeat( + &out_of_order, + envelope_digest(&out_of_order_envelope), + &out_of_order_provider, + 1, + )?; + write_workspace_coverage_provider_heartbeat(&verse, "local", out_of_order_ready.clone())?; + assert!( + crate::workspace_coverage_projector::recover_workspace_coverage_projection( + &runtime, + &verse, + "local", + host.entry(), + &old_launch.launch_id, + &out_of_order.launch_id, + &out_of_order_ready.heartbeat_id, + &acquired.claim.claim_id, + ) + .is_err(), + "unbound replacement must be refused" + ); + assert_eq!(std::fs::read(&body_store)?, body_before_refusal); + + let replacement_provider = provider_key(); + let mut replacement = launch( + &policy, + envelope_digest(&policy_envelope), + &host, + &replacement_provider, + )?; + replacement.replaces_launch_id = Some(old_launch.launch_id.clone()); + replacement.replaces_termination_id = Some(termination.termination_id.clone()); + replacement.replaces_termination_envelope_digest = Some(termination_digest); + sign_workspace_coverage_launch(&mut replacement, &host)?; + write_workspace_coverage_managed_process_launch( + &verse, + "local", + replacement.clone(), + host.entry(), + )?; + let competing_provider = provider_key(); + let mut competing = launch( + &policy, + envelope_digest(&policy_envelope), + &host, + &competing_provider, + )?; + competing.replaces_launch_id = Some(old_launch.launch_id.clone()); + competing.replaces_termination_id = Some(termination.termination_id.clone()); + competing.replaces_termination_envelope_digest = + replacement.replaces_termination_envelope_digest.clone(); + sign_workspace_coverage_launch(&mut competing, &host)?; + assert!( + write_workspace_coverage_managed_process_launch( + &verse, + "local", + competing, + host.entry(), + ) + .is_err(), + "one termination must authorize at most one replacement launch" + ); + let replacement_envelope = open_epiphany_cultmesh_node(&verse, "local")? + .cache() + .get_envelope::(&launch_key( + &replacement.launch_id, + ))? + .context("replacement launch missing")?; + let initial_ready = heartbeat( + &replacement, + envelope_digest(&replacement_envelope), + &replacement_provider, + 1, + )?; + write_workspace_coverage_provider_heartbeat(&verse, "local", initial_ready)?; + + let body_before_refusal = std::fs::read(&body_store)?; + let mut degraded = heartbeat( + &replacement, + envelope_digest(&replacement_envelope), + &replacement_provider, + 2, + )?; + degraded.status = "degraded".into(); + sign_workspace_coverage_heartbeat(&mut degraded, &replacement_provider)?; + write_workspace_coverage_provider_heartbeat(&verse, "local", degraded.clone())?; + assert!( + crate::workspace_coverage_projector::recover_workspace_coverage_projection( + &runtime, + &verse, + "local", + host.entry(), + &old_launch.launch_id, + &replacement.launch_id, + °raded.heartbeat_id, + &acquired.claim.claim_id, + ) + .is_err(), + "degraded replacement must not inherit Body authority" + ); + assert_eq!(std::fs::read(&body_store)?, body_before_refusal); + + let ready = heartbeat( + &replacement, + envelope_digest(&replacement_envelope), + &replacement_provider, + 3, + )?; + write_workspace_coverage_provider_heartbeat(&verse, "local", ready.clone())?; + + let recovered = crate::workspace_coverage_projector::recover_workspace_coverage_projection( + &runtime, + &verse, + "local", + host.entry(), + &old_launch.launch_id, + &replacement.launch_id, + &ready.heartbeat_id, + &acquired.claim.claim_id, + )?; + assert_eq!(recovered.claim_epoch, acquired.claim.claim_epoch + 1); + assert_eq!(recovered.managed_process_launch_id, replacement.launch_id); + assert_eq!( + recovered.executor_incarnation, + replacement.provider_incarnation_id + ); + let opening = SingleFileMessagePackBackingStore::new(&body_store).pull_all()?; + let old_history: crate::workspace_coverage_projector::WorkspaceCoverageProjectionClaim = + rmp_serde::from_slice( + &opening + .iter() + .find(|entry| { + entry.r#type == "gamecult.epiphany.workspace_coverage_projection_claim" + && entry.key == format!("history/{}", acquired.claim.claim_id) + }) + .context("failed claim history missing")? + .payload, + )?; + assert_eq!(old_history.status, "failed"); + assert!(old_history.termination_evidence_digest.is_some()); + let recovery_id = recovered.recovery_receipt_id.as_str(); + let recovery_digest = recovered.recovery_receipt_digest.as_str(); + crate::workspace_coverage_projector::authenticate_workspace_coverage_recovery_receipt( + &body_store, + &verse, + "local", + host.entry(), + recovery_id, + recovery_digest, + )?; + assert_eq!( + old_history.recovery_receipt_id.as_deref(), + Some(recovery_id) + ); + assert_eq!( + old_history.recovery_receipt_digest.as_deref(), + Some(recovery_digest) + ); + assert!( + crate::workspace_coverage_projector::recover_workspace_coverage_projection( + &runtime, + &verse, + "local", + host.entry(), + &old_launch.launch_id, + &replacement.launch_id, + &ready.heartbeat_id, + &acquired.claim.claim_id, + ) + .is_err(), + "moved claim/attempt must refuse replay" + ); + Ok(()) + } } diff --git a/epiphany-core/src/workspace_coverage_projector.rs b/epiphany-core/src/workspace_coverage_projector.rs index 1268b8e5..73bf53c9 100644 --- a/epiphany-core/src/workspace_coverage_projector.rs +++ b/epiphany-core/src/workspace_coverage_projector.rs @@ -4,15 +4,22 @@ use crate::semantic_backend::{ }; use crate::{ BODY_BINDING_KEY, BODY_BINDING_TYPE, BODY_HEAD_KEY, BODY_HEAD_TYPE, BODY_MANIFEST_TYPE, - BODY_OBSERVATION_TYPE, RepositoryBodyHead, RepositoryBodyObservationBasis, - WORKSPACE_COVERAGE_HEAD_SCHEMA_VERSION, WORKSPACE_COVERAGE_RECEIPT_SCHEMA_VERSION, - WorkspaceCoverageChunkDescriptor, WorkspaceCoverageHead, WorkspaceCoverageObligation, - WorkspaceCoveragePointBinding, WorkspaceCoveragePointPayload, WorkspaceCoveragePolicy, - WorkspaceCoverageProjectionPlan, WorkspaceCoverageReceipt, WorkspaceCoverageVectorBinding, + BODY_OBSERVATION_TYPE, HostIncarnationIdentityEntry, RepositoryBodyHead, + RepositoryBodyObservationBasis, WORKSPACE_COVERAGE_HEAD_SCHEMA_VERSION, + WORKSPACE_COVERAGE_RECEIPT_SCHEMA_VERSION, WorkspaceCoverageChunkDescriptor, + WorkspaceCoverageHead, WorkspaceCoverageObligation, WorkspaceCoveragePointBinding, + WorkspaceCoveragePointPayload, WorkspaceCoveragePolicy, WorkspaceCoverageProjectionPlan, + WorkspaceCoverageReceipt, WorkspaceCoverageVectorBinding, derive_workspace_coverage_obligation_from_authenticated_manifest, - derive_workspace_coverage_projection_plan, refine_workspace_coverage_obligation_utf8, + derive_workspace_coverage_projection_plan, load_latest_workspace_coverage_provider_heartbeat, + observe_runtime_repository_body_basis, refine_workspace_coverage_obligation_utf8, runtime_repository_body_store_binding, validate_workspace_coverage_head, validate_workspace_coverage_projection_plan, workspace_coverage_execution_collection, + workspace_coverage_process_documents::{ + authenticate_workspace_coverage_managed_process_launch_with_envelope_digest, + authenticate_workspace_coverage_provider_heartbeat_with_envelope_digest, + authenticate_workspace_coverage_termination_with_envelope_digest, + }, }; use anyhow::{Result, anyhow, bail}; use cultcache_rs::{ @@ -27,6 +34,12 @@ const ATTEMPT_TYPE: &str = "gamecult.epiphany.workspace_coverage_projection_atte const CLAIM_KEY: &str = "workspace-coverage-projector-current"; const CLAIM_SCHEMA: &str = "gamecult.epiphany.workspace_coverage_projection_claim.v1"; const ATTEMPT_SCHEMA: &str = "gamecult.epiphany.workspace_coverage_projection_attempt.v1"; +const CLAIM_SCHEMA_V2: &str = "gamecult.epiphany.workspace_coverage_projection_claim.v2"; +const ATTEMPT_SCHEMA_V2: &str = "gamecult.epiphany.workspace_coverage_projection_attempt.v2"; +const CLAIM_SCHEMA_V3: &str = "gamecult.epiphany.workspace_coverage_projection_claim.v3"; +const ATTEMPT_SCHEMA_V3: &str = "gamecult.epiphany.workspace_coverage_projection_attempt.v3"; +const RECOVERY_TYPE: &str = "gamecult.epiphany.workspace_coverage_recovery_receipt"; +const RECOVERY_SCHEMA: &str = "gamecult.epiphany.workspace_coverage_recovery_receipt.v0"; const PROJECTION_SCHEMA: &str = "gamecult.epiphany.workspace_bytes_projection.v0"; const CHUNKER_ID: &str = "utf8_lines_96_overlap_8_v0"; const MAXIMUM_FILE_BYTES: u64 = 4 * 1024 * 1024; @@ -54,6 +67,23 @@ pub(crate) struct WorkspaceCoverageObservedBinding { vector_binding_set_sha256: String, } +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct WorkspaceCoverageRecoveryTarget { + pub claim_id: String, + pub claim_epoch: u64, + pub managed_process_launch_id: String, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct WorkspaceCoverageRecoveryOutcome { + pub claim_id: String, + pub claim_epoch: u64, + pub managed_process_launch_id: String, + pub executor_incarnation: String, + pub recovery_receipt_id: String, + pub recovery_receipt_digest: String, +} + pub(crate) trait WorkspaceCoverageProjectionPort { fn ensure_exact_collection( &mut self, @@ -127,9 +157,9 @@ pub(crate) struct WorkspaceCoverageProjectionClaim { #[cultcache(key = 0)] schema_version: String, #[cultcache(key = 1)] - claim_id: String, + pub(crate) claim_id: String, #[cultcache(key = 2)] - claim_epoch: u64, + pub(crate) claim_epoch: u64, #[cultcache(key = 3)] plan_id: String, #[cultcache(key = 4)] @@ -143,18 +173,28 @@ pub(crate) struct WorkspaceCoverageProjectionClaim { #[cultcache(key = 8)] manifest_root_sha256: String, #[cultcache(key = 9)] - status: String, + pub(crate) status: String, #[cultcache(key = 10)] executor_id: String, #[cultcache(key = 11)] - executor_incarnation: String, - #[cultcache(key = 12)] - managed_process_launch_id: String, + pub(crate) executor_incarnation: String, + #[cultcache(key = 12, default)] + pub(crate) managed_process_launch_id: String, + #[cultcache(key = 13, default)] + termination_evidence_id: Option, + #[cultcache(key = 14, default)] + pub(crate) termination_evidence_digest: Option, + #[cultcache(key = 15, default)] + pub(crate) recovery_receipt_id: Option, + #[cultcache(key = 16, default)] + pub(crate) recovery_receipt_digest: Option, } fn validate_projection_claim(claim: &WorkspaceCoverageProjectionClaim) -> Result<()> { - if claim.schema_version != CLAIM_SCHEMA - || claim.claim_id.trim().is_empty() + if !matches!( + claim.schema_version.as_str(), + CLAIM_SCHEMA | CLAIM_SCHEMA_V2 | CLAIM_SCHEMA_V3 + ) || claim.claim_id.trim().is_empty() || claim.claim_epoch == 0 || claim.plan_id.trim().is_empty() || claim.attempt_id.trim().is_empty() @@ -168,12 +208,42 @@ fn validate_projection_claim(claim: &WorkspaceCoverageProjectionClaim) -> Result { bail!("invalid workspace coverage projection claim"); } + match claim.status.as_str() { + "failed" if claim.schema_version == CLAIM_SCHEMA_V2 => { + validate_terminal_evidence( + &claim.termination_evidence_id, + &claim.termination_evidence_digest, + )?; + } + _ if claim.termination_evidence_id.is_some() + || claim.termination_evidence_digest.is_some() => + { + bail!("workspace coverage claim has misplaced termination evidence") + } + _ => {} + } + if claim.schema_version == CLAIM_SCHEMA_V2 && claim.status != "failed" { + bail!("workspace coverage recovery claim schema is terminal-only"); + } + match claim.schema_version.as_str() { + CLAIM_SCHEMA + if claim.recovery_receipt_id.is_some() || claim.recovery_receipt_digest.is_some() => + { + bail!("ordinary workspace coverage claim cites recovery evidence") + } + CLAIM_SCHEMA_V2 | CLAIM_SCHEMA_V3 => { + validate_terminal_evidence(&claim.recovery_receipt_id, &claim.recovery_receipt_digest)? + } + _ => {} + } Ok(()) } fn validate_projection_attempt(attempt: &WorkspaceCoverageProjectionAttempt) -> Result<()> { - if attempt.schema_version != ATTEMPT_SCHEMA - || attempt.attempt_id.trim().is_empty() + if !matches!( + attempt.schema_version.as_str(), + ATTEMPT_SCHEMA | ATTEMPT_SCHEMA_V2 | ATTEMPT_SCHEMA_V3 + ) || attempt.attempt_id.trim().is_empty() || attempt.claim_id.trim().is_empty() || attempt.claim_epoch == 0 || attempt.plan_id.trim().is_empty() @@ -202,7 +272,54 @@ fn validate_projection_attempt(attempt: &WorkspaceCoverageProjectionAttempt) -> _ => {} } if let Some(value) = &attempt.completed_at { - chrono::DateTime::parse_from_rfc3339(value)?; + let completed = chrono::DateTime::parse_from_rfc3339(value)?; + let started = chrono::DateTime::parse_from_rfc3339(&attempt.started_at)?; + if completed < started { + bail!("workspace coverage attempt completes before it starts"); + } + } + match attempt.status.as_str() { + "failed" if attempt.schema_version == ATTEMPT_SCHEMA_V2 => { + validate_terminal_evidence( + &attempt.termination_evidence_id, + &attempt.termination_evidence_digest, + )?; + } + _ if attempt.termination_evidence_id.is_some() + || attempt.termination_evidence_digest.is_some() => + { + bail!("workspace coverage attempt has misplaced termination evidence") + } + _ => {} + } + if attempt.schema_version == ATTEMPT_SCHEMA_V2 && attempt.status != "failed" { + bail!("workspace coverage recovery attempt schema is terminal-only"); + } + match attempt.schema_version.as_str() { + ATTEMPT_SCHEMA + if attempt.recovery_receipt_id.is_some() + || attempt.recovery_receipt_digest.is_some() => + { + bail!("ordinary workspace coverage attempt cites recovery evidence") + } + ATTEMPT_SCHEMA_V2 | ATTEMPT_SCHEMA_V3 => validate_terminal_evidence( + &attempt.recovery_receipt_id, + &attempt.recovery_receipt_digest, + )?, + _ => {} + } + Ok(()) +} + +fn validate_terminal_evidence(id: &Option, digest: &Option) -> Result<()> { + let (Some(id), Some(digest)) = (id.as_deref(), digest.as_deref()) else { + bail!("recovered workspace coverage terminal state lacks exact evidence") + }; + let Some(hex) = digest.strip_prefix("sha256-") else { + bail!("recovered workspace coverage terminal evidence is invalid") + }; + if id.trim().is_empty() || hex.len() != 64 || !hex.bytes().all(|b| b.is_ascii_hexdigit()) { + bail!("recovered workspace coverage terminal evidence is invalid") } Ok(()) } @@ -214,6 +331,15 @@ fn validate_claim_attempt_link( validate_projection_claim(claim)?; validate_projection_attempt(attempt)?; if claim.claim_id != attempt.claim_id + || !matches!( + ( + claim.schema_version.as_str(), + attempt.schema_version.as_str() + ), + (CLAIM_SCHEMA, ATTEMPT_SCHEMA) + | (CLAIM_SCHEMA_V2, ATTEMPT_SCHEMA_V2) + | (CLAIM_SCHEMA_V3, ATTEMPT_SCHEMA_V3) + ) || claim.claim_epoch != attempt.claim_epoch || claim.attempt_id != attempt.attempt_id || claim.plan_id != attempt.plan_id @@ -221,6 +347,10 @@ fn validate_claim_attempt_link( || claim.executor_id != attempt.executor_id || claim.executor_incarnation != attempt.executor_incarnation || claim.managed_process_launch_id != attempt.managed_process_launch_id + || claim.termination_evidence_id != attempt.termination_evidence_id + || claim.termination_evidence_digest != attempt.termination_evidence_digest + || claim.recovery_receipt_id != attempt.recovery_receipt_id + || claim.recovery_receipt_digest != attempt.recovery_receipt_digest { bail!("workspace coverage claim/attempt authority is split"); } @@ -257,6 +387,169 @@ pub(crate) struct WorkspaceCoverageProjectionAttempt { executor_incarnation: String, #[cultcache(key = 11)] managed_process_launch_id: String, + #[cultcache(key = 12, default)] + termination_evidence_id: Option, + #[cultcache(key = 13, default)] + termination_evidence_digest: Option, + #[cultcache(key = 14, default)] + recovery_receipt_id: Option, + #[cultcache(key = 15, default)] + recovery_receipt_digest: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "gamecult.epiphany.workspace_coverage_recovery_receipt", + schema = "WorkspaceCoverageRecoveryReceipt" +)] +struct WorkspaceCoverageRecoveryReceipt { + #[cultcache(key = 0)] + schema_version: String, + #[cultcache(key = 1)] + recovery_id: String, + #[cultcache(key = 2)] + old_claim_id: String, + #[cultcache(key = 3)] + old_attempt_id: String, + #[cultcache(key = 4)] + old_claim_epoch: u64, + #[cultcache(key = 5)] + termination_id: String, + #[cultcache(key = 6)] + termination_envelope_digest: String, + #[cultcache(key = 7)] + replacement_launch_id: String, + #[cultcache(key = 8)] + replacement_launch_envelope_digest: String, + #[cultcache(key = 9)] + ready_heartbeat_id: String, + #[cultcache(key = 10)] + ready_heartbeat_envelope_digest: String, + #[cultcache(key = 11)] + new_claim_id: String, + #[cultcache(key = 12)] + new_attempt_id: String, + #[cultcache(key = 13)] + new_claim_epoch: u64, + #[cultcache(key = 14)] + recovered_at_utc: String, + #[cultcache(key = 15)] + old_launch_id: String, +} + +fn validate_recovery_receipt(receipt: &WorkspaceCoverageRecoveryReceipt) -> Result<()> { + if receipt.schema_version != RECOVERY_SCHEMA + || receipt.recovery_id.trim().is_empty() + || receipt.old_claim_id.trim().is_empty() + || receipt.old_attempt_id.trim().is_empty() + || receipt.old_launch_id.trim().is_empty() + || receipt.old_claim_epoch == 0 + || receipt.replacement_launch_id.trim().is_empty() + || receipt.ready_heartbeat_id.trim().is_empty() + || receipt.new_claim_id.trim().is_empty() + || receipt.new_attempt_id.trim().is_empty() + || receipt.old_claim_epoch.checked_add(1) != Some(receipt.new_claim_epoch) + { + bail!("invalid workspace coverage recovery receipt"); + } + validate_terminal_evidence( + &Some(receipt.termination_id.clone()), + &Some(receipt.termination_envelope_digest.clone()), + )?; + validate_terminal_evidence( + &Some(receipt.replacement_launch_id.clone()), + &Some(receipt.replacement_launch_envelope_digest.clone()), + )?; + validate_terminal_evidence( + &Some(receipt.ready_heartbeat_id.clone()), + &Some(receipt.ready_heartbeat_envelope_digest.clone()), + )?; + chrono::DateTime::parse_from_rfc3339(&receipt.recovered_at_utc)?; + Ok(()) +} + +fn cultcache_envelope_digest(value: &CultCacheEnvelope) -> String { + let mut digest = Sha256::new(); + digest.update(value.r#type.as_bytes()); + digest.update([0]); + digest.update(value.key.as_bytes()); + digest.update([0]); + digest.update(&value.payload); + format!("sha256-{:x}", digest.finalize()) +} + +pub fn authenticate_workspace_coverage_recovery_receipt( + body_store: impl AsRef, + cultmesh_store: impl AsRef, + runtime_id: &str, + host: &HostIncarnationIdentityEntry, + recovery_id: &str, + expected_digest: &str, +) -> Result<()> { + let opening = SingleFileMessagePackBackingStore::new(body_store.as_ref()).pull_all()?; + let receipt_env = find(&opening, RECOVERY_TYPE, recovery_id) + .ok_or_else(|| anyhow!("workspace coverage recovery receipt is absent"))?; + if cultcache_envelope_digest(receipt_env) != expected_digest { + bail!("workspace coverage recovery receipt digest disagrees with authority"); + } + let receipt: WorkspaceCoverageRecoveryReceipt = decode(receipt_env)?; + validate_recovery_receipt(&receipt)?; + if receipt.recovery_id != recovery_id { + bail!("workspace coverage recovery receipt key disagrees with payload"); + } + let (_, termination_digest) = authenticate_workspace_coverage_termination_with_envelope_digest( + cultmesh_store.as_ref(), + runtime_id, + &receipt.old_launch_id, + host, + )?; + let (replacement_launch, replacement_launch_digest) = + authenticate_workspace_coverage_managed_process_launch_with_envelope_digest( + cultmesh_store.as_ref(), + runtime_id, + &receipt.replacement_launch_id, + host, + )?; + let (ready, ready_digest) = + authenticate_workspace_coverage_provider_heartbeat_with_envelope_digest( + cultmesh_store.as_ref(), + runtime_id, + &receipt.ready_heartbeat_id, + host, + )?; + let latest = load_latest_workspace_coverage_provider_heartbeat( + cultmesh_store, + runtime_id, + &receipt.replacement_launch_id, + )? + .ok_or_else(|| anyhow!("workspace coverage recovery receipt has no current heartbeat"))?; + if termination_digest != receipt.termination_envelope_digest + || replacement_launch_digest != receipt.replacement_launch_envelope_digest + || ready_digest != receipt.ready_heartbeat_envelope_digest + || replacement_launch.launch_id != receipt.replacement_launch_id + || ready.launch_id != receipt.replacement_launch_id + || ready.status != "ready" + || latest.sequence < ready.sequence + || (latest.sequence == ready.sequence && latest.heartbeat_id != ready.heartbeat_id) + { + bail!("workspace coverage recovery receipt external evidence has moved or disagrees"); + } + let claim_env = find(&opening, CLAIM_TYPE, CLAIM_KEY) + .ok_or_else(|| anyhow!("workspace coverage recovered claim is absent"))?; + let claim: WorkspaceCoverageProjectionClaim = decode(claim_env)?; + let attempt_env = find(&opening, ATTEMPT_TYPE, &claim.attempt_id) + .ok_or_else(|| anyhow!("workspace coverage recovered attempt is absent"))?; + let attempt: WorkspaceCoverageProjectionAttempt = decode(attempt_env)?; + validate_claim_attempt_link(&claim, &attempt)?; + if claim.claim_id != receipt.new_claim_id + || attempt.attempt_id != receipt.new_attempt_id + || claim.claim_epoch != receipt.new_claim_epoch + || claim.recovery_receipt_id.as_deref() != Some(recovery_id) + || claim.recovery_receipt_digest.as_deref() != Some(expected_digest) + { + bail!("workspace coverage recovery receipt disagrees with current Body authority"); + } + Ok(()) } #[derive(Clone, Debug)] @@ -616,6 +909,10 @@ pub(crate) fn acquire_workspace_coverage_projection( executor_id: executor_id.into(), executor_incarnation: executor_incarnation.into(), managed_process_launch_id: managed_process_launch_id.into(), + termination_evidence_id: None, + termination_evidence_digest: None, + recovery_receipt_id: None, + recovery_receipt_digest: None, }; let attempt = WorkspaceCoverageProjectionAttempt { schema_version: ATTEMPT_SCHEMA.into(), @@ -630,6 +927,10 @@ pub(crate) fn acquire_workspace_coverage_projection( executor_id: executor_id.into(), executor_incarnation: executor_incarnation.into(), managed_process_launch_id: managed_process_launch_id.into(), + termination_evidence_id: None, + termination_evidence_digest: None, + recovery_receipt_id: None, + recovery_receipt_digest: None, }; let mut expected = authority.clone(); if let Some(existing) = existing_claim { @@ -665,6 +966,308 @@ pub(crate) fn acquire_workspace_coverage_projection( )) } +/// Atomically transfers a projection lease only after CultMesh proves the old +/// process dead and the replacement process ready. This does not launch either +/// process; the supervisor must establish the evidence ordering first. +pub fn current_workspace_coverage_recovery_target( + runtime_store: impl AsRef, +) -> Result> { + let runtime_store = runtime_store.as_ref(); + let route = runtime_repository_body_store_binding(runtime_store)? + .ok_or_else(|| anyhow!("workspace coverage runtime has no Body-store binding"))?; + let opening = SingleFileMessagePackBackingStore::new(&route.body_store_path).pull_all()?; + let Some(claim_env) = find(&opening, CLAIM_TYPE, CLAIM_KEY) else { + return Ok(None); + }; + let claim: WorkspaceCoverageProjectionClaim = decode(claim_env)?; + let attempt_env = find(&opening, ATTEMPT_TYPE, &claim.attempt_id) + .ok_or_else(|| anyhow!("workspace coverage current attempt is absent"))?; + let attempt: WorkspaceCoverageProjectionAttempt = decode(attempt_env)?; + validate_claim_attempt_link(&claim, &attempt)?; + if claim.status != "running" { + return Ok(None); + } + Ok(Some(WorkspaceCoverageRecoveryTarget { + claim_id: claim.claim_id, + claim_epoch: claim.claim_epoch, + managed_process_launch_id: claim.managed_process_launch_id, + })) +} + +pub fn recover_workspace_coverage_projection( + runtime_store: impl AsRef, + cultmesh_store: impl AsRef, + runtime_id: &str, + host: &HostIncarnationIdentityEntry, + old_launch_id: &str, + replacement_launch_id: &str, + replacement_ready_heartbeat_id: &str, + expected_old_claim_id: &str, +) -> Result { + let runtime_store = runtime_store.as_ref(); + let body_route = runtime_repository_body_store_binding(runtime_store)? + .ok_or_else(|| anyhow!("workspace coverage recovery runtime has no Body-store binding"))?; + if body_route.runtime_id != runtime_id { + bail!("workspace coverage recovery runtime disagrees with its Body route"); + } + let cultmesh_store = cultmesh_store.as_ref(); + let (old_launch, _) = + authenticate_workspace_coverage_managed_process_launch_with_envelope_digest( + cultmesh_store, + runtime_id, + old_launch_id, + host, + )?; + let (termination, termination_digest) = + authenticate_workspace_coverage_termination_with_envelope_digest( + cultmesh_store, + runtime_id, + old_launch_id, + host, + )?; + let (replacement_launch, replacement_launch_digest) = + authenticate_workspace_coverage_managed_process_launch_with_envelope_digest( + cultmesh_store, + runtime_id, + replacement_launch_id, + host, + )?; + let (ready, ready_digest) = + authenticate_workspace_coverage_provider_heartbeat_with_envelope_digest( + cultmesh_store, + runtime_id, + replacement_ready_heartbeat_id, + host, + )?; + let latest = load_latest_workspace_coverage_provider_heartbeat( + cultmesh_store, + runtime_id, + &replacement_launch.launch_id, + )? + .ok_or_else(|| anyhow!("workspace coverage recovery replacement has no current heartbeat"))?; + if old_launch.launch_id == replacement_launch.launch_id + || replacement_launch.replaces_launch_id.as_deref() != Some(old_launch.launch_id.as_str()) + || replacement_launch.replaces_termination_id.as_deref() + != Some(termination.termination_id.as_str()) + || replacement_launch + .replaces_termination_envelope_digest + .as_deref() + != Some(termination_digest.as_str()) + || ready.launch_id != replacement_launch.launch_id + || latest.heartbeat_id != ready.heartbeat_id + || ready.status != "ready" + || ready.sequence == 0 + { + bail!("workspace coverage recovery replacement is not exact and ready"); + } + let terminated_at = chrono::DateTime::parse_from_rfc3339(&termination.observed_at_utc)?; + let replacement_launched_at = + chrono::DateTime::parse_from_rfc3339(&replacement_launch.launched_at_utc)?; + if terminated_at >= replacement_launched_at { + bail!("workspace coverage recovery requires termination before replacement launch"); + } + + let body_store = PathBuf::from(&body_route.body_store_path); + let body_basis = observe_runtime_repository_body_basis(runtime_store)?; + RepositoryBodyReadSession::open(runtime_store, &body_basis)?; + let backing = SingleFileMessagePackBackingStore::new(&body_store); + let opening = backing.pull_all()?; + let claim_env = find(&opening, CLAIM_TYPE, CLAIM_KEY) + .ok_or_else(|| anyhow!("workspace coverage recovery running claim is absent"))?; + let old_claim: WorkspaceCoverageProjectionClaim = decode(claim_env)?; + let attempt_env = find(&opening, ATTEMPT_TYPE, &old_claim.attempt_id) + .ok_or_else(|| anyhow!("workspace coverage recovery running attempt is absent"))?; + let old_attempt: WorkspaceCoverageProjectionAttempt = decode(attempt_env)?; + validate_claim_attempt_link(&old_claim, &old_attempt)?; + if old_claim.schema_version != CLAIM_SCHEMA + || old_attempt.schema_version != ATTEMPT_SCHEMA + || old_claim.status != "running" + || old_claim.claim_id != expected_old_claim_id + || old_claim.managed_process_launch_id != old_launch.launch_id + || old_claim.executor_id != old_launch.provider_daemon_id + || old_claim.executor_incarnation != old_launch.provider_incarnation_id + { + bail!("workspace coverage recovery refuses moved, terminal, legacy, or misbound authority"); + } + + let body_head_env = find(&opening, BODY_HEAD_TYPE, BODY_HEAD_KEY) + .ok_or_else(|| anyhow!("current Body head missing"))?; + let body_head: RepositoryBodyHead = decode(body_head_env)?; + if body_head.observation_id != old_claim.body_observation_id + || body_head.generation != old_claim.body_generation + || body_head.manifest_root_sha256 != old_claim.manifest_root_sha256 + { + bail!("workspace coverage recovery refuses moved Body authority"); + } + let mut authority = vec![ + find(&opening, BODY_BINDING_TYPE, BODY_BINDING_KEY) + .cloned() + .ok_or_else(|| anyhow!("Body binding missing"))?, + body_head_env.clone(), + find( + &opening, + BODY_OBSERVATION_TYPE, + &old_claim.body_observation_id, + ) + .cloned() + .ok_or_else(|| anyhow!("Body observation missing"))?, + find( + &opening, + BODY_MANIFEST_TYPE, + &old_claim.manifest_root_sha256, + ) + .cloned() + .ok_or_else(|| anyhow!("Body manifest missing"))?, + ]; + let obligation_env = find(&opening, OBLIGATION_TYPE, &old_claim.obligation_id) + .cloned() + .ok_or_else(|| anyhow!("projection obligation missing"))?; + let plan_env = find(&opening, PLAN_TYPE, &old_claim.plan_id) + .cloned() + .ok_or_else(|| anyhow!("projection plan missing"))?; + let obligation: WorkspaceCoverageObligation = decode(&obligation_env)?; + let plan: WorkspaceCoverageProjectionPlan = decode(&plan_env)?; + validate_workspace_coverage_projection_plan(&obligation, &plan)?; + if plan.plan_id != old_claim.plan_id + || plan.obligation_id != old_claim.obligation_id + || obligation.runtime_id != body_basis.runtime_id + || obligation.swarm_id != body_basis.swarm_id + || obligation.workspace_id != body_basis.workspace_id + || obligation.body_binding_sha256 != body_basis.body_binding_sha256 + || obligation.body_observation_id != body_basis.observation_id + || obligation.body_generation != body_basis.generation + || obligation.manifest_root_sha256 != body_basis.manifest_root_sha256 + || old_claim.body_observation_id != body_basis.observation_id + || old_claim.body_generation != body_basis.generation + || old_claim.manifest_root_sha256 != body_basis.manifest_root_sha256 + { + bail!("workspace coverage recovery refuses substituted plan"); + } + authority.push(obligation_env.clone()); + authority.push(plan_env.clone()); + let prior_head = find(&opening, HEAD_TYPE, HEAD_KEY).cloned(); + if let Some(head) = &prior_head { + authority.push(head.clone()); + } + + let claim_epoch = old_claim + .claim_epoch + .checked_add(1) + .ok_or_else(|| anyhow!("claim epoch exhausted"))?; + let claim_id = uuid::Uuid::new_v4().to_string(); + let attempt_id = uuid::Uuid::new_v4().to_string(); + let recovery_id = uuid::Uuid::new_v4().to_string(); + let receipt = WorkspaceCoverageRecoveryReceipt { + schema_version: RECOVERY_SCHEMA.into(), + recovery_id: recovery_id.clone(), + old_claim_id: old_claim.claim_id.clone(), + old_attempt_id: old_attempt.attempt_id.clone(), + old_claim_epoch: old_claim.claim_epoch, + termination_id: termination.termination_id.clone(), + termination_envelope_digest: termination_digest.clone(), + replacement_launch_id: replacement_launch.launch_id.clone(), + replacement_launch_envelope_digest: replacement_launch_digest, + ready_heartbeat_id: ready.heartbeat_id.clone(), + ready_heartbeat_envelope_digest: ready_digest, + new_claim_id: claim_id.clone(), + new_attempt_id: attempt_id.clone(), + new_claim_epoch: claim_epoch, + recovered_at_utc: ready.observed_at_utc.clone(), + old_launch_id: old_launch.launch_id.clone(), + }; + validate_recovery_receipt(&receipt)?; + let receipt_env = envelope(RECOVERY_TYPE, &recovery_id, &receipt)?; + let receipt_digest = cultcache_envelope_digest(&receipt_env); + + let completed_at = termination.observed_at_utc.clone(); + let mut failed_claim = old_claim.clone(); + failed_claim.schema_version = CLAIM_SCHEMA_V2.into(); + failed_claim.status = "failed".into(); + failed_claim.termination_evidence_id = Some(termination.termination_id.clone()); + failed_claim.termination_evidence_digest = Some(termination_digest.clone()); + failed_claim.recovery_receipt_id = Some(recovery_id.clone()); + failed_claim.recovery_receipt_digest = Some(receipt_digest.clone()); + let mut failed_attempt = old_attempt.clone(); + failed_attempt.schema_version = ATTEMPT_SCHEMA_V2.into(); + failed_attempt.status = "failed".into(); + failed_attempt.completed_at = Some(completed_at.clone()); + failed_attempt.error = Some("executor process terminated; lease recovered".into()); + failed_attempt.termination_evidence_id = Some(termination.termination_id.clone()); + failed_attempt.termination_evidence_digest = Some(termination_digest); + failed_attempt.recovery_receipt_id = Some(recovery_id.clone()); + failed_attempt.recovery_receipt_digest = Some(receipt_digest.clone()); + validate_claim_attempt_link(&failed_claim, &failed_attempt)?; + + let claim = WorkspaceCoverageProjectionClaim { + schema_version: CLAIM_SCHEMA_V3.into(), + claim_id: claim_id.clone(), + claim_epoch, + plan_id: plan.plan_id.clone(), + attempt_id: attempt_id.clone(), + obligation_id: obligation.obligation_id.clone(), + body_observation_id: old_claim.body_observation_id.clone(), + body_generation: old_claim.body_generation, + manifest_root_sha256: old_claim.manifest_root_sha256.clone(), + status: "running".into(), + executor_id: replacement_launch.provider_daemon_id.clone(), + executor_incarnation: replacement_launch.provider_incarnation_id.clone(), + managed_process_launch_id: replacement_launch.launch_id.clone(), + termination_evidence_id: None, + termination_evidence_digest: None, + recovery_receipt_id: Some(recovery_id.clone()), + recovery_receipt_digest: Some(receipt_digest.clone()), + }; + let attempt = WorkspaceCoverageProjectionAttempt { + schema_version: ATTEMPT_SCHEMA_V3.into(), + attempt_id: attempt_id.clone(), + claim_id, + claim_epoch, + plan_id: plan.plan_id.clone(), + status: "running".into(), + started_at: ready.observed_at_utc.clone(), + completed_at: None, + error: None, + executor_id: replacement_launch.provider_daemon_id.clone(), + executor_incarnation: replacement_launch.provider_incarnation_id.clone(), + managed_process_launch_id: replacement_launch.launch_id.clone(), + termination_evidence_id: None, + termination_evidence_digest: None, + recovery_receipt_id: Some(recovery_id.clone()), + recovery_receipt_digest: Some(receipt_digest), + }; + validate_claim_attempt_link(&claim, &attempt)?; + let mut expected = authority.clone(); + expected.extend([claim_env.clone(), attempt_env.clone()]); + let mut replacements = authority; + let history_key = format!("history/{}", old_claim.claim_id); + if find(&opening, CLAIM_TYPE, &history_key).is_some() { + bail!("workspace coverage recovery claim history identity already exists"); + } + if find(&opening, RECOVERY_TYPE, &recovery_id).is_some() { + bail!("workspace coverage recovery receipt identity already exists"); + } + replacements.extend([ + receipt_env, + envelope(CLAIM_TYPE, &history_key, &failed_claim)?, + envelope(ATTEMPT_TYPE, &failed_attempt.attempt_id, &failed_attempt)?, + envelope(CLAIM_TYPE, CLAIM_KEY, &claim)?, + envelope(ATTEMPT_TYPE, &attempt.attempt_id, &attempt)?, + ]); + if !backing.compare_and_swap_batch(&expected, replacements)? { + bail!("workspace coverage recovery lost exact Body/plan/claim/attempt CAS"); + } + Ok(WorkspaceCoverageRecoveryOutcome { + claim_id: claim.claim_id, + claim_epoch: claim.claim_epoch, + managed_process_launch_id: claim.managed_process_launch_id, + executor_incarnation: claim.executor_incarnation, + recovery_receipt_id: recovery_id, + recovery_receipt_digest: claim + .recovery_receipt_digest + .ok_or_else(|| anyhow!("recovered claim lost recovery receipt digest"))?, + }) +} + fn validate_current_projection( opening: &[CultCacheEnvelope], prepared: &PreparedWorkspaceCoverageProjection, diff --git a/notes/fresh-workspace-handoff.md b/notes/fresh-workspace-handoff.md index 8fd6165e..bf97d61a 100644 --- a/notes/fresh-workspace-handoff.md +++ b/notes/fresh-workspace-handoff.md @@ -1871,3 +1871,37 @@ Body tests before calling recovery operational. The deployment next action is unchanged and permission-bound. Do not reboot without exact live operator approval. With that approval, run the real reboot/logon recovery proof already specified above. + +## Workspace coverage exact Body recovery and Idunn reconciliation (2026-07-16) + +The abandoned-claim path is typed end to end. Recovery refuses caller-owned +Body paths, reopens the runtime's authenticated Body route/basis, joins current +Body, obligation, plan, claim, and attempt, authenticates the old launch plus +immutable host-signed termination, and accepts only a causally linked +replacement launch whose current signed heartbeat is `ready`. One exact Body +CAS archives the failed owner, advances the fencing epoch, installs the +successor claim/attempt, and writes an immutable recovery receipt binding the +old claim, termination envelope, replacement launch envelope, ready heartbeat +envelope, and successor authority. Restart authentication reconstructs both +CultMesh evidence and current Body authority. + +Reserved launch schema v1 adds a signed causal replacement edge. Its writer +exact-CASes a singleton `replacement-for/` slot with exact +termination and launch state. A second replacement refuses; the supervisor +kills a spawned loser when persistence loses CAS. + +Idunn branches workspace coverage away from generic lifecycle PID receipts. It +persists or reuses termination, reuses a causal replacement after interruption, +otherwise launches through the fixed stdin/bootstrap path, waits for the exact +latest signed ready heartbeat, then invokes Body recovery. Initial/pre-claim +launches use the same specialized authority. + +Proof passed: 5 process-document/recovery tests, 14 supervisor tests, 15 +projector tests plus 1 ignored live Qdrant/Ollama test, and all-target check. + +Next cut: termination v0 still requires a provider heartbeat. A child dying +after launch persistence but before heartbeat sequence one strands the chain. +Rebuild termination so signed launch plus enrolled host/boot/process observation +is sufficient, with heartbeat optional additional evidence. Then run a +GUID-scoped live initial launch -> death -> replacement -> Body recovery smoke. +Reboot/logon proof remains permission-bound. diff --git a/state/ledgers.msgpack b/state/ledgers.msgpack index a8564a9624c64331c46cd0fc2d3cd76ac0e26755..798e814fc162a73d99ff787d9f78720e363c90a3 100644 GIT binary patch delta 566 zcmYk2zityj5XL$4=c-fz3MeunC1=ZaVFXlw6p4xgBqX}=-Xu48Z}+mZ>tGre6tonv z%wvEk5TfS+XeiQ9MoQiw4RdEjLfX;p?9P1O{PxqIou4N=Uv6hxqeo+158wQ{lAnM6 zwwoPh+3ydl-p5rh(`fgr8rL+gYuxz0>g5-wH;;dehvP|q_@FscegKI z`!JIR4T14<3V1;4lfU_$rSnICDOThn8;m5NX9!8tn))RLo>K`=eYt=s3QGa>4ow7a zONbO|b{N^a0t_ZWYZqYI&9o&sK1`}ajaAoL^$S2(f=MB00>TH5%B2>dYFC6IP6^{< z@QyZq&z`)5hJu<#fqi7Fs+A@_EOd&-o9g}uBpE{@K6Dg5aMZ`v2UXd4Cov=wacT*; zuG8k!zUU_`AS}rcH*o;Z%jBH42!$57>osh-qXTmLKB&+C$S@_nB#?#-qviM7W-&Qa z6&(~io$Ipb(s47}jIEWfcw(x4%W1tD%K-MgNuokc@GVxMeEJ1*R=}dYd`m|8i}j1M L{8lc|pvd4KUq8yarY#Kx7O*zW@NI8x=DE diff --git a/state/map.yaml b/state/map.yaml index ca05baf0..41477ebf 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -3,7 +3,7 @@ experiment: repository_body_observer: "Native Rust explicitly binds caller workspace ID to an existing validated runtime/swarm/source identity plus canonical Git root/object format/scope/policy. Bind also installs one immutable runtime-side RuntimeRepositoryBodyStoreBinding naming the canonical external Body-store locator and exact Body-binding hash; every read revalidates runtime/swarm/workspace/path/body identity and refuses a second store. Two equal sanitized temporary-index scans enumerate ignore-aware UTF-8 paths, Git modes, and nonrecursive gitlinks with global excludes disabled; raw regular bytes/lengths and non-followed symlink-target bytes are SHA-256 hashed. The domain-separated ordered raw manifest root, not the clean-filtered auxiliary tree OID, is authoritative Body identity. Every Git subprocess strips ambient repository/worktree/object/ref/index/namespace/replacement/graft/shallow and config-injection authority. Observe cannot mint identity. CultCache MessagePack atomically commits immutable manifest+observation and a manifest-root head by exact CAS. Canonical store-path checks precede writes; corrupt HEAD, sparse checkout, unsafe/unrepresentable paths, racing reads, and repo/runtime substitution fail closed. Only accepted stable observations persist; errors advance no head. Unchanged raw manifest preserves generation; status is read-pure. The observer owns bounded current-state identity, makes no historical continuity claim, and has no Ready field. The runtime route is locator authority only, not proof of model grounding." modeling_body_grounding: "Every coordinator-owned Modeling launch observes and carries one typed RepositoryBodyObservationBasis before worker thought. The immutable launch binds runtime/swarm/workspace/scope, Body-binding hash, observation generation/id/root, and scan interval. Modeling output contract v3 requires the exact worker-authored echo; ingress reloads the immutable launch and rejects missing/swapped basis, while non-Modeling cargo is forbidden. Mind admission review v1 and receipt v5 preserve that basis; admission validates exact launch/result/review equality plus the immutable historical Body artifacts, then copies it into the CAS receipt. Admission never samples current Body as a substitute. Direct Mind plan adoption and legacy migration remain explicitly ungrounded." workspace_retrieval_coverage: "The production-unwired retrieval.rs oracle is deleted. Legacy EpiphanyRetrievalState is presentation-only: clean Ready projects Missing and its indexing job is unavailable/unowned. Native coverage now binds an authenticated Repository Body manifest, exhaustive versioned inclusion policy, deterministic chunks/point payloads, immutable Ollama tag+artifact digest identity, vector dimensions, exact Qdrant point/payload/vector observations, and one terminal-CAS receipt/head. Query admission and the race-bounded repository readiness join remain absent; coverage success proves only this exact Body projection." - workspace_coverage_projector: "The dedicated workspace-coverage projector is the only coverage writer. Its reserved supervisor/CultMesh policy fixes the packaged sibling binary and exact runtime/Verse/Qdrant/Ollama args; generic policy, arbitrary command, workspace, Body-store, collection, and dimension inputs are forbidden. The binary authenticates current policy-bound launch receipt, PID, executable hash, runtime id, and host singleton before the service derives the immutable Body route. One persisted Body basis plus one authenticated read session prepares verified UTF-8 chunks; an exact Current fast path reads no worktree files. Claims/attempts bind executor incarnation/startup receipt and reject malformed prior state. Every pulse re-resolves the mutable Ollama tag to its installed artifact digest, probes dimensions, embeds only after claim acquisition, writes an epoch-fenced managed Qdrant collection, whole-scrolls typed payloads and unnamed vectors, rejects cyclic offsets and duplicate/extra/missing/substituted/nonfinite/wrong-dimension/wrong-vector points, then terminal-CASes receipt/head against exact Body/plan/claim/prior-head authority. Body history owns bounded retirement of terminal non-current collections; running/current collections are preserved and deletion requires exact managed metadata. Managed stdout exposes only sanitized fault classes. Recovery is intentionally absent: newer launch plus heartbeat is not death proof. Before recovery, the supervisor/process observer must publish immutable host+boot+process-instance termination evidence bound to the exact old launch, PID, executable, policy, heartbeat, and provider incarnation. Query admission/readiness remain absent." + workspace_coverage_projector: "The dedicated workspace-coverage projector is the only coverage writer. Its reserved supervisor/CultMesh policy fixes the packaged sibling binary and exact runtime/Verse/Qdrant/Ollama args; generic policy, arbitrary command, workspace, Body-store, collection, and dimension inputs are forbidden. The binary authenticates current policy-bound launch receipt, PID, executable hash, runtime id, and host singleton before the service derives the immutable Body route. One persisted Body basis plus one authenticated read session prepares verified UTF-8 chunks; an exact Current fast path reads no worktree files. Claims/attempts bind exact managed-process launch and executor incarnation. Every pulse re-resolves the mutable Ollama tag to its installed artifact digest, probes dimensions, embeds only after claim acquisition, writes an epoch-fenced managed Qdrant collection, whole-scrolls typed payloads and unnamed vectors, rejects cyclic offsets and duplicate/extra/missing/substituted/nonfinite/wrong-dimension/wrong-vector points, then terminal-CASes receipt/head against exact Body/plan/claim/prior-head authority. Body history owns bounded retirement of terminal non-current collections; running/current collections are preserved and deletion requires exact managed metadata. Idunn recovery uses specialized process authority only: immutable exact predecessor termination must exist before a host-signed replacement launch can claim the one replacement slot; the replacement's latest signed heartbeat must be ready; one Body CAS then terminalizes the abandoned claim/attempt, advances the fencing epoch, and persists an immutable recovery receipt binding old claim, termination, replacement launch, ready heartbeat, and successor claim. Query admission remains absent." repository_readiness_join: "Mind must own one race-bounded derived repository-readiness operation from exact authenticated authorities: freshly observe Body root R1, validate canonical RepoModel plus exact Mind admission and Body-grounding receipt, authenticated semantic projection eligibility for that admitted model, and workspace retrieval coverage bound to R1 and a named scope policy, then freshly observe R2 and require R2=R1. Its output is observed-ready-at with a truth interval ending at the second scan, not timeless Ready; later reads revalidate. A→B→A is harmless for content-addressed artifacts. Watcher/journal continuity and Hands receipts are invalidation triggers only. Legacy retrieval Ready/empty dirty paths, timestamps, watcher silence, Git OIDs, Qdrant existence, counts, and unrelated generations are forbidden substitutes." status: active phase: phase_6_reflection_boundary @@ -729,4 +729,5 @@ current_status: - Specialized reserved coverage process documents now exist as typed CultMesh state. Idunn's mandatory host-signed launch binds the exact current specialized policy envelope, enrolled host record digest, proven boot, PID creation token, captured executable path and bytes digest, provider incarnation, and ephemeral provider public key. Provider heartbeats are signed by that ephemeral key, bind the exact launch envelope and repeated host/boot/process tuple, and advance exact per-launch sequence/time under CAS. Hostile tests reject forged signatures, sequence gaps, identity collisions, wrong supervisor/status/runtime, stale policy, and tuple drift. These documents are not operational owners until supervisor and projector migrate and generic writers reject reserved coverage identities. - The live workspace-coverage process path now uses only specialized authority. Supervisor generates a per-launch provider key, spawns with reserved stdin piped and all other child stdin null, captures exact process identity, sends one fixed-size launch-id/seed frame, host-signs and CAS-persists the specialized launch, and kills/waits on any bootstrap or persistence failure. The child requires exact frame plus EOF, authenticates the enrolled host, current policy, boot, its PID creation token/path, launch digest, and provider public key before constructing the service or publishing its signed per-launch heartbeat. Generic lifecycle and generic daemon-heartbeat writers reject the reserved coverage identities; their old authenticator is deleted. Body claim/attempt v1 binds `managed_process_launch_id`; v0 lifecycle semantics cannot deserialize as the new owner. This Windows installation is explicitly enrolled at the default DPAPI CurrentUser host store and status re-open proved identity `17f041421045aa66d4c8ab0488f462dbb5ea1a7d8507dbf75216f5ec368dbb7a` without exposing private state. - Idunn now has one immutable, host-signed process-termination observation per reserved coverage launch, with no latest/global mirror. Its writer reloads and exact-CAS-preserves the current specialized policy, exact launch, and exact per-launch latest signed heartbeat; it immediately observes the enrolled host's proven boot and exact PID incarnation. A distinct proven boot yields `boot_superseded`; same-boot proof permits only `exact_exited`, `process_missing`, or a complete different `process_replaced` identity. ExactAlive, Inaccessible, Indeterminate, unknown boot, host mismatch, identity collision, and moved source envelopes refuse publication. The injectable observation seam is private to module tests. Recovery-facing authentication reconstructs the entire persisted policy/launch/heartbeat chain rather than trusting the signed termination blob alone. Four hostile document/termination tests and all-target check pass. - next_action: Wire Idunn reconciliation ordering around the termination document: observe and persist exact predecessor termination before any replacement spawn, then launch the replacement through the specialized stdin/bootstrap path and wait for its signed ready heartbeat. Implement one Body-store recovery CAS that authenticates exact old launch/heartbeat/termination plus exact replacement launch/ready heartbeat, terminalizes the v1 old claim/attempt, and acquires epoch+1 for the replacement. Falsify partial writes, source movement, stale Body, wrong replacement, and spawn-before-termination. No timeout or staleness recovery. The separate reboot/logon proof remains permission-bound. + - Abandoned workspace-coverage authority now has a complete typed transfer. Recovery derives the Body store from the runtime route, reopens the authenticated Body basis, joins obligation/plan/claim to that basis, authenticates current policy plus exact old launch and immutable termination, requires a signed causal termination-to-replacement edge and the replacement's latest ready heartbeat, then performs one Body CAS over Body authority, plan, old claim/attempt, immutable recovery receipt, failed claim history, and epoch+1 successor. The receipt stores exact termination, launch, and heartbeat envelope digests and can reconstruct both CultMesh evidence and current Body authority after restart. Claim/attempt schema and evidence pairs cannot split; impossible completion-before-start timelines refuse. Idunn's reserved reconciler no longer consults generic lifecycle PID receipts for this service. It persists/reuses termination before spawn, uses the specialized bootstrap launch, waits for signed readiness, and then invokes the recovery CAS. One termination admits one replacement slot; concurrent losers are killed when launch persistence loses CAS. + next_action: Close the remaining pre-readiness death seam: a reserved child that dies after launch persistence but before its first heartbeat currently cannot produce termination evidence because termination v0 requires a heartbeat. Rebuild termination evidence so exact launch plus native host/boot/process proof is sufficient and latest heartbeat is optional additional evidence, while preserving immutable causal replacement and recovery receipts. Then run an isolated live initial-launch/death/replacement/Body-recovery smoke. Reboot/logon proof remains permission-bound. diff --git a/state/scratch.md b/state/scratch.md index 0016e057..9311b3e0 100644 --- a/state/scratch.md +++ b/state/scratch.md @@ -137,3 +137,23 @@ Authentication rejoins the signed record to all exact persisted sources. Next wire reconcile order: termination first, replacement launch second, replacement signed-ready third, then one Body recovery CAS. No timer gets a vote. + +## Exact abandoned-claim transfer and Idunn wiring (2026-07-16) + +Recovery now derives the Body store from the runtime route, authenticates the +live Body basis, joins obligation/plan/claim to it, verifies exact old launch +and immutable termination, requires the replacement's signed causal edge plus +current ready heartbeat, and commits failed history, epoch+1 successor, and an +immutable recovery receipt in one Body CAS. The receipt binds exact CultMesh +envelope digests and reconstructs against CultMesh plus current Body state. + +Idunn has a reserved reconciliation branch. Generic lifecycle PID receipts no +longer decide workspace-coverage survival. Exact termination precedes spawn; +the launch writer admits one replacement for that termination; signed ready +precedes Body recovery. A concurrent launch loser is killed. + +Remaining wound: termination requires a latest heartbeat. A child dying after +signed launch persistence but before heartbeat sequence one can strand the +chain. Make heartbeat optional additional termination evidence, then run an +isolated live death/replacement/recovery smoke. Reboot still needs live operator +approval.