diff --git a/epiphany-core/Cargo.toml b/epiphany-core/Cargo.toml index 7af24250..6d1b923b 100644 --- a/epiphany-core/Cargo.toml +++ b/epiphany-core/Cargo.toml @@ -11,12 +11,18 @@ path = "src/lib.rs" [features] semantic-recovery-smoke = [] +workspace-coverage-recovery-smoke = [] [[bin]] name = "epiphany-semantic-recovery-smoke" path = "src/bin/epiphany-semantic-recovery-smoke.rs" required-features = ["semantic-recovery-smoke"] +[[bin]] +name = "epiphany-workspace-coverage-recovery-smoke" +path = "src/bin/epiphany-workspace-coverage-recovery-smoke.rs" +required-features = ["workspace-coverage-recovery-smoke"] + [dependencies] anyhow = "1" bm25 = "2.3.2" diff --git a/epiphany-core/src/bin/epiphany-daemon-supervisor.rs b/epiphany-core/src/bin/epiphany-daemon-supervisor.rs index 785b3e73..a1f4b445 100644 --- a/epiphany-core/src/bin/epiphany-daemon-supervisor.rs +++ b/epiphany-core/src/bin/epiphany-daemon-supervisor.rs @@ -1788,6 +1788,7 @@ fn reconcile_workspace_coverage_projector( "claimEpoch": recovered.claim_epoch, "replacementLaunchId": replacement.launch_id, "recoveryReceiptId": recovered.recovery_receipt_id, + "recoveryReceiptDigest": recovered.recovery_receipt_digest, "restarted": true, "privateStateExposed": false, }))? diff --git a/epiphany-core/src/bin/epiphany-workspace-coverage-recovery-smoke.rs b/epiphany-core/src/bin/epiphany-workspace-coverage-recovery-smoke.rs new file mode 100644 index 00000000..f859d741 --- /dev/null +++ b/epiphany-core/src/bin/epiphany-workspace-coverage-recovery-smoke.rs @@ -0,0 +1,517 @@ +use anyhow::{Context, Result, anyhow, bail}; +use epiphany_core::{ + ProcessInstanceObservation, RuntimeSpineInitOptions, + WorkspaceCoverageManagedProcessLaunchEntry, authenticate_workspace_coverage_recovery_receipt, + bind_repository_body, bind_runtime_to_agent_memory_swarm, + current_workspace_coverage_recovery_target, ensure_agent_memory_swarm_identity, + initialize_runtime_spine, load_latest_workspace_coverage_managed_process_launch, + observe_process_instance, observe_repository_body, + process_identity_from_workspace_coverage_launch, seed_epiphany_local_verse_context, + workspace_coverage_execution_collection, +}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeSet; +use std::ffi::OsStr; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::process::Stdio; +use std::thread; +use std::time::{Duration, Instant}; +use uuid::Uuid; + +const RUNTIME_ID: &str = "workspace-coverage-recovery-smoke"; +const SERVICE_ID: &str = "epiphany-workspace-coverage-projector-service"; +const SMOKE_SOURCE: &[u8] = include_bytes!("epiphany-workspace-coverage-recovery-smoke.rs"); + +fn main() -> Result<()> { + let smoke_id = Uuid::new_v4().to_string(); + let root = std::env::temp_dir().join(format!("epiphany-workspace-coverage-{smoke_id}")); + fs::create_dir_all(&root)?; + let repo = root.join("repo"); + let runtime = root.join("runtime.ccmp"); + let agents = root.join("agents.ccmp"); + let body = root.join("body.ccmp"); + let verse = root.join("verse.ccmp"); + let proof_path = root.join("proof.json"); + let qdrant_url = std::env::var("EPIPHANY_SMOKE_QDRANT_URL") + .unwrap_or_else(|_| "http://127.0.0.1:6333".into()); + let ollama_url = std::env::var("EPIPHANY_SMOKE_OLLAMA_URL") + .unwrap_or_else(|_| "http://127.0.0.1:11434".into()); + let model = std::env::var("EPIPHANY_SMOKE_OLLAMA_MODEL") + .unwrap_or_else(|_| "qwen3-embedding:0.6b".into()); + require_http_ok(&format!("{qdrant_url}/collections"))?; + let mut qdrant_cleanup = QdrantCollectionCleanup { + base_url: qdrant_url.clone(), + owned: BTreeSet::new(), + }; + require_http_ok(&format!("{ollama_url}/api/tags"))?; + prewarm_ollama(&ollama_url, &model)?; + eprintln!("smoke: endpoints ready"); + create_fixture_repo(&repo)?; + initialize_runtime_spine( + &runtime, + RuntimeSpineInitOptions { + runtime_id: RUNTIME_ID.into(), + display_name: "workspace coverage recovery smoke".into(), + created_at: chrono::Utc::now().to_rfc3339(), + }, + )?; + ensure_agent_memory_swarm_identity(&agents, &format!("smoke-{smoke_id}"))?; + bind_runtime_to_agent_memory_swarm(&runtime, &agents, &chrono::Utc::now().to_rfc3339())?; + bind_repository_body(&repo, &body, &runtime, &format!("smoke-{smoke_id}"))?; + observe_repository_body(&repo, &body, &runtime)?; + seed_epiphany_local_verse_context(&verse, RUNTIME_ID, chrono::Utc::now().to_rfc3339())?; + let _process_cleanup = ManagedProcessCleanup { + verse: verse.clone(), + }; + + let supervisor = sibling_binary("epiphany-daemon-supervisor")?; + run_supervisor( + &supervisor, + &[ + "workspace-coverage-projector-service-policy", + "--store", + path(&verse)?, + "--runtime-id", + RUNTIME_ID, + "--daemon-id", + "epiphany-daemon-supervisor", + "--runtime-store", + path(&runtime)?, + "--qdrant-url", + &qdrant_url, + "--ollama-base-url", + &ollama_url, + "--ollama-model", + &model, + "--loop-interval-seconds", + "30", + "--stdout-artifact", + path(&root.join("projector.stdout.log"))?, + "--stderr-artifact", + path(&root.join("projector.stderr.log"))?, + ], + )?; + eprintln!("smoke: policy persisted"); + run_supervisor(&supervisor, reconcile_args(&verse, &runtime)?)?; + eprintln!("smoke: initial launch persisted"); + + let first = wait_for_claim(&runtime, 1, Duration::from_secs(90))?; + qdrant_cleanup + .owned + .insert(workspace_coverage_execution_collection( + &first.plan_id, + &first.claim_id, + first.claim_epoch, + )?); + eprintln!( + "smoke: initial claim acquired at epoch {}", + first.claim_epoch + ); + let first_launch = load_latest_workspace_coverage_managed_process_launch(&verse, RUNTIME_ID)? + .context("initial managed launch disappeared")?; + if first.managed_process_launch_id != first_launch.launch_id { + bail!("initial claim disagrees with latest managed launch"); + } + kill_process(&first_launch)?; + wait_for_process_exit(&first_launch, Duration::from_secs(20))?; + eprintln!("smoke: initial process killed"); + + let recovery_output = run_supervisor(&supervisor, reconcile_args(&verse, &runtime)?)?; + let recovery_json: Value = serde_json::from_str(&recovery_output) + .context("supervisor recovery output was not JSON")?; + if recovery_json.get("status").and_then(Value::as_str) != Some("recovered") { + bail!("supervisor did not recover workspace coverage: {recovery_output}"); + } + let recovery_id = required_json_string(&recovery_json, "recoveryReceiptId")?; + let recovery_digest = required_json_string(&recovery_json, "recoveryReceiptDigest")?; + eprintln!("smoke: supervisor recovery returned"); + let second = wait_for_claim(&runtime, 2, Duration::from_secs(20))?; + qdrant_cleanup + .owned + .insert(workspace_coverage_execution_collection( + &second.plan_id, + &second.claim_id, + second.claim_epoch, + )?); + if second.claim_id == first.claim_id + || second.claim_epoch != first.claim_epoch + 1 + || second.managed_process_launch_id == first.managed_process_launch_id + { + bail!("recovered Body authority did not advance exactly once"); + } + let replacement = load_latest_workspace_coverage_managed_process_launch(&verse, RUNTIME_ID)? + .context("replacement managed launch disappeared")?; + authenticate_workspace_coverage_recovery_receipt( + &body, + &verse, + RUNTIME_ID, + epiphany_core::open_default_host_identity()?.entry(), + recovery_id, + recovery_digest, + )?; + kill_process(&replacement)?; + qdrant_cleanup.cleanup_and_verify()?; + + let executable_sha256 = hash_file(&std::env::current_exe()?)?; + let smoke_source_sha256 = hash_bytes(SMOKE_SOURCE); + let source_head = git_output(&["rev-parse", "HEAD"])?; + let source_diff_sha256 = hash_bytes(&git_bytes(&["diff", "--binary", "HEAD"])?); + + let proof = json!({ + "schemaVersion": "epiphany.workspace_coverage_recovery_smoke.v0", + "smokeId": smoke_id, + "status": "passed", + "runtimeId": RUNTIME_ID, + "initialClaimId": first.claim_id, + "initialClaimEpoch": first.claim_epoch, + "initialLaunchId": first.managed_process_launch_id, + "recoveredClaimId": second.claim_id, + "recoveredClaimEpoch": second.claim_epoch, + "replacementLaunchId": second.managed_process_launch_id, + "recoveryReceiptId": recovery_id, + "recoveryReceiptDigest": recovery_digest, + "executableSha256": executable_sha256, + "smokeSourceSha256": smoke_source_sha256, + "sourceGitHead": source_head, + "sourceDiffSha256": source_diff_sha256, + "ownedCollectionsRemoved": true, + "bodyStore": body, + "verseStore": verse, + "privateStateExposed": false + }); + fs::write(&proof_path, serde_json::to_vec_pretty(&proof)?)?; + println!( + "{}", + serde_json::to_string_pretty(&json!({ + "status": "passed", + "proof": proof_path, + "recoveryReceiptId": recovery_id, + "initialEpoch": first.claim_epoch, + "recoveredEpoch": second.claim_epoch, + "privateStateExposed": false + }))? + ); + Ok(()) +} + +fn create_fixture_repo(repo: &Path) -> Result<()> { + fs::create_dir_all(repo)?; + run(Command::new("git") + .arg("init") + .arg("-b") + .arg("main") + .current_dir(repo))?; + for file_index in 0..160 { + let mut text = String::new(); + for line in 0..120 { + text.push_str(&format!( + "pub fn smoke_{file_index}_{line}() -> usize {{ {} }}\n", + file_index + line + )); + } + fs::write(repo.join(format!("body_{file_index:03}.rs")), text)?; + } + run(Command::new("git").arg("add").arg(".").current_dir(repo))?; + Ok(()) +} + +fn reconcile_args(verse: &Path, runtime: &Path) -> Result> { + Ok(vec![ + "managed-service-reconcile".into(), + "--store".into(), + path(verse)?.into(), + "--runtime-id".into(), + RUNTIME_ID.into(), + "--daemon-id".into(), + "epiphany-daemon-supervisor".into(), + "--service-id".into(), + SERVICE_ID.into(), + "--runtime-store".into(), + path(runtime)?.into(), + "--force".into(), + ]) +} + +fn wait_for_claim( + runtime: &Path, + minimum_epoch: u64, + timeout: Duration, +) -> Result { + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if let Some(target) = current_workspace_coverage_recovery_target(runtime)? + && target.claim_epoch >= minimum_epoch + { + return Ok(target); + } + thread::sleep(Duration::from_millis(20)); + } + Err(anyhow!( + "workspace coverage claim epoch {minimum_epoch} did not appear" + )) +} + +fn run_supervisor(supervisor: &Path, args: I) -> Result +where + I: IntoIterator, + S: AsRef, +{ + let command_id = Uuid::new_v4(); + let stdout_path = std::env::temp_dir().join(format!("epiphany-smoke-{command_id}.stdout")); + let stderr_path = std::env::temp_dir().join(format!("epiphany-smoke-{command_id}.stderr")); + let stdout = fs::File::create(&stdout_path)?; + let stderr = fs::File::create(&stderr_path)?; + let status = Command::new(supervisor) + .args(args) + .env("EPIPHANY_WORKSPACE_COVERAGE_SMOKE_DIAGNOSTICS", "1") + .env("EPIPHANY_OLLAMA_TIMEOUT_MS", "120000") + .stdout(Stdio::from(stdout)) + .stderr(Stdio::from(stderr)) + .status()?; + let stdout = fs::read_to_string(&stdout_path)?; + let stderr = fs::read_to_string(&stderr_path)?; + let _ = fs::remove_file(stdout_path); + let _ = fs::remove_file(stderr_path); + if !status.success() { + bail!("supervisor failed: {stderr}"); + } + Ok(stdout) +} + +fn run(command: &mut Command) -> Result<()> { + let output = command.output()?; + if !output.status.success() { + bail!( + "command failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + Ok(()) +} + +fn sibling_binary(name: &str) -> Result { + let file = if cfg!(windows) { + format!("{name}.exe") + } else { + name.into() + }; + let path = std::env::current_exe()?.with_file_name(file); + if !path.is_file() { + bail!("required sibling binary is absent: {}", path.display()); + } + Ok(path) +} + +fn path(value: &Path) -> Result<&str> { + value.to_str().context("smoke path is not UTF-8") +} + +fn required_json_string<'a>(value: &'a Value, key: &str) -> Result<&'a str> { + value + .get(key) + .and_then(Value::as_str) + .with_context(|| format!("recovery output lacks {key}")) +} + +fn require_http_ok(url: &str) -> Result<()> { + let response = reqwest::blocking::get(url)?; + if !response.status().is_success() { + bail!("required endpoint is unavailable: {url}"); + } + Ok(()) +} + +fn prewarm_ollama(base_url: &str, model: &str) -> Result<()> { + let response = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(120)) + .build()? + .post(format!("{base_url}/api/embed")) + .json(&json!({"model": model, "input": ["epiphany live smoke warmup"]})) + .send()?; + if !response.status().is_success() { + bail!("Ollama smoke warmup failed with {}", response.status()); + } + Ok(()) +} + +struct ManagedProcessCleanup { + verse: PathBuf, +} + +struct QdrantCollectionCleanup { + base_url: String, + owned: BTreeSet, +} + +impl QdrantCollectionCleanup { + fn cleanup_and_verify(&mut self) -> Result<()> { + let client = reqwest::blocking::Client::new(); + let names = self.owned.iter().cloned().collect::>(); + for name in names { + let response = client + .delete(format!("{}/collections/{name}", self.base_url)) + .send()?; + if !response.status().is_success() + && response.status() != reqwest::StatusCode::NOT_FOUND + { + bail!("Qdrant refused fixture collection deletion for {name}"); + } + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let status = client + .get(format!("{}/collections/{name}", self.base_url)) + .send()? + .status(); + if status == reqwest::StatusCode::NOT_FOUND { + self.owned.remove(&name); + break; + } + if Instant::now() >= deadline { + bail!("fixture collection {name} remained after deletion"); + } + thread::sleep(Duration::from_millis(50)); + } + } + Ok(()) + } +} + +impl Drop for QdrantCollectionCleanup { + fn drop(&mut self) { + let client = reqwest::blocking::Client::new(); + for name in &self.owned { + let _ = client + .delete(format!("{}/collections/{name}", self.base_url)) + .send(); + } + } +} + +impl Drop for ManagedProcessCleanup { + fn drop(&mut self) { + if let Ok(Some(launch)) = + load_latest_workspace_coverage_managed_process_launch(&self.verse, RUNTIME_ID) + { + let _ = kill_process(&launch); + } + } +} + +#[cfg(windows)] +fn kill_native_process(pid: u32) -> Result<()> { + let status = Command::new("taskkill") + .args(["/PID", &pid.to_string(), "/T", "/F"]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status()?; + if !status.success() { + bail!("taskkill failed for PID {pid}"); + } + Ok(()) +} + +#[cfg(unix)] +fn kill_native_process(pid: u32) -> Result<()> { + let status = Command::new("kill") + .args(["-KILL", &pid.to_string()]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status()?; + if !status.success() { + bail!("kill failed for PID {pid}"); + } + Ok(()) +} + +fn kill_process(launch: &WorkspaceCoverageManagedProcessLaunchEntry) -> Result<()> { + let expected = process_identity_from_workspace_coverage_launch(launch); + match observe_process_instance(&expected) { + ProcessInstanceObservation::ExactAlive => kill_native_process(expected.process_id), + ProcessInstanceObservation::ExactExited { .. } | ProcessInstanceObservation::Missing => { + bail!("managed launch {} already exited", launch.launch_id) + } + ProcessInstanceObservation::Replaced { .. } => { + bail!( + "managed launch {} PID has been reused; refusing kill", + launch.launch_id + ) + } + ProcessInstanceObservation::Inaccessible => { + bail!( + "managed launch {} is inaccessible; refusing kill", + launch.launch_id + ) + } + ProcessInstanceObservation::Indeterminate { reason } => { + bail!( + "managed launch {} is indeterminate: {reason}", + launch.launch_id + ) + } + } +} + +fn wait_for_process_exit( + launch: &WorkspaceCoverageManagedProcessLaunchEntry, + timeout: Duration, +) -> Result<()> { + let expected = process_identity_from_workspace_coverage_launch(launch); + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + match observe_process_instance(&expected) { + ProcessInstanceObservation::ExactAlive => {} + ProcessInstanceObservation::ExactExited { .. } + | ProcessInstanceObservation::Missing => return Ok(()), + ProcessInstanceObservation::Replaced { .. } => { + bail!( + "managed launch {} PID was reused while awaiting exit", + launch.launch_id + ) + } + ProcessInstanceObservation::Inaccessible => { + bail!("managed launch {} became inaccessible", launch.launch_id) + } + ProcessInstanceObservation::Indeterminate { reason } => { + bail!( + "managed launch {} became indeterminate: {reason}", + launch.launch_id + ) + } + } + thread::sleep(Duration::from_millis(50)); + } + bail!("managed launch {} did not exit", launch.launch_id) +} + +fn hash_file(path: &Path) -> Result { + Ok(hash_bytes(&fs::read(path)?)) +} + +fn hash_bytes(bytes: &[u8]) -> String { + format!("sha256-{:x}", Sha256::digest(bytes)) +} + +fn git_bytes(args: &[&str]) -> Result> { + let repo = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .context("epiphany-core has no repository parent")?; + let output = Command::new("git") + .arg("-C") + .arg(repo) + .args(args) + .output()?; + if !output.status.success() { + bail!( + "git provenance command failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + Ok(output.stdout) +} + +fn git_output(args: &[&str]) -> Result { + Ok(String::from_utf8(git_bytes(args)?)?.trim().to_string()) +} diff --git a/epiphany-core/src/process_observation.rs b/epiphany-core/src/process_observation.rs index 8e6a1577..f784cfe4 100644 --- a/epiphany-core/src/process_observation.rs +++ b/epiphany-core/src/process_observation.rs @@ -99,6 +99,27 @@ mod platform { QueryFullProcessImageNameW, WaitForSingleObject, }; + #[repr(C)] + struct SystemTimeOfDayInformation { + boot_time: i64, + current_time: i64, + time_zone_bias: i64, + current_time_zone_id: u32, + reserved: u32, + boot_time_bias: u64, + sleep_time_bias: u64, + } + + #[link(name = "ntdll")] + unsafe extern "system" { + fn NtQuerySystemInformation( + system_information_class: u32, + system_information: *mut core::ffi::c_void, + system_information_length: u32, + return_length: *mut u32, + ) -> i32; + } + const WINDOWS_TO_UNIX_100NS: u64 = 116_444_736_000_000_000; const SYNCHRONIZE_ACCESS: u32 = 0x0010_0000; @@ -267,12 +288,25 @@ mod platform { } pub(super) fn boot_identity() -> Option { - capture(4).ok().map(|identity| { - format!( - "windows-system-process-filetime:{}", - identity.creation_token + let mut information = SystemTimeOfDayInformation { + boot_time: 0, + current_time: 0, + time_zone_bias: 0, + current_time_zone_id: 0, + reserved: 0, + boot_time_bias: 0, + sleep_time_bias: 0, + }; + let status = unsafe { + NtQuerySystemInformation( + 3, + (&mut information as *mut SystemTimeOfDayInformation).cast(), + std::mem::size_of::() as u32, + std::ptr::null_mut(), ) - }) + }; + (status >= 0 && information.boot_time > 0) + .then(|| format!("windows-kernel-boot-filetime:{}", information.boot_time)) } pub(super) fn is_missing_error(error: &anyhow::Error) -> bool { @@ -388,6 +422,13 @@ mod tests { assert!(capture_process_instance(0).is_err()); } + #[test] + fn native_boot_identity_is_available_and_stable() { + let first = native_boot_identity().expect("native boot identity unavailable"); + let second = native_boot_identity().expect("native boot identity disappeared"); + assert_eq!(first, second); + } + #[test] fn live_current_process_has_stable_exact_identity() { let identity = capture_process_instance(std::process::id()).unwrap(); diff --git a/epiphany-core/src/workspace_coverage_process_documents.rs b/epiphany-core/src/workspace_coverage_process_documents.rs index d70ec359..081da347 100644 --- a/epiphany-core/src/workspace_coverage_process_documents.rs +++ b/epiphany-core/src/workspace_coverage_process_documents.rs @@ -2281,6 +2281,26 @@ mod tests { recovered.executor_incarnation, replacement.provider_incarnation_id ); + match crate::workspace_coverage_projector::acquire_workspace_coverage_projection( + &prepared, + EPIPHANY_WORKSPACE_COVERAGE_PROJECTOR_DAEMON_ID, + &replacement.provider_incarnation_id, + &replacement.launch_id, + )? { + crate::workspace_coverage_projector::WorkspaceCoverageAcquireResult::Acquired( + resumed, + ) => assert_eq!(resumed.claim.claim_id, recovered.claim_id), + _ => bail!("exact replacement incarnation did not resume its recovered claim"), + } + assert!(matches!( + crate::workspace_coverage_projector::acquire_workspace_coverage_projection( + &prepared, + EPIPHANY_WORKSPACE_COVERAGE_PROJECTOR_DAEMON_ID, + "wrong-incarnation", + &replacement.launch_id, + )?, + crate::workspace_coverage_projector::WorkspaceCoverageAcquireResult::Contended + )); let opening = SingleFileMessagePackBackingStore::new(&body_store).pull_all()?; let old_history: crate::workspace_coverage_projector::WorkspaceCoverageProjectionClaim = rmp_serde::from_slice( diff --git a/epiphany-core/src/workspace_coverage_projector.rs b/epiphany-core/src/workspace_coverage_projector.rs index 73bf53c9..8170729e 100644 --- a/epiphany-core/src/workspace_coverage_projector.rs +++ b/epiphany-core/src/workspace_coverage_projector.rs @@ -71,6 +71,7 @@ pub(crate) struct WorkspaceCoverageObservedBinding { pub struct WorkspaceCoverageRecoveryTarget { pub claim_id: String, pub claim_epoch: u64, + pub plan_id: String, pub managed_process_launch_id: String, } @@ -884,6 +885,28 @@ pub(crate) fn acquire_workspace_coverage_projection( let prior_attempt: WorkspaceCoverageProjectionAttempt = decode(prior_attempt_env)?; validate_claim_attempt_link(&prior, &prior_attempt)?; if prior.status == "running" { + if prior.schema_version == CLAIM_SCHEMA_V3 + && prior_attempt.schema_version == ATTEMPT_SCHEMA_V3 + && prior.executor_id == executor_id + && prior.executor_incarnation == executor_incarnation + && prior.managed_process_launch_id == managed_process_launch_id + && prior.plan_id == plan.plan_id + && prior.obligation_id == obligation.obligation_id + && prior.body_observation_id == basis.observation_id + && prior.body_generation == basis.generation + && prior.manifest_root_sha256 == basis.manifest_root_sha256 + { + return Ok(WorkspaceCoverageAcquireResult::Acquired( + WorkspaceCoverageAcquisition { + body_store, + obligation, + plan, + claim: prior, + attempt: prior_attempt, + prior_head: find(&opening, HEAD_TYPE, HEAD_KEY).cloned(), + }, + )); + } return Ok(WorkspaceCoverageAcquireResult::Contended); } prior @@ -990,6 +1013,7 @@ pub fn current_workspace_coverage_recovery_target( Ok(Some(WorkspaceCoverageRecoveryTarget { claim_id: claim.claim_id, claim_epoch: claim.claim_epoch, + plan_id: claim.plan_id, managed_process_launch_id: claim.managed_process_launch_id, })) } diff --git a/epiphany-core/src/workspace_coverage_projector_service.rs b/epiphany-core/src/workspace_coverage_projector_service.rs index 16ba6ec4..b7374e4b 100644 --- a/epiphany-core/src/workspace_coverage_projector_service.rs +++ b/epiphany-core/src/workspace_coverage_projector_service.rs @@ -244,16 +244,22 @@ impl WorkspaceCoverageProjectorServiceBody { pub fn pulse(&mut self) -> WorkspaceCoverageProjectorServicePulse { match self.pulse_inner() { Ok(pulse) => pulse, - Err(_error) => WorkspaceCoverageProjectorServicePulse { - status: WorkspaceCoverageProjectorPulseStatus::Refused, - body_observation_id: None, - body_generation: None, - plan_id: None, - receipt_id: None, - // Managed stdout is an operator-safe projection. Raw backend - // bodies and filesystem paths remain private to the process. - fault: Some("workspace_coverage_pulse_refused".into()), - }, + Err(_error) => { + #[cfg(feature = "workspace-coverage-recovery-smoke")] + if std::env::var_os("EPIPHANY_WORKSPACE_COVERAGE_SMOKE_DIAGNOSTICS").is_some() { + eprintln!("workspace coverage smoke diagnostic: {_error:#}"); + } + WorkspaceCoverageProjectorServicePulse { + status: WorkspaceCoverageProjectorPulseStatus::Refused, + body_observation_id: None, + body_generation: None, + plan_id: None, + receipt_id: None, + // Managed stdout is an operator-safe projection. Raw backend + // bodies and filesystem paths remain private to the process. + fault: Some("workspace_coverage_pulse_refused".into()), + } + } } } diff --git a/notes/epiphany-current-algorithmic-map.md b/notes/epiphany-current-algorithmic-map.md index e8fac96e..b036930f 100644 --- a/notes/epiphany-current-algorithmic-map.md +++ b/notes/epiphany-current-algorithmic-map.md @@ -890,8 +890,8 @@ the indexed content. Receipt/head constructors and writers remain projector- private. Live query eligibility re-observes Qdrant; a stored receipt proves only that one exact observation completed. -The crate-private execution path now exists but remains deliberately -production-unreachable. It validates plan-sealed text hashes and vector +The execution path is reachable only through the reserved packaged workspace- +coverage projector managed by Idunn. It validates plan-sealed text hashes and vector dimensions, creates or authenticates one exact managed claim/epoch Qdrant collection, writes no empty upsert, whole-scrolls typed payloads, rejects cyclic pagination plus duplicate/extra/missing/substituted points, and terminal-CASes @@ -899,17 +899,15 @@ the receipt/head against the exact current Body authority, immutable plan, running claim/attempt, and prior coverage head acquired at start. A CAS loser cannot mint a receipt; failure can terminalize after Body advance. -The service fork is resolved in favor of a dedicated workspace-coverage -projector. Before it becomes reachable, claims must bind an executor -incarnation and startup lifecycle receipt authenticated by the reserved managed -service. The pulse needs an exact `Current` result to avoid endless reprojection -and a sealed text-rematerialization path rather than caller-supplied workspace -bytes. Abandoned-claim recovery may use only a newer latest managed launch and -its correlated ready heartbeat to fence the old incarnation in one Body-store -CAS. Time, Qdrant state, generic policy, process guesses, and caller strings are -not abandonment authority. If Body advanced, recovery terminalizes the stale -claim and lets normal acquisition derive a new plan; it never resurrects stale -work. +The dedicated workspace-coverage projector binds claims to an authenticated +executor incarnation and reserved managed-process launch. Its pulse has an exact +`Current` result and rematerializes text through the authenticated historical +Body session rather than caller-supplied bytes. Abandoned-claim recovery admits +only immutable host-signed termination, one causally linked replacement launch, +and that replacement's latest signed ready heartbeat before one Body-store CAS. +Time, Qdrant state, generic policy, process guesses, and caller strings are not +abandonment authority. If Body advanced, recovery terminalizes the stale claim +and lets normal acquisition derive a new plan; it never resurrects stale work. ### Dedicated service and exact semantic proof @@ -931,13 +929,15 @@ success. Body history derives retirement candidates for terminal non-current claim collections; current and running collections are preserved, and Qdrant deletion requires exact managed metadata. -Recovery is deliberately not present. A newer launch and ready heartbeat prove -a successor exists, not that the old writer died. Honest recovery first needs a -supervisor-owned termination observation bound to host+boot incarnation, -process-instance identity, exact old launch/PID/executable/policy, and the old -launch-correlated heartbeat/provider incarnation. The existing process observer -collapses access failure/PID reuse too aggressively and cannot issue that proof. -Timeouts remain inadmissible. +Recovery is present through Idunn's reserved reconciliation path. Launch, +heartbeat, and termination serialize through one per-launch process-evidence +head. Host-signed termination binds host+boot incarnation, exact native process +identity, executable, policy, and launch; it may seal death before heartbeat one +without turning timeout or PID absence into authority. One termination admits +one causal replacement slot. Recovery then requires the replacement's latest +signed ready heartbeat and commits failed history, epoch+1 successor, and an +immutable evidence-digest receipt in one Body CAS. PID reuse, access failure, +indeterminate observation, stale ready replay, and timeout remain inadmissible. This map must change when ownership changes. Historical scars belong in git, evidence, or an explicitly archived note—not in the machine's Modeling state. diff --git a/notes/fresh-workspace-handoff.md b/notes/fresh-workspace-handoff.md index 70781fcb..d64b5b77 100644 --- a/notes/fresh-workspace-handoff.md +++ b/notes/fresh-workspace-handoff.md @@ -1925,3 +1925,42 @@ Next: create/run a GUID-scoped live smoke through real packaged supervisor and projector binaries, local Qdrant/Ollama, exact child kill, causal replacement, epoch+1 recovery, and restart-time receipt authentication. Restore the prior Qdrant container state afterward. Reboot/logon remains permission-bound. + +## Live packaged workspace coverage recovery proof (2026-07-16) + +Feature-gated binary `epiphany-workspace-coverage-recovery-smoke` constructs +GUID-scoped repo/runtime/Body/Verse state, uses the real packaged supervisor and +projector siblings, warms local Ollama, derives its exact fenced Qdrant collection names, launches +through specialized bootstrap, catches the epoch 1 claim, kills the exact child, +runs Idunn reconciliation, verifies epoch 2 authority and fresh-store-reopen +recovery receipt authentication, kills the replacement, and removes only its +exact owned collections. Process cleanup revalidates the signed launch's native +incarnation before issuing any kill. + +Passed proof: +`C:\Users\Meta\AppData\Local\Temp\epiphany-workspace-coverage-a577f820-b0e2-4733-b628-529ee3bdb143\proof.json` + +Receipt: `2b44f28f-7c0f-4ad2-aefe-d7c8222e8642`. The proof binds executable +SHA-256 `d262b7f49d45f15c31589bdfb591988cabcfa9ba65564d3e61493dc00b072093`, +the exact embedded smoke-source digest, source head, and dirty tracked-source +diff digest. It verifies both owned collections absent before emitting success. +Operator orchestration restored the +local Qdrant container to its prior stopped state; that is not a harness claim. + +The live rite exposed two real defects and did not pass until both were cut. +Windows boot identity no longer opens privileged PID 4; it queries native +kernel boot time and a focused test proves availability/stability under the +current normal user. Recovery no longer deadlocks its successor: the exact v3 +replacement incarnation may resume its own running claim, while a wrong +incarnation remains contended. The harness uses file-backed supervisor output +because Windows grandchildren can retain captured pipe handles, and it has +exact-incarnation process cleanup and exact-owned-collection cleanup guards. + +Verification: five native process-observation tests, six process/recovery tests, +fourteen supervisor tests, fifteen projector tests plus one ignored live test, +and feature-enabled all-target check pass. The separate live Qdrant/Ollama +projection test passed before the recovery smoke. + +Next: audit deployability against the swarm readiness plan and live host state. +Do not conflate code-complete physiology with the permission-bound reboot/logon +proof; no reboot without explicit live operator approval. diff --git a/state/ledgers.msgpack b/state/ledgers.msgpack index feefbaab..a98f5a9a 100644 Binary files a/state/ledgers.msgpack and b/state/ledgers.msgpack differ diff --git a/state/map.yaml b/state/map.yaml index 3644c023..4f0e089d 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -731,4 +731,5 @@ current_status: - Idunn now has one immutable, host-signed process-termination observation per reserved coverage launch, with no latest/global mirror. Its writer reloads and exact-CAS-preserves the current specialized policy, exact launch, and exact per-launch latest signed heartbeat; it immediately observes the enrolled host's proven boot and exact PID incarnation. A distinct proven boot yields `boot_superseded`; same-boot proof permits only `exact_exited`, `process_missing`, or a complete different `process_replaced` identity. ExactAlive, Inaccessible, Indeterminate, unknown boot, host mismatch, identity collision, and moved source envelopes refuse publication. The injectable observation seam is private to module tests. Recovery-facing authentication reconstructs the entire persisted policy/launch/heartbeat chain rather than trusting the signed termination blob alone. Four hostile document/termination tests and all-target check pass. - Abandoned workspace-coverage authority now has a complete typed transfer. Recovery derives the Body store from the runtime route, reopens the authenticated Body basis, joins obligation/plan/claim to that basis, authenticates current policy plus exact old launch and immutable termination, requires a signed causal termination-to-replacement edge and the replacement's latest ready heartbeat, then performs one Body CAS over Body authority, plan, old claim/attempt, immutable recovery receipt, failed claim history, and epoch+1 successor. The receipt stores exact termination, launch, and heartbeat envelope digests and can reconstruct both CultMesh evidence and current Body authority after restart. Claim/attempt schema and evidence pairs cannot split; impossible completion-before-start timelines refuse. Idunn's reserved reconciler no longer consults generic lifecycle PID receipts for this service. It persists/reuses termination before spawn, uses the specialized bootstrap launch, waits for signed readiness, and then invokes the recovery CAS. One termination admits one replacement slot; concurrent losers are killed when launch persistence loses CAS. - Workspace-coverage launch, heartbeat, and termination now serialize through one typed per-launch process-evidence head. Launch initializes generation 1 as `launched`; each signed heartbeat exact-CAS advances it to `heartbeat`; host-signed termination exact-CAS seals it as `terminated`. Termination v1 accepts exact launch plus enrolled host/boot/process proof before heartbeat sequence one and records heartbeat id/digest only when the head names current signed heartbeat evidence. A heartbeat cannot advance a terminated head. This removes the pre-readiness death strand without allowing heartbeat/termination split authority. - next_action: Build and run a GUID-scoped live workspace-coverage recovery smoke through the real packaged supervisor/projector binaries: initial specialized launch, acquired Body claim, exact child death, immutable termination, unique causal replacement, signed readiness, epoch+1 Body recovery, and restart-time recovery-receipt authentication. Use local Qdrant/Ollama only inside isolated fixture state and restore Qdrant container state afterward. Reboot/logon proof remains permission-bound. + - The feature-gated native `epiphany-workspace-coverage-recovery-smoke` proves the real packaged supervisor/projector boundary in GUID-scoped repo/runtime/Body/Verse state. It warms local Ollama, derives only its exact fenced Qdrant collection names, publishes reserved policy, launches the initial child through fixed bootstrap, observes epoch 1 claim, kills only after exact native launch-incarnation revalidation, invokes Idunn reconcile, authenticates epoch 2 successor authority plus the immutable recovery receipt through freshly reopened stores, kills the replacement under the same identity fence, and verifies only its owned collections absent before success. Live proof `C:\Users\Meta\AppData\Local\Temp\epiphany-workspace-coverage-a577f820-b0e2-4733-b628-529ee3bdb143\proof.json` passed with receipt `2b44f28f-7c0f-4ad2-aefe-d7c8222e8642` and binds the executed binary, exact embedded smoke source, source head, and tracked-source diff digests. Operator orchestration, not the harness, restored local Qdrant to its prior stopped state. The run also repaired normal-user Windows boot identity and exact recovered-owner claim resumption. + next_action: Perform a fresh deployability audit against `notes/epiphany-swarm-readiness-plan.md`, the live machine map, packaged binaries, Task Scheduler presence, local dependency configuration, and current tests. Separate code-complete physiology from the permission-bound reboot/logon survival proof; do not reboot without explicit live operator approval. Cut any remaining non-permission architectural wound before calling the swarm deployable. diff --git a/state/scratch.md b/state/scratch.md index e0edca1f..745bd1b9 100644 --- a/state/scratch.md +++ b/state/scratch.md @@ -166,3 +166,23 @@ seals it. Termination v1 can carry no heartbeat when the exact signed launch and native host/boot/process proof already establish death. A late heartbeat cannot resurrect the launch. Six focused process tests, supervisor/projector suites, and all-target check pass. Next: real GUID-scoped recovery smoke. + +## Live workspace coverage recovery passed (2026-07-16) + +Proof: `C:\Users\Meta\AppData\Local\Temp\epiphany-workspace-coverage-a577f820-b0e2-4733-b628-529ee3bdb143\proof.json`. +Initial epoch 1 owner was killed; Idunn produced exact termination, unique causal +replacement, signed readiness, epoch 2 Body authority, and reconstructable +receipt `2b44f28f-7c0f-4ad2-aefe-d7c8222e8642`. The replacement was killed only +after exact launch-incarnation revalidation and only the two cryptographically +derived fixture collections were eligible for deletion. Operator orchestration +restored `voidbot-qdrant` to stopped. Before emitting success the smoke verified +both owned collections absent. The proof binds the executed binary hash, exact +embedded smoke-source hash, source head, and tracked-source diff digest; it +proves fresh-store reopen authentication, not an OS process restart. + +The run found and cut two last-minute lies: PID 4 was not a normal-user Windows +boot oracle, so boot identity now comes from native kernel boot time; and the +replacement could not resume its own recovered running claim, so exact v3 owner +resumption is allowed while all foreign incarnations remain contended. + +Next: full deployability audit. Reboot/logon proof still requires permission.