diff --git a/epiphany-core/src/bin/epiphany-memory-graph.rs b/epiphany-core/src/bin/epiphany-memory-graph.rs new file mode 100644 index 00000000..cf37098a --- /dev/null +++ b/epiphany-core/src/bin/epiphany-memory-graph.rs @@ -0,0 +1,305 @@ +use anyhow::Context; +use anyhow::Result; +use anyhow::anyhow; +use epiphany_core::EpiphanyMemoryAnchor; +use epiphany_core::EpiphanyMemoryContextQuery; +use epiphany_core::EpiphanyMemoryDomain; +use epiphany_core::EpiphanyMemoryEdge; +use epiphany_core::EpiphanyMemoryEdgeKind; +use epiphany_core::EpiphanyMemoryFreshnessStatus; +use epiphany_core::EpiphanyMemoryGraphSnapshot; +use epiphany_core::EpiphanyMemoryLifecycle; +use epiphany_core::EpiphanyMemoryNode; +use epiphany_core::EpiphanyMemoryNodeKind; +use epiphany_core::EpiphanyMemoryProfile; +use epiphany_core::EpiphanyMemorySummary; +use epiphany_core::derive_memory_graph_freshness; +use epiphany_core::load_memory_graph_snapshot; +use epiphany_core::memory_graph_domain_id; +use epiphany_core::memory_graph_edge_id; +use epiphany_core::memory_graph_node_id; +use epiphany_core::plan_memory_graph_context_cut; +use epiphany_core::validate_memory_graph_snapshot; +use epiphany_core::write_memory_graph_snapshot; +use std::env; +use std::path::PathBuf; + +fn main() -> Result<()> { + let mut args = env::args().skip(1); + let Some(command) = args.next() else { + print_usage(); + std::process::exit(2); + }; + + match command.as_str() { + "status" => { + let store = require_path_arg(&mut args, "--store")?; + let output = memory_graph_status(&store)?; + print_json(&output)?; + } + "validate" => { + let store = require_path_arg(&mut args, "--store")?; + let snapshot = load_memory_graph_snapshot(&store)? + .ok_or_else(|| anyhow!("memory graph store {} is missing", store.display()))?; + let errors = validate_memory_graph_snapshot(&snapshot); + print_json(&serde_json::json!({ + "ok": errors.is_empty(), + "store": store, + "graphId": snapshot.graph_id, + "errors": errors.iter().map(|error| serde_json::json!({ + "path": error.path, + "message": error.message, + })).collect::>(), + }))?; + if !errors.is_empty() { + std::process::exit(1); + } + } + "context" => { + let store = require_path_arg(&mut args, "--store")?; + let query = read_context_query(args)?; + let snapshot = load_memory_graph_snapshot(&store)? + .ok_or_else(|| anyhow!("memory graph store {} is missing", store.display()))?; + let packet = plan_memory_graph_context_cut(&snapshot, &query); + print_json(&packet)?; + } + "smoke" => { + let store = optional_path_arg(&mut args, "--store")? + .unwrap_or_else(|| scoped_temp_store("epiphany-memory-graph-smoke")); + let output = run_smoke(store)?; + let ok = output["ok"].as_bool().unwrap_or(false); + print_json(&output)?; + if !ok { + std::process::exit(1); + } + } + _ => { + print_usage(); + std::process::exit(2); + } + } + Ok(()) +} + +fn memory_graph_status(store: &PathBuf) -> Result { + let Some(snapshot) = load_memory_graph_snapshot(store)? else { + return Ok(serde_json::json!({ + "present": false, + "store": store, + })); + }; + let errors = validate_memory_graph_snapshot(&snapshot); + let freshness = snapshot + .freshness + .clone() + .unwrap_or_else(|| derive_memory_graph_freshness(&snapshot, &[])); + Ok(serde_json::json!({ + "present": true, + "store": store, + "graphId": snapshot.graph_id, + "domains": snapshot.domains.len(), + "nodes": snapshot.nodes.len(), + "edges": snapshot.edges.len(), + "summaries": snapshot.summaries.len(), + "freshness": freshness, + "valid": errors.is_empty(), + "errors": errors.iter().map(|error| serde_json::json!({ + "path": error.path, + "message": error.message, + })).collect::>(), + })) +} + +fn read_context_query(args: impl Iterator) -> Result { + let mut query = EpiphanyMemoryContextQuery { + id: "memory-graph-context-query".to_string(), + ..Default::default() + }; + let mut args = args.peekable(); + while let Some(flag) = args.next() { + match flag.as_str() { + "--query-id" => query.id = next_value(&mut args, "--query-id")?, + "--profile" => { + query.profile = Some(parse_profile(&next_value(&mut args, "--profile")?)?) + } + "--domain-id" => query.domain_ids.push(next_value(&mut args, "--domain-id")?), + "--node-id" => query.node_ids.push(next_value(&mut args, "--node-id")?), + "--edge-id" => query.edge_ids.push(next_value(&mut args, "--edge-id")?), + "--text" => query.text = Some(next_value(&mut args, "--text")?), + "--budget" => { + let value = next_value(&mut args, "--budget")?; + query.budget = Some( + value + .parse::() + .with_context(|| format!("invalid --budget {value:?}"))?, + ); + } + _ => return Err(anyhow!("unexpected context argument {flag:?}")), + } + } + Ok(query) +} + +fn require_path_arg(args: &mut impl Iterator, name: &str) -> Result { + Ok(PathBuf::from(require_string_arg(args, name)?)) +} + +fn require_string_arg(args: &mut impl Iterator, name: &str) -> Result { + let Some(flag) = args.next() else { + return Err(anyhow!("missing {name}")); + }; + if flag != name { + return Err(anyhow!("expected {name}, got {flag}")); + } + args.next() + .with_context(|| format!("missing value for {name}")) +} + +fn optional_path_arg( + args: &mut impl Iterator, + name: &str, +) -> Result> { + let values = args.collect::>(); + if values.is_empty() { + return Ok(None); + } + if values.len() != 2 || values[0] != name { + return Err(anyhow!("expected optional {name} , got {values:?}")); + } + Ok(Some(PathBuf::from(&values[1]))) +} + +fn next_value(args: &mut impl Iterator, name: &str) -> Result { + args.next() + .with_context(|| format!("missing value for {name}")) +} + +fn parse_profile(value: &str) -> Result { + match value { + "repo_architecture" => Ok(EpiphanyMemoryProfile::RepoArchitecture), + "repo_dataflow" => Ok(EpiphanyMemoryProfile::RepoDataflow), + "role_self" => Ok(EpiphanyMemoryProfile::RoleSelf), + "short_term" => Ok(EpiphanyMemoryProfile::ShortTerm), + "incubation" => Ok(EpiphanyMemoryProfile::Incubation), + "agency_pressure" => Ok(EpiphanyMemoryProfile::AgencyPressure), + "candidate_intervention" => Ok(EpiphanyMemoryProfile::CandidateIntervention), + "identity" => Ok(EpiphanyMemoryProfile::Identity), + "evidence" => Ok(EpiphanyMemoryProfile::Evidence), + _ => Err(anyhow!("unknown memory graph profile {value:?}")), + } +} + +fn print_json(value: &T) -> Result<()> { + println!("{}", serde_json::to_string_pretty(value)?); + Ok(()) +} + +fn print_usage() { + eprintln!("usage: epiphany-memory-graph --store ..."); +} + +fn scoped_temp_store(prefix: &str) -> PathBuf { + env::temp_dir().join(format!("{prefix}-{}.msgpack", uuid::Uuid::new_v4())) +} + +fn run_smoke(store: PathBuf) -> Result { + let snapshot = smoke_snapshot(); + write_memory_graph_snapshot(&store, &snapshot)?; + let status = memory_graph_status(&store)?; + let packet = plan_memory_graph_context_cut( + &snapshot, + &EpiphanyMemoryContextQuery { + id: "smoke-query".to_string(), + profile: Some(EpiphanyMemoryProfile::RepoArchitecture), + text: Some("shared graph law".to_string()), + ..Default::default() + }, + ); + Ok(serde_json::json!({ + "ok": status["valid"].as_bool().unwrap_or(false) + && packet.summaries.len() == 1 + && packet.nodes.is_empty(), + "store": store, + "status": status, + "contextPacket": packet, + })) +} + +fn smoke_snapshot() -> EpiphanyMemoryGraphSnapshot { + let domain_id = memory_graph_domain_id( + EpiphanyMemoryProfile::RepoArchitecture, + "crate", + "epiphany-core", + ); + let node_id = memory_graph_node_id( + &domain_id, + "module", + "epiphany-core/src/memory_graph.rs", + Some("memory_graph"), + ); + EpiphanyMemoryGraphSnapshot { + graph_id: "memory-graph-smoke".to_string(), + domains: vec![EpiphanyMemoryDomain { + id: domain_id.clone(), + profile: EpiphanyMemoryProfile::RepoArchitecture, + title: "epiphany-core".to_string(), + lifecycle: EpiphanyMemoryLifecycle::Accepted, + ..Default::default() + }], + nodes: vec![EpiphanyMemoryNode { + id: node_id.clone(), + domain_id: domain_id.clone(), + profile: EpiphanyMemoryProfile::RepoArchitecture, + kind: EpiphanyMemoryNodeKind::Module, + title: "memory_graph".to_string(), + claim: "Shared memory graph law owns repo and agent memory graph invariants." + .to_string(), + question: "Which profile-specific producers are allowed after the shared store?" + .to_string(), + tension: String::new(), + action_implication: "Add profile producers only after the shared typed store works." + .to_string(), + anchors: vec![EpiphanyMemoryAnchor { + id: "anchor-memory-graph-rs".to_string(), + kind: "source".to_string(), + target: "epiphany-core/src/memory_graph.rs".to_string(), + source_hash: Some("smoke-hash".to_string()), + ..Default::default() + }], + source_hashes: vec!["smoke-hash".to_string()], + lifecycle: EpiphanyMemoryLifecycle::Accepted, + confidence: 90, + salience: 80, + ..Default::default() + }], + edges: vec![EpiphanyMemoryEdge { + id: memory_graph_edge_id(&node_id, &node_id, "verifies", ["smoke"]), + source_id: node_id.clone(), + target_id: node_id.clone(), + kind: EpiphanyMemoryEdgeKind::Verifies, + profile: EpiphanyMemoryProfile::RepoArchitecture, + claim: "The smoke packet verifies the memory graph context cut can use summaries." + .to_string(), + lifecycle: EpiphanyMemoryLifecycle::Accepted, + confidence: 80, + ..Default::default() + }], + summaries: vec![EpiphanyMemorySummary { + id: "summary-memory-graph-smoke".to_string(), + domain_id, + covers_node_ids: vec![node_id], + target: "memory_graph".to_string(), + claim: "The shared graph skeleton can persist and return context without Qdrant." + .to_string(), + question: "Which producer should populate it first?".to_string(), + tension: String::new(), + action_implication: "Use this store before scanner or sleep-runner work.".to_string(), + anchor_count: 1, + source_hashes: vec!["smoke-hash".to_string()], + freshness: EpiphanyMemoryFreshnessStatus::Ready, + confidence: 95, + ..Default::default() + }], + ..Default::default() + } +} diff --git a/epiphany-core/src/lib.rs b/epiphany-core/src/lib.rs index 1e148d01..85f8c59b 100644 --- a/epiphany-core/src/lib.rs +++ b/epiphany-core/src/lib.rs @@ -1,6 +1,7 @@ mod agent_memory; mod distillation; mod heartbeat_state; +mod memory_graph; mod promotion; mod prompt; mod proposal; @@ -38,6 +39,17 @@ pub use cultcache_rs::SingleFileMessagePackBackingStore; pub use distillation::EpiphanyDistillInput; pub use distillation::EpiphanyDistillProposal; pub use distillation::distill_observation; +pub use epiphany_state_model::EpiphanyMemoryAnchor; +pub use epiphany_state_model::EpiphanyMemoryContextPacket; +pub use epiphany_state_model::EpiphanyMemoryContextQuery; +pub use epiphany_state_model::EpiphanyMemoryDomain; +pub use epiphany_state_model::EpiphanyMemoryEdge; +pub use epiphany_state_model::EpiphanyMemoryFreshness; +pub use epiphany_state_model::EpiphanyMemoryFreshnessStatus; +pub use epiphany_state_model::EpiphanyMemoryLifecycle; +pub use epiphany_state_model::EpiphanyMemoryNode; +pub use epiphany_state_model::EpiphanyMemoryProfile; +pub use epiphany_state_model::EpiphanyMemorySummary; pub use heartbeat_state::EpiphanyHeartbeatCognitionEntry; pub use heartbeat_state::EpiphanyHeartbeatStateEntry; pub use heartbeat_state::GhostlightSceneParticipantSeed; @@ -72,6 +84,30 @@ pub use heartbeat_state::validate_heartbeat_cognition; pub use heartbeat_state::validate_heartbeat_state; pub use heartbeat_state::write_heartbeat_cognition_entry; pub use heartbeat_state::write_heartbeat_state_entry; +pub use memory_graph::EpiphanyMemoryEdgeKind; +pub use memory_graph::EpiphanyMemoryEmbeddingManifest; +pub use memory_graph::EpiphanyMemoryGraphEntry; +pub use memory_graph::EpiphanyMemoryGraphSnapshot; +pub use memory_graph::EpiphanyMemoryGraphValidationError; +pub use memory_graph::EpiphanyMemoryLifecycleReceipt; +pub use memory_graph::EpiphanyMemoryNodeKind; +pub use memory_graph::EpiphanyMemoryPatchCandidate; +pub use memory_graph::MEMORY_GRAPH_KEY; +pub use memory_graph::MEMORY_GRAPH_SCHEMA_VERSION; +pub use memory_graph::MEMORY_GRAPH_TYPE; +pub use memory_graph::derive_memory_graph_freshness; +pub use memory_graph::lifecycle_allowed_for_profile; +pub use memory_graph::load_memory_graph_entry; +pub use memory_graph::load_memory_graph_snapshot; +pub use memory_graph::memory_graph_cache; +pub use memory_graph::memory_graph_domain_id; +pub use memory_graph::memory_graph_edge_id; +pub use memory_graph::memory_graph_node_id; +pub use memory_graph::plan_memory_graph_context_cut; +pub use memory_graph::validate_memory_graph_entry; +pub use memory_graph::validate_memory_graph_snapshot; +pub use memory_graph::write_memory_graph_entry; +pub use memory_graph::write_memory_graph_snapshot; pub use promotion::EpiphanyPromotionDecision; pub use promotion::EpiphanyPromotionInput; pub use promotion::EpiphanyStateReplacementValidationInput; diff --git a/epiphany-core/src/memory_graph/context_cut.rs b/epiphany-core/src/memory_graph/context_cut.rs new file mode 100644 index 00000000..828775f4 --- /dev/null +++ b/epiphany-core/src/memory_graph/context_cut.rs @@ -0,0 +1,294 @@ +use super::EpiphanyMemoryAnchor; +use super::EpiphanyMemoryContextPacket; +use super::EpiphanyMemoryContextQuery; +use super::EpiphanyMemoryEdge; +use super::EpiphanyMemoryFreshnessStatus; +use super::EpiphanyMemoryGraphSnapshot; +use super::EpiphanyMemoryNode; +use super::EpiphanyMemoryProfile; +use super::EpiphanyMemorySummary; +use super::derive_memory_graph_freshness; +use super::ids::normalized_key; +use super::ids::stable_memory_graph_id; +use std::collections::HashMap; +use std::collections::HashSet; + +pub fn plan_memory_graph_context_cut( + snapshot: &EpiphanyMemoryGraphSnapshot, + query: &EpiphanyMemoryContextQuery, +) -> EpiphanyMemoryContextPacket { + let budget = query.budget.unwrap_or(12).clamp(1, 64) as usize; + let freshness = snapshot + .freshness + .clone() + .unwrap_or_else(|| derive_memory_graph_freshness(snapshot, &[])); + let stale_nodes: HashSet<&str> = freshness + .stale_node_ids + .iter() + .map(String::as_str) + .collect(); + let stale_edges: HashSet<&str> = freshness + .stale_edge_ids + .iter() + .map(String::as_str) + .collect(); + let stale_summaries: HashSet<&str> = freshness + .stale_summary_ids + .iter() + .map(String::as_str) + .collect(); + + let domains = query + .domain_ids + .iter() + .map(String::as_str) + .collect::>(); + let requested_nodes = query + .node_ids + .iter() + .map(String::as_str) + .collect::>(); + let requested_edges = query + .edge_ids + .iter() + .map(String::as_str) + .collect::>(); + let terms = query_terms(query); + + let node_by_id = snapshot + .nodes + .iter() + .map(|node| (node.id.as_str(), node)) + .collect::>(); + let edge_by_id = snapshot + .edges + .iter() + .map(|edge| (edge.id.as_str(), edge)) + .collect::>(); + + let mut nodes = Vec::new(); + let mut edges = Vec::new(); + let mut summaries = Vec::new(); + let mut warnings = Vec::new(); + + for summary in &snapshot.summaries { + if summaries.len() >= budget { + break; + } + if !summary_matches(summary, query.profile, &domains, &terms, &node_by_id) { + continue; + } + if summary_is_usable(summary, &stale_summaries) { + summaries.push(summary.clone()); + continue; + } + warnings.push(format!( + "summary {} is stale or low-confidence; descended into children", + summary.id + )); + include_summary_children( + summary, + &node_by_id, + &edge_by_id, + &stale_nodes, + &stale_edges, + budget, + &mut nodes, + &mut edges, + ); + } + + for node_id in &query.node_ids { + if let Some(node) = node_by_id.get(node_id.as_str()) { + push_unique_node(&mut nodes, (*node).clone(), budget); + } + } + for edge_id in &query.edge_ids { + if let Some(edge) = edge_by_id.get(edge_id.as_str()) { + push_unique_edge(&mut edges, (*edge).clone(), budget); + if let Some(node) = node_by_id.get(edge.source_id.as_str()) { + push_unique_node(&mut nodes, (*node).clone(), budget); + } + if let Some(node) = node_by_id.get(edge.target_id.as_str()) { + push_unique_node(&mut nodes, (*node).clone(), budget); + } + } + } + + if summaries.is_empty() && nodes.is_empty() { + for node in &snapshot.nodes { + if nodes.len() >= budget { + break; + } + if node_matches(node, query.profile, &domains, &terms) { + push_unique_node(&mut nodes, node.clone(), budget); + } + } + } + + let anchors = unique_anchors( + nodes + .iter() + .flat_map(|node| node.anchors.iter()) + .chain(edges.iter().flat_map(|edge| edge.anchors.iter())), + ); + + EpiphanyMemoryContextPacket { + id: stable_memory_graph_id("memctx", [query.id.as_str(), snapshot.graph_id.as_str()]), + query_id: query.id.clone(), + nodes, + edges, + summaries, + anchors, + warnings, + missing_node_ids: query + .node_ids + .iter() + .filter(|id| !requested_nodes.is_empty() && !node_by_id.contains_key(id.as_str())) + .cloned() + .collect(), + missing_edge_ids: query + .edge_ids + .iter() + .filter(|id| !requested_edges.is_empty() && !edge_by_id.contains_key(id.as_str())) + .cloned() + .collect(), + } +} + +fn query_terms(query: &EpiphanyMemoryContextQuery) -> Vec { + query + .text + .as_deref() + .unwrap_or_default() + .split(|ch: char| !ch.is_alphanumeric() && ch != '_') + .filter(|term| term.len() > 2) + .map(normalized_key) + .collect() +} + +fn summary_matches( + summary: &EpiphanyMemorySummary, + profile: Option, + domain_ids: &HashSet<&str>, + terms: &[String], + node_by_id: &HashMap<&str, &EpiphanyMemoryNode>, +) -> bool { + if !domain_ids.is_empty() && !domain_ids.contains(summary.domain_id.as_str()) { + return false; + } + if let Some(profile) = profile { + let covers_profile = summary + .covers_node_ids + .iter() + .filter_map(|id| node_by_id.get(id.as_str())) + .any(|node| node.profile == profile); + if !covers_profile { + return false; + } + } + terms.is_empty() + || text_matches_terms( + [ + &summary.target, + &summary.claim, + &summary.question, + &summary.tension, + ], + terms, + ) +} + +fn node_matches( + node: &EpiphanyMemoryNode, + profile: Option, + domain_ids: &HashSet<&str>, + terms: &[String], +) -> bool { + if profile.is_some_and(|profile| node.profile != profile) { + return false; + } + if !domain_ids.is_empty() && !domain_ids.contains(node.domain_id.as_str()) { + return false; + } + terms.is_empty() + || text_matches_terms( + [ + &node.title, + &node.claim, + &node.question, + &node.tension, + &node.action_implication, + ], + terms, + ) +} + +fn text_matches_terms<'a>(values: impl IntoIterator, terms: &[String]) -> bool { + let joined = values + .into_iter() + .map(|value| value.to_lowercase()) + .collect::>() + .join(" "); + terms.iter().any(|term| joined.contains(term)) +} + +fn summary_is_usable(summary: &EpiphanyMemorySummary, stale_summaries: &HashSet<&str>) -> bool { + summary.freshness == EpiphanyMemoryFreshnessStatus::Ready + && summary.confidence >= 70 + && !stale_summaries.contains(summary.id.as_str()) + && summary.known_omissions.is_empty() +} + +fn include_summary_children( + summary: &EpiphanyMemorySummary, + node_by_id: &HashMap<&str, &EpiphanyMemoryNode>, + edge_by_id: &HashMap<&str, &EpiphanyMemoryEdge>, + stale_nodes: &HashSet<&str>, + stale_edges: &HashSet<&str>, + budget: usize, + nodes: &mut Vec, + edges: &mut Vec, +) { + for node_id in &summary.covers_node_ids { + if let Some(node) = node_by_id.get(node_id.as_str()) { + push_unique_node(nodes, (*node).clone(), budget); + if stale_nodes.contains(node.id.as_str()) { + continue; + } + } + } + for edge_id in &summary.covers_edge_ids { + if let Some(edge) = edge_by_id.get(edge_id.as_str()) { + push_unique_edge(edges, (*edge).clone(), budget); + if stale_edges.contains(edge.id.as_str()) { + continue; + } + } + } +} + +fn unique_anchors<'a>( + anchors: impl IntoIterator, +) -> Vec { + let mut out = Vec::new(); + let mut seen = HashSet::new(); + for anchor in anchors { + if seen.insert(anchor.id.as_str()) { + out.push(anchor.clone()); + } + } + out +} + +fn push_unique_node(target: &mut Vec, node: EpiphanyMemoryNode, budget: usize) { + if target.len() < budget && !target.iter().any(|existing| existing.id == node.id) { + target.push(node); + } +} + +fn push_unique_edge(target: &mut Vec, edge: EpiphanyMemoryEdge, budget: usize) { + if target.len() < budget && !target.iter().any(|existing| existing.id == edge.id) { + target.push(edge); + } +} diff --git a/epiphany-core/src/memory_graph/freshness.rs b/epiphany-core/src/memory_graph/freshness.rs new file mode 100644 index 00000000..ef3d2cc5 --- /dev/null +++ b/epiphany-core/src/memory_graph/freshness.rs @@ -0,0 +1,84 @@ +use super::EpiphanyMemoryFreshness; +use super::EpiphanyMemoryFreshnessStatus; +use super::EpiphanyMemoryGraphSnapshot; +use super::EpiphanyMemoryLifecycle; +use super::push_unique; +use super::unique_strings; +use std::collections::HashSet; + +pub fn derive_memory_graph_freshness( + snapshot: &EpiphanyMemoryGraphSnapshot, + dirty_source_hashes: &[String], +) -> EpiphanyMemoryFreshness { + let dirty: HashSet<&str> = dirty_source_hashes.iter().map(String::as_str).collect(); + let mut stale_node_ids = Vec::new(); + let mut stale_edge_ids = Vec::new(); + let mut stale_summary_ids = Vec::new(); + + for node in &snapshot.nodes { + if node.lifecycle == EpiphanyMemoryLifecycle::Stale + || node + .source_hashes + .iter() + .any(|hash| dirty.contains(hash.as_str())) + { + push_unique(&mut stale_node_ids, node.id.clone()); + } + } + + for edge in &snapshot.edges { + if edge.lifecycle == EpiphanyMemoryLifecycle::Stale + || edge + .anchors + .iter() + .filter_map(|anchor| anchor.source_hash.as_deref()) + .any(|hash| dirty.contains(hash)) + { + push_unique(&mut stale_edge_ids, edge.id.clone()); + } + } + + for summary in &snapshot.summaries { + if summary.freshness == EpiphanyMemoryFreshnessStatus::Stale + || summary + .source_hashes + .iter() + .any(|hash| dirty.contains(hash.as_str())) + || summary + .covers_node_ids + .iter() + .any(|id| stale_node_ids.iter().any(|stale| stale == id)) + || summary + .covers_edge_ids + .iter() + .any(|id| stale_edge_ids.iter().any(|stale| stale == id)) + { + push_unique(&mut stale_summary_ids, summary.id.clone()); + } + } + + let status = if stale_node_ids.is_empty() + && stale_edge_ids.is_empty() + && stale_summary_ids.is_empty() + && dirty_source_hashes.is_empty() + { + EpiphanyMemoryFreshnessStatus::Ready + } else { + EpiphanyMemoryFreshnessStatus::Stale + }; + + EpiphanyMemoryFreshness { + status, + stale_node_ids, + stale_edge_ids, + stale_summary_ids, + dirty_source_hashes: unique_strings(dirty_source_hashes.iter().cloned()), + note: Some(match status { + EpiphanyMemoryFreshnessStatus::Ready => "Memory graph is ready.".to_string(), + EpiphanyMemoryFreshnessStatus::Stale => { + "Memory graph has stale source-backed records.".to_string() + } + _ => "Memory graph freshness is not derived.".to_string(), + }), + } +} diff --git a/epiphany-core/src/memory_graph/ids.rs b/epiphany-core/src/memory_graph/ids.rs new file mode 100644 index 00000000..67e44973 --- /dev/null +++ b/epiphany-core/src/memory_graph/ids.rs @@ -0,0 +1,87 @@ +use super::EpiphanyMemoryProfile; +use sha1::Digest; +use sha1::Sha1; + +pub fn memory_graph_domain_id( + profile: EpiphanyMemoryProfile, + kind: impl AsRef, + path_or_name: impl AsRef, +) -> String { + let normalized = normalized_key(path_or_name.as_ref()); + stable_memory_graph_id( + "memdom", + [profile_key(profile), kind.as_ref(), normalized.as_str()], + ) +} + +pub fn memory_graph_node_id( + domain_id: impl AsRef, + kind: impl AsRef, + path: impl AsRef, + symbol: Option<&str>, +) -> String { + let normalized = normalized_key(path.as_ref()); + stable_memory_graph_id( + "memnode", + [ + domain_id.as_ref(), + kind.as_ref(), + normalized.as_str(), + symbol.unwrap_or_default(), + ], + ) +} + +pub fn memory_graph_edge_id( + source_id: impl AsRef, + target_id: impl AsRef, + kind: impl AsRef, + anchor_keys: impl IntoIterator>, +) -> String { + let mut parts = vec![ + source_id.as_ref().to_string(), + target_id.as_ref().to_string(), + kind.as_ref().to_string(), + ]; + let mut anchors = anchor_keys + .into_iter() + .map(|key| normalized_key(key.as_ref())) + .collect::>(); + anchors.sort(); + parts.extend(anchors); + stable_memory_graph_id("memedge", parts.iter().map(String::as_str)) +} + +pub(crate) fn stable_memory_graph_id<'a>( + prefix: &str, + parts: impl IntoIterator, +) -> String { + let mut hasher = Sha1::new(); + for part in parts { + hasher.update(part.trim().to_lowercase().as_bytes()); + hasher.update([0]); + } + format!("{prefix}-{:x}", hasher.finalize()) +} + +fn profile_key(profile: EpiphanyMemoryProfile) -> &'static str { + match profile { + EpiphanyMemoryProfile::RepoArchitecture => "repo_architecture", + EpiphanyMemoryProfile::RepoDataflow => "repo_dataflow", + EpiphanyMemoryProfile::RoleSelf => "role_self", + EpiphanyMemoryProfile::ShortTerm => "short_term", + EpiphanyMemoryProfile::Incubation => "incubation", + EpiphanyMemoryProfile::AgencyPressure => "agency_pressure", + EpiphanyMemoryProfile::CandidateIntervention => "candidate_intervention", + EpiphanyMemoryProfile::Identity => "identity", + EpiphanyMemoryProfile::Evidence => "evidence", + } +} + +pub(crate) fn normalized_key(value: &str) -> String { + value + .trim() + .replace('\\', "/") + .trim_start_matches("./") + .to_lowercase() +} diff --git a/epiphany-core/src/memory_graph/mod.rs b/epiphany-core/src/memory_graph/mod.rs new file mode 100644 index 00000000..dd6dabf3 --- /dev/null +++ b/epiphany-core/src/memory_graph/mod.rs @@ -0,0 +1,59 @@ +pub mod context_cut; +pub mod freshness; +pub mod ids; +pub mod store; +pub mod validation; + +pub use epiphany_state_model::EpiphanyMemoryAnchor; +pub use epiphany_state_model::EpiphanyMemoryContextPacket; +pub use epiphany_state_model::EpiphanyMemoryContextQuery; +pub use epiphany_state_model::EpiphanyMemoryDomain; +pub use epiphany_state_model::EpiphanyMemoryEdge; +pub use epiphany_state_model::EpiphanyMemoryEdgeKind; +pub use epiphany_state_model::EpiphanyMemoryEmbeddingManifest; +pub use epiphany_state_model::EpiphanyMemoryFreshness; +pub use epiphany_state_model::EpiphanyMemoryFreshnessStatus; +pub use epiphany_state_model::EpiphanyMemoryGraphSnapshot; +pub use epiphany_state_model::EpiphanyMemoryLifecycle; +pub use epiphany_state_model::EpiphanyMemoryLifecycleReceipt; +pub use epiphany_state_model::EpiphanyMemoryNode; +pub use epiphany_state_model::EpiphanyMemoryNodeKind; +pub use epiphany_state_model::EpiphanyMemoryPatchCandidate; +pub use epiphany_state_model::EpiphanyMemoryProfile; +pub use epiphany_state_model::EpiphanyMemorySummary; + +pub use context_cut::plan_memory_graph_context_cut; +pub use freshness::derive_memory_graph_freshness; +pub use ids::memory_graph_domain_id; +pub use ids::memory_graph_edge_id; +pub use ids::memory_graph_node_id; +pub use store::EpiphanyMemoryGraphEntry; +pub use store::MEMORY_GRAPH_KEY; +pub use store::MEMORY_GRAPH_SCHEMA_VERSION; +pub use store::MEMORY_GRAPH_TYPE; +pub use store::load_memory_graph_entry; +pub use store::load_memory_graph_snapshot; +pub use store::memory_graph_cache; +pub use store::validate_memory_graph_entry; +pub use store::write_memory_graph_entry; +pub use store::write_memory_graph_snapshot; +pub use validation::EpiphanyMemoryGraphValidationError; +pub use validation::lifecycle_allowed_for_profile; +pub use validation::validate_memory_graph_snapshot; + +pub(crate) fn push_unique(target: &mut Vec, value: String) { + if !target.iter().any(|existing| existing == &value) { + target.push(value); + } +} + +pub(crate) fn unique_strings(values: impl IntoIterator) -> Vec { + let mut out = Vec::new(); + for value in values { + push_unique(&mut out, value); + } + out +} + +#[cfg(test)] +mod tests; diff --git a/epiphany-core/src/memory_graph/store.rs b/epiphany-core/src/memory_graph/store.rs new file mode 100644 index 00000000..1df8155f --- /dev/null +++ b/epiphany-core/src/memory_graph/store.rs @@ -0,0 +1,109 @@ +use super::EpiphanyMemoryGraphSnapshot; +use super::validate_memory_graph_snapshot; +use anyhow::Result; +use anyhow::anyhow; +use cultcache_rs::CultCache; +use cultcache_rs::DatabaseEntry; +use cultcache_rs::SingleFileMessagePackBackingStore; +use std::path::Path; + +pub const MEMORY_GRAPH_TYPE: &str = "epiphany.memory_graph"; +pub const MEMORY_GRAPH_KEY: &str = "default"; +pub const MEMORY_GRAPH_SCHEMA_VERSION: &str = "epiphany.memory_graph.v0"; + +#[derive(Clone, Debug, PartialEq, DatabaseEntry)] +#[cultcache(type = "epiphany.memory_graph", schema = "EpiphanyMemoryGraphEntry")] +pub struct EpiphanyMemoryGraphEntry { + #[cultcache(key = 0)] + pub schema_version: String, + #[cultcache(key = 1)] + pub graph_id: String, + #[cultcache(key = 2)] + pub snapshot_msgpack: Vec, +} + +impl EpiphanyMemoryGraphEntry { + pub fn from_snapshot(snapshot: &EpiphanyMemoryGraphSnapshot) -> Result { + let snapshot_msgpack = rmp_serde::to_vec_named(snapshot)?; + Ok(Self { + schema_version: MEMORY_GRAPH_SCHEMA_VERSION.to_string(), + graph_id: snapshot.graph_id.clone(), + snapshot_msgpack, + }) + } + + pub fn snapshot(&self) -> Result { + rmp_serde::from_slice(&self.snapshot_msgpack) + .map_err(|error| anyhow!("failed to decode memory graph snapshot MessagePack: {error}")) + } +} + +pub fn memory_graph_cache(store_path: impl AsRef) -> Result { + let mut cache = CultCache::new(); + cache.register_entry_type::()?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(store_path.as_ref())); + cache.pull_all_backing_stores()?; + Ok(cache) +} + +pub fn load_memory_graph_entry( + store_path: impl AsRef, +) -> Result> { + let cache = memory_graph_cache(store_path)?; + cache.get::(MEMORY_GRAPH_KEY) +} + +pub fn load_memory_graph_snapshot( + store_path: impl AsRef, +) -> Result> { + let Some(entry) = load_memory_graph_entry(store_path)? else { + return Ok(None); + }; + validate_memory_graph_entry(&entry)?; + Ok(Some(entry.snapshot()?)) +} + +pub fn write_memory_graph_entry( + store_path: impl AsRef, + entry: &EpiphanyMemoryGraphEntry, +) -> Result { + validate_memory_graph_entry(entry)?; + let mut cache = memory_graph_cache(store_path)?; + cache.put(MEMORY_GRAPH_KEY, entry) +} + +pub fn write_memory_graph_snapshot( + store_path: impl AsRef, + snapshot: &EpiphanyMemoryGraphSnapshot, +) -> Result { + let entry = EpiphanyMemoryGraphEntry::from_snapshot(snapshot)?; + write_memory_graph_entry(store_path, &entry) +} + +pub fn validate_memory_graph_entry(entry: &EpiphanyMemoryGraphEntry) -> Result<()> { + if entry.schema_version != MEMORY_GRAPH_SCHEMA_VERSION { + return Err(anyhow!( + "memory graph schema_version is {:?}, expected {:?}", + entry.schema_version, + MEMORY_GRAPH_SCHEMA_VERSION + )); + } + let snapshot = entry.snapshot()?; + if entry.graph_id != snapshot.graph_id { + return Err(anyhow!( + "memory graph entry graph_id {:?} does not match snapshot graph_id {:?}", + entry.graph_id, + snapshot.graph_id + )); + } + let errors = validate_memory_graph_snapshot(&snapshot); + if !errors.is_empty() { + let message = errors + .iter() + .map(|error| format!("{}: {}", error.path, error.message)) + .collect::>() + .join("; "); + return Err(anyhow!("memory graph validation failed: {message}")); + } + Ok(()) +} diff --git a/epiphany-core/src/memory_graph/tests.rs b/epiphany-core/src/memory_graph/tests.rs new file mode 100644 index 00000000..e8989f91 --- /dev/null +++ b/epiphany-core/src/memory_graph/tests.rs @@ -0,0 +1,280 @@ +use super::*; +use anyhow::Result; +use epiphany_state_model::EpiphanyMemoryEdgeKind; +use epiphany_state_model::EpiphanyMemoryNodeKind; + +fn anchor(id: &str, hash: &str) -> EpiphanyMemoryAnchor { + EpiphanyMemoryAnchor { + id: id.to_string(), + kind: "source".to_string(), + target: format!("src/{id}.rs"), + source_hash: Some(hash.to_string()), + ..Default::default() + } +} + +fn fixture_snapshot() -> EpiphanyMemoryGraphSnapshot { + let domain_id = memory_graph_domain_id( + EpiphanyMemoryProfile::RepoArchitecture, + "crate", + "epiphany-core", + ); + let node_a = memory_graph_node_id(&domain_id, "module", "src/memory_graph.rs", Some("a")); + let node_b = memory_graph_node_id(&domain_id, "module", "src/memory_graph.rs", Some("b")); + let edge = memory_graph_edge_id(&node_a, &node_b, "owns", ["src/memory_graph.rs"]); + EpiphanyMemoryGraphSnapshot { + graph_id: "graph".to_string(), + domains: vec![EpiphanyMemoryDomain { + id: domain_id.clone(), + profile: EpiphanyMemoryProfile::RepoArchitecture, + title: "Core".to_string(), + lifecycle: EpiphanyMemoryLifecycle::Accepted, + ..Default::default() + }], + nodes: vec![ + EpiphanyMemoryNode { + id: node_a.clone(), + domain_id: domain_id.clone(), + profile: EpiphanyMemoryProfile::RepoArchitecture, + kind: EpiphanyMemoryNodeKind::Module, + title: "Memory graph".to_string(), + claim: "Memory graph owns shared graph law.".to_string(), + question: "How much profile policy belongs here?".to_string(), + tension: String::new(), + action_implication: "Route repo and agent memory through this module.".to_string(), + anchors: vec![anchor("a", "hash-a")], + source_hashes: vec!["hash-a".to_string()], + lifecycle: EpiphanyMemoryLifecycle::Accepted, + confidence: 90, + ..Default::default() + }, + EpiphanyMemoryNode { + id: node_b.clone(), + domain_id: domain_id.clone(), + profile: EpiphanyMemoryProfile::RepoArchitecture, + kind: EpiphanyMemoryNodeKind::TestSeam, + title: "Validation tests".to_string(), + claim: "Validation tests pin the shared graph skeleton.".to_string(), + question: String::new(), + tension: "Without them profile code can bypass graph law.".to_string(), + action_implication: "Run memory_graph tests before profile work.".to_string(), + anchors: vec![anchor("b", "hash-b")], + source_hashes: vec!["hash-b".to_string()], + lifecycle: EpiphanyMemoryLifecycle::Accepted, + confidence: 85, + ..Default::default() + }, + ], + edges: vec![EpiphanyMemoryEdge { + id: edge.clone(), + source_id: node_a.clone(), + target_id: node_b.clone(), + kind: EpiphanyMemoryEdgeKind::Verifies, + profile: EpiphanyMemoryProfile::RepoArchitecture, + claim: "Validation tests verify graph law.".to_string(), + anchors: vec![anchor("edge", "hash-a")], + lifecycle: EpiphanyMemoryLifecycle::Accepted, + confidence: 80, + }], + summaries: vec![EpiphanyMemorySummary { + id: "summary-1".to_string(), + domain_id, + covers_node_ids: vec![node_a, node_b], + covers_edge_ids: vec![edge], + target: "memory_graph".to_string(), + claim: "Memory graph owns shared repo and agent memory graph law.".to_string(), + question: "Which profile policy is allowed at the substrate?".to_string(), + tension: String::new(), + action_implication: "Use shared context packets before profile producers.".to_string(), + anchor_count: 3, + source_hashes: vec!["hash-a".to_string(), "hash-b".to_string()], + freshness: EpiphanyMemoryFreshnessStatus::Ready, + confidence: 92, + ..Default::default() + }], + ..Default::default() + } +} + +#[test] +fn memory_graph_ids_are_stable_and_normalized() { + let a = memory_graph_node_id("domain", "module", ".\\Src\\Lib.rs", Some("Thing")); + let b = memory_graph_node_id("domain", "module", "src/lib.rs", Some("Thing")); + assert_eq!(a, b); + assert!(a.starts_with("memnode-")); +} + +#[test] +fn memory_graph_documents_round_trip_through_serde() { + let snapshot = fixture_snapshot(); + + let encoded = serde_json::to_string(&snapshot).expect("serialize memory graph snapshot"); + let decoded: EpiphanyMemoryGraphSnapshot = + serde_json::from_str(&encoded).expect("deserialize memory graph snapshot"); + + assert_eq!(decoded, snapshot); + assert_eq!( + decoded.nodes[0].profile, + EpiphanyMemoryProfile::RepoArchitecture + ); +} + +#[test] +fn memory_graph_entry_round_trips_through_cultcache() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("memory-graph.msgpack"); + let snapshot = fixture_snapshot(); + + let written = write_memory_graph_snapshot(&store_path, &snapshot)?; + let loaded = + load_memory_graph_snapshot(&store_path)?.expect("memory graph snapshot should load"); + + assert_eq!(written.graph_id, snapshot.graph_id); + assert_eq!(loaded, snapshot); + Ok(()) +} + +#[test] +fn memory_graph_entry_validation_rejects_mismatched_identity() { + let snapshot = fixture_snapshot(); + let mut entry = EpiphanyMemoryGraphEntry::from_snapshot(&snapshot).expect("snapshot entry"); + entry.graph_id = "other".to_string(); + + let error = validate_memory_graph_entry(&entry).expect_err("identity mismatch is invalid"); + + assert!(error.to_string().contains("does not match snapshot")); +} + +#[test] +fn memory_graph_profile_law_keeps_role_memory_from_repo_lifecycle() { + assert!(lifecycle_allowed_for_profile( + EpiphanyMemoryProfile::RepoArchitecture, + EpiphanyMemoryLifecycle::Observed + )); + assert!(!lifecycle_allowed_for_profile( + EpiphanyMemoryProfile::RoleSelf, + EpiphanyMemoryLifecycle::Observed + )); + assert!(!lifecycle_allowed_for_profile( + EpiphanyMemoryProfile::RepoDataflow, + EpiphanyMemoryLifecycle::Spoken + )); +} + +#[test] +fn memory_graph_validation_rejects_missing_edge_node_and_bad_lifecycle() { + let mut snapshot = fixture_snapshot(); + snapshot.edges[0].target_id = "missing".to_string(); + snapshot.nodes[0].lifecycle = EpiphanyMemoryLifecycle::Spoken; + + let errors = validate_memory_graph_snapshot(&snapshot); + + assert!( + errors + .iter() + .any(|error| error.message.contains("missing target node")) + ); + assert!( + errors + .iter() + .any(|error| error.message.contains("lifecycle is not legal")) + ); +} + +#[test] +fn memory_graph_validation_requires_conservative_summary_anatomy() { + let mut snapshot = fixture_snapshot(); + snapshot.summaries[0].action_implication.clear(); + snapshot.summaries[0] + .covers_node_ids + .push("missing".to_string()); + + let errors = validate_memory_graph_snapshot(&snapshot); + + assert!( + errors + .iter() + .any(|error| error.message.contains("action implication")) + ); + assert!( + errors + .iter() + .any(|error| error.message.contains("covers missing node")) + ); +} + +#[test] +fn memory_graph_freshness_propagates_dirty_source_hash_to_summary() { + let snapshot = fixture_snapshot(); + + let freshness = derive_memory_graph_freshness(&snapshot, &["hash-a".to_string()]); + + assert_eq!(freshness.status, EpiphanyMemoryFreshnessStatus::Stale); + assert_eq!(freshness.stale_node_ids.len(), 1); + assert_eq!(freshness.stale_edge_ids.len(), 1); + assert_eq!(freshness.stale_summary_ids, vec!["summary-1".to_string()]); +} + +#[test] +fn memory_graph_context_cut_uses_fresh_high_confidence_summary() { + let snapshot = fixture_snapshot(); + let packet = plan_memory_graph_context_cut( + &snapshot, + &EpiphanyMemoryContextQuery { + id: "query".to_string(), + profile: Some(EpiphanyMemoryProfile::RepoArchitecture), + text: Some("shared graph law".to_string()), + ..Default::default() + }, + ); + + assert_eq!(packet.summaries.len(), 1); + assert!(packet.nodes.is_empty()); + assert!(packet.warnings.is_empty()); +} + +#[test] +fn memory_graph_context_cut_descends_when_summary_is_stale() { + let mut snapshot = fixture_snapshot(); + snapshot.freshness = Some(derive_memory_graph_freshness( + &snapshot, + &["hash-a".to_string()], + )); + let packet = plan_memory_graph_context_cut( + &snapshot, + &EpiphanyMemoryContextQuery { + id: "query".to_string(), + profile: Some(EpiphanyMemoryProfile::RepoArchitecture), + text: Some("shared graph law".to_string()), + ..Default::default() + }, + ); + + assert!(packet.summaries.is_empty()); + assert_eq!(packet.nodes.len(), 2); + assert_eq!(packet.edges.len(), 1); + assert_eq!(packet.anchors.len(), 3); + assert!( + packet + .warnings + .iter() + .any(|warning| warning.contains("descended")) + ); +} + +#[test] +fn memory_graph_context_cut_reports_missing_explicit_ids() { + let snapshot = fixture_snapshot(); + let packet = plan_memory_graph_context_cut( + &snapshot, + &EpiphanyMemoryContextQuery { + id: "query".to_string(), + node_ids: vec!["missing-node".to_string()], + edge_ids: vec!["missing-edge".to_string()], + ..Default::default() + }, + ); + + assert_eq!(packet.missing_node_ids, vec!["missing-node".to_string()]); + assert_eq!(packet.missing_edge_ids, vec!["missing-edge".to_string()]); +} diff --git a/epiphany-core/src/memory_graph/validation.rs b/epiphany-core/src/memory_graph/validation.rs new file mode 100644 index 00000000..d5b6c442 --- /dev/null +++ b/epiphany-core/src/memory_graph/validation.rs @@ -0,0 +1,338 @@ +use super::EpiphanyMemoryDomain; +use super::EpiphanyMemoryEdge; +use super::EpiphanyMemoryGraphSnapshot; +use super::EpiphanyMemoryLifecycle; +use super::EpiphanyMemoryNode; +use super::EpiphanyMemoryProfile; +use super::EpiphanyMemorySummary; +use std::collections::HashSet; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EpiphanyMemoryGraphValidationError { + pub path: String, + pub message: String, +} + +impl EpiphanyMemoryGraphValidationError { + pub(crate) fn new(path: impl Into, message: impl Into) -> Self { + Self { + path: path.into(), + message: message.into(), + } + } +} + +pub fn validate_memory_graph_snapshot( + snapshot: &EpiphanyMemoryGraphSnapshot, +) -> Vec { + let mut errors = Vec::new(); + if snapshot.graph_id.trim().is_empty() { + errors.push(EpiphanyMemoryGraphValidationError::new( + "graph_id", + "graph id is required", + )); + } + + let domain_ids = collect_unique( + snapshot.domains.iter().map(|domain| domain.id.as_str()), + "domains", + &mut errors, + ); + for (index, domain) in snapshot.domains.iter().enumerate() { + validate_domain(domain, index, &mut errors); + } + + let node_ids = collect_unique( + snapshot.nodes.iter().map(|node| node.id.as_str()), + "nodes", + &mut errors, + ); + for (index, node) in snapshot.nodes.iter().enumerate() { + validate_node(node, index, &domain_ids, &mut errors); + } + + let edge_ids = collect_unique( + snapshot.edges.iter().map(|edge| edge.id.as_str()), + "edges", + &mut errors, + ); + for (index, edge) in snapshot.edges.iter().enumerate() { + validate_edge(edge, index, &node_ids, &mut errors); + } + + for (index, summary) in snapshot.summaries.iter().enumerate() { + validate_summary( + summary, + index, + &domain_ids, + &node_ids, + &edge_ids, + &mut errors, + ); + } + + errors +} + +fn validate_domain( + domain: &EpiphanyMemoryDomain, + index: usize, + errors: &mut Vec, +) { + let path = format!("domains[{index}]"); + required( + &domain.id, + format!("{path}.id"), + "domain id is required", + errors, + ); + required( + &domain.title, + format!("{path}.title"), + "domain title is required", + errors, + ); + if !lifecycle_allowed_for_profile(domain.profile, domain.lifecycle) { + errors.push(EpiphanyMemoryGraphValidationError::new( + format!("{path}.lifecycle"), + "domain lifecycle is not legal for profile", + )); + } +} + +fn validate_node( + node: &EpiphanyMemoryNode, + index: usize, + domain_ids: &HashSet, + errors: &mut Vec, +) { + let path = format!("nodes[{index}]"); + required( + &node.id, + format!("{path}.id"), + "node id is required", + errors, + ); + required( + &node.domain_id, + format!("{path}.domain_id"), + "node domain id is required", + errors, + ); + if !domain_ids.contains(&node.domain_id) { + errors.push(EpiphanyMemoryGraphValidationError::new( + format!("{path}.domain_id"), + "node references missing domain", + )); + } + required( + &node.title, + format!("{path}.title"), + "node title is required", + errors, + ); + required( + &node.claim, + format!("{path}.claim"), + "node claim is required", + errors, + ); + required( + &node.action_implication, + format!("{path}.action_implication"), + "node action implication is required", + errors, + ); + if node.question.trim().is_empty() && node.tension.trim().is_empty() { + errors.push(EpiphanyMemoryGraphValidationError::new( + format!("{path}.question"), + "node must preserve a question or tension", + )); + } + if node.anchors.is_empty() + && !node + .source_hashes + .iter() + .any(|hash| hash == "anchor:missing") + { + errors.push(EpiphanyMemoryGraphValidationError::new( + format!("{path}.anchors"), + "node requires anchors or source hash anchor:missing", + )); + } + if !lifecycle_allowed_for_profile(node.profile, node.lifecycle) { + errors.push(EpiphanyMemoryGraphValidationError::new( + format!("{path}.lifecycle"), + "node lifecycle is not legal for profile", + )); + } +} + +fn validate_edge( + edge: &EpiphanyMemoryEdge, + index: usize, + node_ids: &HashSet, + errors: &mut Vec, +) { + let path = format!("edges[{index}]"); + required( + &edge.id, + format!("{path}.id"), + "edge id is required", + errors, + ); + if !node_ids.contains(&edge.source_id) { + errors.push(EpiphanyMemoryGraphValidationError::new( + format!("{path}.source_id"), + "edge references missing source node", + )); + } + if !node_ids.contains(&edge.target_id) { + errors.push(EpiphanyMemoryGraphValidationError::new( + format!("{path}.target_id"), + "edge references missing target node", + )); + } + required( + &edge.claim, + format!("{path}.claim"), + "edge claim is required", + errors, + ); + if !lifecycle_allowed_for_profile(edge.profile, edge.lifecycle) { + errors.push(EpiphanyMemoryGraphValidationError::new( + format!("{path}.lifecycle"), + "edge lifecycle is not legal for profile", + )); + } +} + +fn validate_summary( + summary: &EpiphanyMemorySummary, + index: usize, + domain_ids: &HashSet, + node_ids: &HashSet, + edge_ids: &HashSet, + errors: &mut Vec, +) { + let path = format!("summaries[{index}]"); + required( + &summary.id, + format!("{path}.id"), + "summary id is required", + errors, + ); + if !domain_ids.contains(&summary.domain_id) { + errors.push(EpiphanyMemoryGraphValidationError::new( + format!("{path}.domain_id"), + "summary references missing domain", + )); + } + if summary.covers_node_ids.is_empty() && summary.covers_edge_ids.is_empty() { + errors.push(EpiphanyMemoryGraphValidationError::new( + format!("{path}.covers_node_ids"), + "summary must cover at least one node or edge", + )); + } + for node_id in &summary.covers_node_ids { + if !node_ids.contains(node_id) { + errors.push(EpiphanyMemoryGraphValidationError::new( + format!("{path}.covers_node_ids"), + format!("summary covers missing node {node_id}"), + )); + } + } + for edge_id in &summary.covers_edge_ids { + if !edge_ids.contains(edge_id) { + errors.push(EpiphanyMemoryGraphValidationError::new( + format!("{path}.covers_edge_ids"), + format!("summary covers missing edge {edge_id}"), + )); + } + } + required( + &summary.target, + format!("{path}.target"), + "summary target is required", + errors, + ); + required( + &summary.claim, + format!("{path}.claim"), + "summary claim is required", + errors, + ); + required( + &summary.action_implication, + format!("{path}.action_implication"), + "summary action implication is required", + errors, + ); + if summary.question.trim().is_empty() && summary.tension.trim().is_empty() { + errors.push(EpiphanyMemoryGraphValidationError::new( + format!("{path}.question"), + "summary must preserve a question or tension", + )); + } +} + +pub fn lifecycle_allowed_for_profile( + profile: EpiphanyMemoryProfile, + lifecycle: EpiphanyMemoryLifecycle, +) -> bool { + use EpiphanyMemoryLifecycle::*; + use EpiphanyMemoryProfile::*; + match profile { + RepoArchitecture | RepoDataflow => { + matches!(lifecycle, Observed | Proposed | Accepted | Stale | Retired) + } + RoleSelf | Identity => matches!( + lifecycle, + Proposed | Accepted | Revised | Retired | Crystallized + ), + ShortTerm => matches!( + lifecycle, + Active | Clustered | Distilled | Incubated | Pruned + ), + Incubation => matches!(lifecycle, Active | Deepening | Cooling | Promoted | Retired), + AgencyPressure => matches!(lifecycle, Active | Obligated | Cooling | Answered | Retired), + CandidateIntervention => { + matches!(lifecycle, Queued | Deferred | Spoken | Applied | Retired) + } + Evidence => matches!( + lifecycle, + Observed | Reviewed | Accepted | Contradicted | Superseded | Retired + ), + } +} + +fn collect_unique<'a>( + ids: impl IntoIterator, + path: &str, + errors: &mut Vec, +) -> HashSet { + let mut seen = HashSet::new(); + for id in ids { + if id.trim().is_empty() { + continue; + } + if !seen.insert(id.to_string()) { + errors.push(EpiphanyMemoryGraphValidationError::new( + path, + format!("duplicate id {id}"), + )); + } + } + seen +} + +fn required( + value: &str, + path: String, + message: &str, + errors: &mut Vec, +) { + if value.trim().is_empty() { + errors.push(EpiphanyMemoryGraphValidationError::new(path, message)); + } +} diff --git a/epiphany-state-model/src/lib.rs b/epiphany-state-model/src/lib.rs index 3c84fc94..52eca43a 100644 --- a/epiphany-state-model/src/lib.rs +++ b/epiphany-state-model/src/lib.rs @@ -717,3 +717,392 @@ pub struct EpiphanyObjectiveDraftLanePlan { #[ts(type = "string | null")] pub life: Option, } + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema, TS, Default)] +pub struct EpiphanyMemoryGraphSnapshot { + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(type = "string | null")] + pub schema_version: Option, + pub graph_id: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub domains: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub nodes: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub edges: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub summaries: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(type = "EpiphanyMemoryEmbeddingManifest | null")] + pub embedding_manifest: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(type = "EpiphanyMemoryFreshness | null")] + pub freshness: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub lifecycle_receipts: Vec, +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema, TS, Default)] +pub struct EpiphanyMemoryDomain { + pub id: String, + pub profile: EpiphanyMemoryProfile, + pub title: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(type = "string | null")] + pub description: Option, + #[serde(default)] + pub lifecycle: EpiphanyMemoryLifecycle, +} + +#[derive( + Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq, Hash, JsonSchema, TS, Default, +)] +#[serde(rename_all = "snake_case")] +#[ts(rename_all = "snake_case")] +pub enum EpiphanyMemoryProfile { + #[default] + RepoArchitecture, + RepoDataflow, + RoleSelf, + ShortTerm, + Incubation, + AgencyPressure, + CandidateIntervention, + Identity, + Evidence, +} + +#[derive( + Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq, Hash, JsonSchema, TS, Default, +)] +#[serde(rename_all = "snake_case")] +#[ts(rename_all = "snake_case")] +pub enum EpiphanyMemoryLifecycle { + #[default] + Observed, + Proposed, + Accepted, + Active, + Clustered, + Distilled, + Incubated, + Pruned, + Revised, + Retired, + Crystallized, + Stale, + Deepening, + Cooling, + Promoted, + Queued, + Deferred, + Spoken, + Applied, + Obligated, + Answered, + Reviewed, + Contradicted, + Superseded, +} + +#[derive( + Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq, Hash, JsonSchema, TS, Default, +)] +#[serde(rename_all = "snake_case")] +#[ts(rename_all = "snake_case")] +pub enum EpiphanyMemoryNodeKind { + #[default] + Domain, + Module, + Crate, + Binary, + Schema, + RuntimeContract, + Adapter, + TestSeam, + StateStore, + RoleMemory, + ShortTermThought, + IncubationThread, + AgencyPressure, + CandidateIntervention, + Identity, + Evidence, + Summary, + Other, +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema, TS, Default)] +pub struct EpiphanyMemoryNode { + pub id: String, + pub domain_id: String, + pub profile: EpiphanyMemoryProfile, + pub kind: EpiphanyMemoryNodeKind, + pub title: String, + pub claim: String, + pub question: String, + pub tension: String, + pub action_implication: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub anchors: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub source_hashes: Vec, + #[serde(default)] + pub lifecycle: EpiphanyMemoryLifecycle, + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub salience: u32, + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub confidence: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(type = "string | null")] + pub created_at: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(type = "string | null")] + pub updated_at: Option, +} + +#[derive( + Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq, Hash, JsonSchema, TS, Default, +)] +#[serde(rename_all = "snake_case")] +#[ts(rename_all = "snake_case")] +pub enum EpiphanyMemoryEdgeKind { + #[default] + Owns, + Reads, + Writes, + Derives, + Adapts, + Persists, + Launches, + Verifies, + Supports, + Contradicts, + Distills, + Revises, + Retires, + Grounds, + Triggers, + SpokenAs, + Cools, + ClustersWith, + ResonatesWith, + DependsOn, + Other, +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema, TS, Default)] +pub struct EpiphanyMemoryEdge { + pub id: String, + pub source_id: String, + pub target_id: String, + pub kind: EpiphanyMemoryEdgeKind, + pub profile: EpiphanyMemoryProfile, + pub claim: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub anchors: Vec, + #[serde(default)] + pub lifecycle: EpiphanyMemoryLifecycle, + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub confidence: u32, +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema, TS, Default)] +pub struct EpiphanyMemoryAnchor { + pub id: String, + pub kind: String, + pub target: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(type = "EpiphanyCodeRef | null")] + pub code_ref: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(type = "string | null")] + pub evidence_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(type = "string | null")] + pub source_hash: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(type = "string | null")] + pub note: Option, +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema, TS, Default)] +pub struct EpiphanyMemorySummary { + pub id: String, + pub domain_id: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub covers_node_ids: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub covers_edge_ids: Vec, + pub target: String, + pub claim: String, + pub question: String, + pub tension: String, + pub action_implication: String, + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub anchor_count: u32, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub source_hashes: Vec, + #[serde(default)] + pub freshness: EpiphanyMemoryFreshnessStatus, + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub confidence: u32, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub known_omissions: Vec, +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema, TS, Default)] +pub struct EpiphanyMemoryLifecycleReceipt { + pub id: String, + pub operation: String, + pub status: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub node_ids: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub edge_ids: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(type = "string | null")] + pub reason: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(type = "string | null")] + pub created_at: Option, +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema, TS, Default)] +pub struct EpiphanyMemoryEmbeddingManifest { + pub id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(type = "string | null")] + pub collection_name: Option, + pub embedding_model: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(type = "number | null")] + pub vector_dimensions: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub indexed_document_ids: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub stale_document_ids: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub source_hashes: Vec, +} + +#[derive( + Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq, Hash, JsonSchema, TS, Default, +)] +#[serde(rename_all = "snake_case")] +#[ts(rename_all = "snake_case")] +pub enum EpiphanyMemoryFreshnessStatus { + #[default] + Missing, + Ready, + Stale, + Indexing, + Unavailable, +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema, TS, Default)] +pub struct EpiphanyMemoryFreshness { + pub status: EpiphanyMemoryFreshnessStatus, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub stale_node_ids: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub stale_edge_ids: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub stale_summary_ids: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub dirty_source_hashes: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(type = "string | null")] + pub note: Option, +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema, TS, Default)] +pub struct EpiphanyMemoryContextQuery { + pub id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(type = "EpiphanyMemoryProfile | null")] + pub profile: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub domain_ids: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub node_ids: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub edge_ids: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(type = "string | null")] + pub text: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(type = "number | null")] + pub budget: Option, +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema, TS, Default)] +pub struct EpiphanyMemoryContextPacket { + pub id: String, + pub query_id: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub nodes: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub edges: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub summaries: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub anchors: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub warnings: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub missing_node_ids: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub missing_edge_ids: Vec, +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema, TS, Default)] +pub struct EpiphanyMemoryPatchCandidate { + pub id: String, + pub profile: EpiphanyMemoryProfile, + pub status: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub proposed_nodes: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub proposed_edges: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(type = "Array")] + pub reasons: Vec, +} + +fn is_zero_u32(value: &u32) -> bool { + *value == 0 +} diff --git a/notes/fresh-workspace-handoff.md b/notes/fresh-workspace-handoff.md index ed0097d0..f95cd622 100644 --- a/notes/fresh-workspace-handoff.md +++ b/notes/fresh-workspace-handoff.md @@ -39,7 +39,7 @@ remembers doctrine; the branch remembers the blade. - `epiphany-openai-runtime` worker result ingress now parses assistant output directly into typed role/reorient ingress structs selected from the durable launch document, stores nested `statePatch` and `selfPatch` as typed MessagePack in `EpiphanyRuntimeRoleWorkerResult`, and no longer passes worker result data through generic `serde_json::Value`. JSON survives there only as the OpenAI text-output edge before typed deserialization. - Agent self-memory review/application now has typed `AgentSelfPatch` core APIs. JSON-value selfPatch functions are compatibility wrappers for CLI/artifact ingress; native callers should parse once at the edge and then call `review_agent_self_patch_document` / `apply_agent_self_patch_document`. Role-result self-persistence no longer exports a JSON-value reviewer; the live API is `review_role_self_patch_document`. - `epiphany-openai-codex-spine` no longer uses Codex's Responses client, model-provider layer, `codex-api`, `codex-model-provider`, `codex-model-provider-info`, or `codex-protocol` for outbound model requests. It reaches `codex-login` only through `epiphany-openai-auth-spine` so credential loading/refresh stays Codex-compatible while request/status/event/runtime surfaces stay typed Epiphany documents. -- `notes/epiphany-memory-graph-unified-plan.md` is the current memory architecture direction drawn from `Fractal Domains And The Cache That Bites` plus the VoidBot memory scar tissue. The repo semantic architecture/dataflow graph and agent memory are one `EpiphanyMemoryGraph` substrate with repo, role, short-term, incubation, agency, candidate-intervention, and evidence profiles. Qdrant is a contribution cache for typed graph documents, not truth. The first implementable ticket is shared memory graph documents plus pure `epiphany-core::memory_graph` id/validation/freshness/context-cut tests; profile-specific scanners, sleep consolidation, CultCache persistence, Qdrant, CultNet advertisement, and prompt integration come after the shared skeleton exists. `notes/repo-fractal-dataflow-cache-plan.md` and `notes/agent-memory-fractal-cache-plan.md` are profile plans, not rival organs. +- `notes/epiphany-memory-graph-unified-plan.md` is the current memory architecture direction drawn from `Fractal Domains And The Cache That Bites` plus the VoidBot memory scar tissue. The repo semantic architecture/dataflow graph and agent memory are one `EpiphanyMemoryGraph` substrate with repo, role, short-term, incubation, agency, candidate-intervention, and evidence profiles. The first native skeleton is landed: `epiphany-state-model` owns shared graph document types, `epiphany-core::memory_graph` owns stable ids, validation, freshness, context cuts, and CultCache persistence, and `epiphany-memory-graph` can status/validate/context/smoke the store without Qdrant. Qdrant is a contribution cache for typed graph documents, not truth. `notes/repo-fractal-dataflow-cache-plan.md` and `notes/agent-memory-fractal-cache-plan.md` are profile plans, not rival organs. - Durable Epiphany state types now live in root crate `epiphany-state-model`. `codex-protocol` re-exports them only so old rollout/API callers keep compiling; do not add new durable `Epiphany*` state structs under `vendor/codex`. `epiphany-core` imports the native model directly and no longer depends on `codex-protocol`. Pressure derives from native `EpiphanyTokenUsageSnapshot`; Codex token telemetry is mapped at the bridge. Codex rollout/event reconstruction now lives in `codex-core::epiphany_rollout`, where the Codex `RolloutItem` / `EventMsg` types belong. Exact retrieval path matching is native to `epiphany-core`, so `epiphany-core` has no direct vendored Codex crate dependency. - State-update contract proof and job launch/interrupt DTO ownership now live in native `epiphany-core`. The old giant `codex_thread.rs` Epiphany state-update test fixture was cut, and `epiphany-codex-bridge` no longer imports Epiphany job launch request types from `codex_core`; vendored `CodexThread` may call the native mutation/launch law for compatibility, but it must not regain ownership of state-update semantics, heartbeat launch-plan policy, or Epiphany DTOs. - MCP remains JSON because MCP is a JSON protocol. The latest processor cut only moved MCP refresh/OAuth/status/resource/tool-call route handling into `codex_message_processor/mcp_routes.rs`, reducing the root processor to about 8,977 lines; it did not claim MCP payloads are CultNet or make MCP an Epiphany-owned runtime contract. @@ -96,7 +96,7 @@ remembers doctrine; the branch remembers the blade. - The heartbeat initiative pass is landed as a bounded tool seam. `state/agent-heartbeats.msgpack` tracks Self, Face, Imagination, Eyes, Body, Hands, Soul, and Life as Ghostlight-style initiative participants with arena, participant kind, speed, next-ready time, reaction bias, interrupt threshold, load, status, constraints, personality cooldown, mood cooldown, effective cooldown, adaptive pacing, history, and pending turns through `epiphany-core::EpiphanyHeartbeatStateEntry` and the native `epiphany-heartbeat-store` binary. Role dossiers set baseline timing; appraisal mood/anxiety bends it through urgency, arousal, thought pressure, guardedness, and reaction intensity, so Hands-like work pressure and high-need anxious lanes recover sooner without being heartbeaten while a prior turn is running. `epiphany-heartbeat-store pump` computes pressure from external urgency, mood/anxiety, reaction intensity, thought pressure, and current pending load, then chooses tempo plus target concurrency: calm can launch zero and sleep slow; alarm can fill most active lanes. Idle turns are for rumination: light thought shuffling, role-quality attention, and candidate selfPatch pressure. Sleep/dream cycle passes are the intended distillation window for durable self-memory and doctrine. JSON heartbeat state and Python wrapper state are gone; general CultCache schema sync, polyglot domain loading, and debug display tools belong in CultLib. This is a callable scheduler seam, not yet an always-on daemon; a heart valve, not a whole circulatory god. - The first Ghostlight-derived timing slice is landed in Epiphany, but Ghostlight is reference lineage rather than a sibling runtime to preserve. `epiphany-heartbeat-store init --profile ghostlight-scene --scene-id --scene-participant ` creates a typed CultCache MessagePack scene heartbeat store whose participants are `arena=scene`, `participantKind=character`; `tick` emits `ghostlight.initiative_schedule.v0` receipts with `scene_turn` actions and local-affordance basis. The generic Epiphany maintenance lanes are not auto-patched into scene stores. - The first Void-derived routine slice is landed in Epiphany, but VoidBot is reference lineage rather than a runtime dependency. `epiphany-heartbeat-store routine --store .\state\agent-heartbeats.msgpack --artifact-dir .\.epiphany-heartbeats --agent-store .\state\agents.msgpack` reads typed role dossiers, computes bounded memory resonance, maintains incubation themes, runs analytic and associative cognition lanes, writes bridge syntheses/saturation/tension, projects the active thought cluster through each role's Ghostlight-shaped personality vectors, derives participant-local reactions, applies personality/mood timing, advances the sleep/dream cycle, updates the typed heartbeat store, and emits an auditable `epiphany.void_routine.v0` receipt. The routine now also carries anti-loop physiology imported from Void's calmer brain: `noveltyToSelf`, `noveltyToRoom`, source coverage, saturation pressure, refractory cooling, and explicit permission to let a live unsaturated thought deepen without forcing novelty theater every pass. This mutates only heartbeat physiology fields: project truth and role memory mutation remain on their reviewed surfaces. -- The next memory foundation must not build separate engines for repo graph and agent mind. Heartbeat cognition, role self-memory, repo architecture/dataflow, evidence scars, incubation, agency pressure, and context packets should converge through `EpiphanyMemoryGraph` profiles so one typed graph owns anchors, lifecycle, summaries, freshness, context cuts, and Qdrant embedding manifests. +- The next memory foundation must not build separate engines for repo graph and agent mind. Heartbeat cognition, role self-memory, repo architecture/dataflow, evidence scars, incubation, agency pressure, and context packets should converge through `EpiphanyMemoryGraph` profiles so one typed graph owns anchors, lifecycle, summaries, freshness, context cuts, and Qdrant embedding manifests. The next clean implementation slice is a repo profile producer that maps existing accepted architecture/dataflow graph state into memory graph fixtures/profile documents before any scanner, sleep runner, Qdrant, CultNet, or prompt integration. - Manual Codex-run rumination still has an explicit aftercare rule until Epiphany owns the full sleep consolidator. In the intended Epiphany cycle, sub-agents ruminate when idle and distill when they sleep; in this supervising Codex thread, a heartbeat/routine vigil is only physiology until a separate closing pass reviews the receipts and decides whether map, handoff, evidence, or role self-memory should change. - The Face public-surface pass is landed as a bounded lane. Face's role dossier lives in `state/agents.msgpack`; `epiphany_specialists.toml` gives it a VoidBot-heartbeat-derived prompt stripped of moderation authority; `state/face-discord.toml` and the native `epiphany-face-discord` binary enforce that Face may interact only through #aquarium. Missing channel id or token writes candidate chat artifacts instead of posting elsewhere. If a Face has a persona name/avatar, `post` uses the shared Discord webhook pipe so each Epiphany can speak with its own nickname and avatar without a separate bot identity. - The Void memory bridge is now native enough to keep the useful organs attached. `state/void-memory.toml` and `epiphany-void-memory` can check Void's Docker Postgres state spine, query Qdrant Discord-history and source/lore collections with the configured Ollama embedding model, and fetch raw archive context windows from Void's file-backed Discord archive. Native `epiphany-mvp-status` includes a `voidMemory` block so Aquarium/backend clients can inspect whether the mouth and memory are actually wired. diff --git a/state/ledgers.msgpack b/state/ledgers.msgpack index 7b4f00bf..1d00db4c 100644 Binary files a/state/ledgers.msgpack and b/state/ledgers.msgpack differ diff --git a/state/map.yaml b/state/map.yaml index 2f07b03c..c49f7476 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -75,7 +75,7 @@ active_subgoals: - The May 2026 teardown/rebuild migration in `notes/epiphany-ideal-architecture-rebuild-plan.md` is closed unless a concrete regression reopens it. Preserve the new foundation: Epiphany view/coordinator policy lives in `epiphany-core`, runtime-spine owns job/result lifecycle identity through `runtime_links`, duplicate acceptance is receipt-keyed, public findings stay typed/sealed, view expansion requires a core contract, and app-server remains routing/adaptation rather than a second policy throne. - Treat Phase 5 update/distill/propose/promote/read/notification as complete enough unless a concrete regression appears. - Preserve read-only boundaries for retrieve, distill, propose, freshness, context, graphQuery, roleResult, reorientResult, and the canonical view lenses for scene, jobs, roles, planning, pressure, reorient, crrc, and coordinator. - - Use `notes/epiphany-memory-graph-unified-plan.md` as the source-grounded memory architecture direction. The repo fractal dataflow graph and agent memory are one `EpiphanyMemoryGraph` substrate with repo, role, short-term, incubation, agency, candidate-intervention, and evidence profiles. Qdrant stores rebuildable semantic embeddings for typed graph documents; it is never canonical memory. The first implementable ticket is shared memory graph documents plus pure validation/context-cut tests before repo scanners, sleep runners, CultCache store wiring, Qdrant cache plumbing, CultNet advertisement, or prompt integration. + - Use `notes/epiphany-memory-graph-unified-plan.md` as the source-grounded memory architecture direction. The repo fractal dataflow graph and agent memory are one `EpiphanyMemoryGraph` substrate with repo, role, short-term, incubation, agency, candidate-intervention, and evidence profiles. Qdrant stores rebuildable semantic embeddings for typed graph documents; it is never canonical memory. The first live skeleton now exists in `epiphany-state-model` plus `epiphany-core::memory_graph`: typed graph documents, stable ids, profile/lifecycle validation, freshness propagation, context cuts, a CultCache store seam, and the native `epiphany-memory-graph` status/validate/context/smoke CLI. Do not add repo scanners, sleep runners, Qdrant cache plumbing, CultNet advertisement, or prompt integration before routing them through this shared substrate. - Keep the landed Phase 6 coordinator, limited safe-boundary CRRC automation, and pre-compaction checkpoint intervention ready for the next bridge-equipped dogfood run. - Preserve the Ghostlight-derived heartbeat initiative seam: every lane, including coordinator/Self and Face, has a role dossier and competes for heartbeat slots through readiness/reaction timing; idle winners ruminate instead of inventing project work, and sleep windows distill the accumulated thought pressure into reviewed self-memory or doctrine. - Treat Ghostlight as reference lineage, not a sibling runtime to preserve. The useful organ is Epiphany-native character-loop protocol: any role can project its typed dossier plus visible stimulus into a local turn packet, with Face as the first public-surface actor. @@ -114,7 +114,7 @@ accepted_design: - `thread/epiphany/retrieve`, `thread/epiphany/distill`, `thread/epiphany/propose`, `thread/epiphany/freshness`, `thread/epiphany/context`, `thread/epiphany/graphQuery`, and `thread/epiphany/view` stay read-only. - `thread/epiphany/index` writes the retrieval catalog, not durable Epiphany understanding; catalog is auspex output, not doctrine. - Qdrant is the preferred persistent semantic backend; BM25 remains the bootstrap/fallback/control path. - - `notes/epiphany-memory-graph-unified-plan.md` applies the mapped-fractal-domain/cache lesson to Epiphany and corrects the organ boundary: repo architecture/dataflow and agent memory are profiles over one `EpiphanyMemoryGraph`, not separate graph engines. `notes/repo-fractal-dataflow-cache-plan.md` and `notes/agent-memory-fractal-cache-plan.md` are profile plans beneath that substrate. If deleting Qdrant deletes understanding, the design is wrong; the fallback must be slower but still correct typed graph traversal. Start with shared typed documents and pure validation/context-cut tests, not vector plumbing or profile-specific producers. + - `notes/epiphany-memory-graph-unified-plan.md` applies the mapped-fractal-domain/cache lesson to Epiphany and corrects the organ boundary: repo architecture/dataflow and agent memory are profiles over one `EpiphanyMemoryGraph`, not separate graph engines. The first native skeleton is landed: `epiphany-state-model` owns the public document vocabulary, `epiphany-core::memory_graph` owns stable ids/validation/freshness/context cuts/CultCache persistence, and `epiphany-memory-graph` exposes status/validate/context/smoke without Qdrant. `notes/repo-fractal-dataflow-cache-plan.md` and `notes/agent-memory-fractal-cache-plan.md` are profile plans beneath that substrate. If deleting Qdrant deletes understanding, the design is wrong; the fallback must be slower but still correct typed graph traversal. - `epiphany-core` owns prompt rendering, rollout replay helpers, retrieval/indexing, distillation, proposal, promotion policy, scene projection, freshness derivation, pressure derivation, job/progress view derivation, planning view derivation, reorientation resume/regather verdict policy, pure CRRC recommendation policy, fixed-lane coordinator decision policy, role-board projection policy, and role-result interpretation/self-persistence review policy where possible. - Vendored Codex keeps the thin typed host seam: protocol types, thread bridges, app-server protocol, request routing, watcher lifecycle, notifications, and rollout integration. Epiphany projection and coordinator policy should keep moving into `epiphany-core` / `epiphany-codex-bridge` instead of living in `codex_message_processor.rs`. - The current algorithmic map is the source-grounded description of live control flow. @@ -291,12 +291,10 @@ rejected_paths: current_status: summary: The Codex starvation pass has five typing receipts, fifty-six processor/foundation cuts, a real physical extraction in `epiphany-codex-bridge`, a root `epiphany-state-model` crate for durable Epiphany state, a source-grounded Codex auth spine inventory, the native `epiphany-openai-adapter` document boundary, a buildable Codex auth/model wrapper in `epiphany-openai-codex-spine`, the first native runtime caller in `epiphany-openai-runtime`, native heartbeat launch planning/job opening in `epiphany-core`, native state-update contract/validation/application in `epiphany-core`, native exact retrieval path matching in `epiphany-core`, and the route-facing mutation/view/coordinator projection services in `epiphany-codex-bridge`. `codex-protocol` no longer defines the durable `Epiphany*` state structs; it re-exports the native model for legacy rollout/API compatibility while `epiphany-core` imports the native model directly. `epiphany-core` now has no direct vendored Codex dependency: pressure derivation accepts native `EpiphanyTokenUsageSnapshot`, Codex rollout replay moved to `codex-core::epiphany_rollout`, and exact retrieval search uses Epiphany-owned workspace walking/path scoring instead of `codex-file-search`. The bridge owns Codex-facing Epiphany adapters for scene, jobs, context/planning/graph query, retrieval/indexing projection, result projection, runtime-spine result adaptation, launch/prompt doctrine, mutation document mechanics, mutation service update/promote/role-accept/reorient-accept/role-launch/reorient-launch/job-launch/job-interrupt application, live-state projection, pressure/pre-compaction rendering, freshness/reorientation projection, CRRC/coordinator/role-board projection, watcher invalidation adaptation, full `thread/epiphany/view` response construction, freshness/context/graph-query response construction, role/reorient result response construction, distill/propose patch-response construction, coordinator status/view construction, and acceptance/evidence signal helpers. Vendored app-server no longer retains root Epiphany modules; `codex_message_processor.rs` still contains route dispatch, JSON-RPC response shaping, watcher registration/snapshotting calls, and notification emission, but its root no longer imports the evacuated Epiphany policy/protocol registry or owns the watcher invalidation manager. Codex product surfaces for apps, skills, plugins, marketplace, and external agent config import are cut from the live app-server/protocol path: list routes return empty lists only where compatibility shells still exist, mutation/detail routes report disabled where they survive, `externalAgentConfig/*` routes and import-completed notification are gone, helper/import modules and stale implementation-shape tests were deleted, runtime feature enablement no longer supports `apps` or `plugins`, config writes no longer emit plugin-toggle analytics, plugin/skill cache invalidation on config writes is gone, plugin startup warmups and app-list refreshes were removed, and app-server no longer directly depends on `codex-plugin`, `codex-chatgpt`, or `codex-core-plugins`. `Config::to_mcp_config` now keeps user-declared MCP servers but no longer accepts `PluginsManager`, folds plugin MCP servers into the runtime, enables Codex apps, enables skill MCP dependency install, or advertises plugin capability summaries; `TurnContext::apps_enabled()` now returns false so model turns cannot awaken Codex Apps through ChatGPT auth or feature flags; session startup and per-turn context no longer load skills, turns receive an empty `SkillLoadOutcome`, internal `Op::ListSkills` is gone from `codex-protocol` / `codex-core` / TUI command routing, the Codex skill watcher has been collapsed to a no-op marker with no file watcher, skill-root registration, cache-clear task, or `SkillsUpdateAvailable` session listener, model turns no longer run skill mention collection, skill env-var prompts, skill MCP dependency installation, skill instruction injection, or implicit skill invocation analytics from shell tools, and the turn prompt/router path no longer loads plugin/app capability summaries, injects available-skill/available-plugin/plugin instruction fragments, parses plugin/app mentions, records app/plugin invocation telemetry, or filters MCP tools through Codex app connectors. Skill mutation/list/notification is gone from the app-server edge too: `skills/list`, `skills/config/write`, `skills/changed`, `Op::ListSkills`, `EventMsg::ListSkillsResponse`, `EventMsg::SkillsUpdateAvailable`, the app-server handlers, TUI slash/startup/list/manage-skill surfaces, generated schema entries, core skill-list suite, and fake live-reload/developer-message tests were deleted. Skill mention plumbing is also gone from the live TUI/protocol edge: no `UserInput::Skill` variant, app-server schema branch, TUI skill popup, `skills_all` cache, composer `set_skill_mentions`, skill mention insertion, `skill://` / `SKILL.md` history decoding, or stale skill popup snapshot remains. The `codex-core-skills` crate, core `skills` re-export shim, no-op `skills_watcher`, session `SkillsManager` service, and per-turn `TurnSkillsContext` are deleted; core/protocol no longer keep legacy `` or `` prompt compatibility, memory usage classification, snapshot normalization, or cleanup paths. Remaining product-shaped skill residue is also cut from memory templates, app-server docs/tests/warnings, TUI tooltips/placeholders/warnings/comments, app-server v1 docs, agent-tool schema text, stale `UserInput::Skill` test arms, and obsolete feature flags/schema entries. App-instruction prompt compatibility is cut too: no `APPS_INSTRUCTIONS` protocol constants, no `` snapshot canonicalization, and MCP fixture tests now use neutral `demo_mcp` / `calendar_mcp` names instead of `codex_apps` / `/api/codex/apps` branding. The TUI app connector product path is also deleted: no `/apps` slash command, app-link view, connector AppEvent variants, app connector cache/popup/refresh loop, app mention insertion, `app://` mention codec support, stale app-popup tests, or TUI browser-opening shim remains. The `app/list` route and `app/list/updated` notification are gone from app-server protocol, generated schemas, app-server dispatch, test helpers, and docs; `AppInfo`/app-list metadata schema structs are deleted; the orphaned `codex-connectors` crate and `core::connectors` shim are deleted from the workspace; Codex app analytics no longer tracks `codex_app_mentioned` / `codex_app_used` facts, events, reducer arms, client APIs, or dedupe cache state; `[apps.*]`, `Feature::Apps`, `include_apps_instructions`, app config/read schemas, app managed requirements, app prompt tests, and app file-MCP tests are gone; `Feature::Plugins`, `Feature::RemotePlugin`, `plugin://` structured mentions, plugin instruction tags, and stale plugin popup snapshots are gone too; `[plugins]` / `[marketplaces]`, `PluginConfig`, `MarketplaceConfig`, marketplace edit helpers, stale plugin/external-agent README route claims, and generated schema entries are removed from config; and the bundled `codex-skills` sample-skill crate plus embedded imagegen/openai-docs/plugin-creator/skill-creator/skill-installer payloads are deleted from the workspace. The vendored `vendor/codex/sdk` subtree is deleted as non-spine app-server client/release scaffolding, including stale generated Python SDK route types for removed product surfaces; the root pnpm workspace no longer lists `sdk/typescript`. Live MCP tool approval is named `McpToolApproval`, not `AppToolApproval`, and remains under MCP server config; live structured user-input mentions are MCP-shaped only. `codex_message_processor.rs` is about 1,009 lines after route/lifecycle/projection/resume-helper/config-helper extraction; root `message_processor.rs` is about 1,292 lines after the external-agent/config-product cuts; `epiphany_read_routes.rs` is about 579 lines after the full view and read-response extractions; `epiphany_mutation_routes.rs` is about 694 lines after route-load/notification helper cuts and product-surface starvation; the old vendored `epiphany_state_helpers.rs` rollout-state loader is deleted and its remaining function lives in `epiphany-codex-bridge::state`; and `codex_thread.rs` is about 1,425 lines after state-update law moved native. `epiphany-openai-adapter` defines CultCache `DatabaseEntry` request/event/receipt/auth-status documents with no Codex app-server or generic JSON dependency. `epiphany-openai-codex-spine` owns the native OpenAI model transport path: it exposes `default_codex_home` and `auth_manager`, projects `epiphany-openai-auth-spine` `AuthManager` / `CodexAuth` into typed `EpiphanyOpenAiAdapterStatus`, builds a local serializable OpenAI Responses request body instead of Codex protocol request/item/config types, opens HTTP Responses streams through `codex-client`, parses Responses SSE frames into typed stream events and receipts, and no longer directly or transitively depends on Codex's Responses client, model-provider stack, `codex-protocol`, `codex-api`, `codex-model-provider`, or `codex-model-provider-info`; it intentionally reaches retained vendored `codex-login` only through `epiphany-openai-auth-spine`. Auth identity, credential storage, and ChatGPT token refresh remain owned by the Codex-compatible auth organ; external bearer command auth and agent identity remain retained Codex auth capabilities to expose only if a typed Epiphany surface needs them. `epiphany-openai-runtime` no longer depends directly on `codex-login`; it writes OpenAI status/request/stream-event/receipt records plus runtime sessions/jobs/results/events into `epiphany-core::runtime_spine_cache`, can emit the assistant text artifact for legacy result consumers, is now the default executor for startup-only repo birth specialists, and now has `run-worker` to load a durable `EpiphanyRuntimeWorkerLaunchRequest`, call the typed OpenAI runtime through the spine, persist typed `EpiphanyRuntimeRoleWorkerResult` / `EpiphanyRuntimeReorientWorkerResult` documents, and complete the original heartbeat/specialist runtime job without Codex worker execution; `codex exec` survives only as explicit fallback in repo-birth startup. `epiphany-mvp-coordinator` now invokes `epiphany-openai-runtime run-worker` for launched runtime job ids and no longer creates a second coordinator-owned shadow runtime job/result as a compensator. `epiphany-core::plan_runtime_spine_heartbeat_launch` validates launch contracts/conflicts and projects durable job bindings/runtime links; `epiphany-core::open_runtime_spine_heartbeat_job` owns the initialize-runtime/ensure-session/create-job sequence and now persists `EpiphanyRuntimeWorkerLaunchRequest` CultCache records keyed by runtime job id, with indexed kind/contract fields plus MessagePack-encoded typed launch documents instead of discarding work intent after queue receipt creation; `roleResult` and `reorientResult` now require typed worker result documents; generic job summaries are lifecycle receipts, not reviewable findings; `epiphany-core::EpiphanyStateUpdate`, `epiphany_state_update_validation_errors`, and `apply_epiphany_state_update` own state-update document shape and mutation law while `codex-core` calls them for compatibility. Vendored `CodexThread::epiphany_launch_job` is now mostly revision check, persistence validation, launch request handoff, and rollout/session writeback for the compatibility route. Vendored `codex-core` no longer has `epiphany_distillation`, `epiphany_promotion`, `epiphany_proposal`, or `epiphany_retrieval` husk modules; it re-exports native surfaces from `epiphany-core` directly, and now owns only the real Codex rollout adapter because it is a host seam. `epiphany-runtime-spine` advertises OpenAI adapter status/model-request/stream-event/receipt CultNet schemas and mutation contracts, so the native caller now has a real storage route instead of only JSON-RPC. MCP itself is correctly JSON at its protocol edge; the target is typed Epiphany intent/result/receipt wrappers around MCP, not replacing MCP JSON. The current OpenAI request path no longer pulls `codex-api`; `codex-login` remains deliberately sealed behind `epiphany-openai-auth-spine`, and the next spine decision is another Epiphany-in-vendor evacuation surface unless a concrete typed auth-status/runtime need requires exposing external bearer or agent-identity auth. latest_cut: > - `notes/epiphany-memory-graph-unified-plan.md` now corrects the repo - fractal graph and agent memory roadmaps into one shared - `EpiphanyMemoryGraph` substrate with domain profiles for repo - architecture/dataflow, role self-memory, short-term thought, incubation, - agency pressure, candidate interventions, and evidence. The first ticket - is shared typed memory graph documents plus pure validation/context-cut - tests, with scanners, sleep consolidation, CultCache store wiring, Qdrant - cache, CultNet advertisement, and prompt integration staged afterward. - next_action: Continue the Codex starvation campaign unless the user directs the next slice into memory graph work. If memory graph work starts, first add shared `EpiphanyMemoryGraph` typed document structs and pure `epiphany-core::memory_graph` id/validation/freshness/context-cut tests before adding repo profile producers, agent sleep consolidation, or Qdrant. + The first `EpiphanyMemoryGraph` skeleton is implemented. Public typed + graph documents live in `epiphany-state-model`; `epiphany-core::memory_graph` + owns stable ids, profile/lifecycle legality, snapshot validation, + freshness propagation, summary-versus-child context cuts, and a validated + CultCache store entry; the native `epiphany-memory-graph` binary can + status/validate/query/smoke the store without Qdrant. + next_action: Continue the memory graph build unless a more urgent Codex starvation regression appears. The next clean slice is a repo profile producer that maps existing accepted architecture/dataflow graph state into `EpiphanyMemoryGraph` fixture/profile documents before any scanner, Qdrant, sleep consolidation, CultNet advertisement, or prompt integration.