From e64eee951f3cd959a0866afa29f3cc71a14666c7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 23 Apr 2026 22:01:17 +0100 Subject: [PATCH] Initial EpiphanyAgent scaffold and planning docs --- .gitignore | 2 + .gitmodules | 3 + AGENTS.md | 61 + README.md | 50 + eval-plan.md | 37 + notes/architecture-rationale.md | 134 ++ notes/codex-epiphany-mode-plan.md | 173 +++ notes/codex-repository-algorithmic-map.md | 490 +++++++ notes/epiphany-core-harness-surfaces.md | 1596 +++++++++++++++++++++ notes/fresh-workspace-handoff.md | 91 ++ protocol/controller-actions.md | 119 ++ state/branches.json | 15 + state/evidence.jsonl | 26 + state/map.yaml | 90 ++ state/scratch.md | 26 + tools/epiphany_state.py | 179 +++ vendor/codex | 1 + 17 files changed, 3093 insertions(+) create mode 100644 .gitignore create mode 100644 .gitmodules create mode 100644 AGENTS.md create mode 100644 README.md create mode 100644 eval-plan.md create mode 100644 notes/architecture-rationale.md create mode 100644 notes/codex-epiphany-mode-plan.md create mode 100644 notes/codex-repository-algorithmic-map.md create mode 100644 notes/epiphany-core-harness-surfaces.md create mode 100644 notes/fresh-workspace-handoff.md create mode 100644 protocol/controller-actions.md create mode 100644 state/branches.json create mode 100644 state/evidence.jsonl create mode 100644 state/map.yaml create mode 100644 state/scratch.md create mode 100644 tools/epiphany_state.py create mode 160000 vendor/codex diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..7a60b85e --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +*.pyc diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..bcae2ac2 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "vendor/codex"] + path = vendor/codex + url = https://github.com/openai/codex.git diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..de145ccb --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,61 @@ +# EpiphanyAgent Instructions + +## Project Purpose + +This repo explores an Epiphany mode for AI coding agents: external typed state, explicit mental maps, bounded scratch work, verifier evidence, and aggressive anti-churn discipline. + +The motivating failure mode is that an agent can make many plausible local edits after global coherence has already failed. The project goal is to test whether explicit map/scratch/evidence channels reduce that drift. + +## Canonical State + +- Treat `state/map.yaml` as the canonical project map. +- Treat `state/scratch.md` as disposable working memory for one bounded subgoal. +- Treat `state/evidence.jsonl` as the durable log of what was learned, verified, rejected, or accepted. +- Treat `notes/codex-epiphany-mode-plan.md` as the current patch plan for adding Epiphany mode to the vendored Codex repo. +- Update `state/map.yaml` when project understanding changes. +- Append evidence after meaningful research, implementation, verification, or rejected paths. + +## Current Recommendation + +Do not start with full protocol enum surgery. + +Start by implementing a lightweight preset-backed Epiphany mode in the vendored Codex repo: + +- add an `epiphany.md` collaboration-mode template +- add an `Epiphany` collaboration preset backed by `ModeKind::Default` +- patch the TUI to display the active preset name so the mode is visible to users +- only add `ModeKind::Epiphany` later if the preset-backed experiment proves useful + +## Important Paths + +- Project root: `E:\Projects\EpiphanyAgent` +- Vendored Codex repo: `E:\Projects\EpiphanyAgent\vendor\codex` +- Patch plan: `E:\Projects\EpiphanyAgent\notes\codex-epiphany-mode-plan.md` +- Handoff summary: `E:\Projects\EpiphanyAgent\notes\fresh-workspace-handoff.md` +- State CLI: `E:\Projects\EpiphanyAgent\tools\epiphany_state.py` + +## Useful Commands + +Use the bundled Python runtime if `python` is not on PATH: + +```powershell +& 'C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe' '.\tools\epiphany_state.py' status +& 'C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe' '.\tools\epiphany_state.py' add-evidence --type research --status ok --note '...' +``` + +Useful Codex repo searches: + +```powershell +rg -n "pub enum ModeKind|TUI_VISIBLE_COLLABORATION_MODES" .\vendor\codex\codex-rs\protocol\src\config_types.rs +rg -n "builtin_collaboration_mode_presets|fn plan_preset|fn default_preset" .\vendor\codex\codex-rs\models-manager\src\collaboration_mode_presets.rs +rg -n "collaboration_mode_label|collaboration_mode_indicator|set_collaboration_mask" .\vendor\codex\codex-rs\tui\src\chatwidget.rs +``` + +## Operating Discipline + +- Before substantial edits, restate the current mechanism and intended change. +- Prefer one clear hypothesis per iteration. +- Verify with checks that reflect the real goal, not just proxy success. +- Revert or discard changes that do not clearly improve the target. +- If the diff grows while understanding shrinks, stop implementation and switch to diagnosis. +- Keep maps and prose together; do not replace useful maps with prose-only explanations. diff --git a/README.md b/README.md new file mode 100644 index 00000000..cee914d0 --- /dev/null +++ b/README.md @@ -0,0 +1,50 @@ +# EpiphanyAgent + +A cheap prototype for testing whether typed external state makes a strong model less likely to build an impressive pile of locally plausible nonsense. + +## Premise + +Most coding agents are one token stream pretending to be a planner, scratchpad, architect, and release engineer at the same time. This workspace separates those roles outside the model first. + +The goal is not to prove a new frontier architecture in a weekend. The goal is to find out whether explicit state channels and bounded controller actions reduce drift enough to matter. + +## Core Artifacts + +- `AGENTS.md`: project-specific instructions for future Codex sessions +- `state/map.yaml`: canonical, slow-changing model of the task or system +- `state/scratch.md`: temporary reasoning for one subgoal +- `state/branches.json`: branch hypotheses and branch outcomes +- `state/evidence.jsonl`: verifier outputs, acceptance reasons, and reversions +- `protocol/controller-actions.md`: allowed actions and write rules +- `notes/fresh-workspace-handoff.md`: concise summary for a fresh workspace +- `notes/codex-epiphany-mode-plan.md`: concrete patch plan for vendored Codex +- `notes/architecture-rationale.md`: why the map/scratch/evidence architecture exists +- `tools/epiphany_state.py`: tiny CLI for inspecting and updating branch/evidence state +- `eval-plan.md`: how to compare this protocol against plain prompting + +## Basic Loop + +1. Observe the task, repo state, and current verifier outputs. +2. Update `state/map.yaml` only when understanding stabilizes. +3. Use `state/scratch.md` for one bounded subgoal at a time. +4. Propose one change or one answer. +5. Verify against task success, map consistency, and simplicity. +6. Record evidence. +7. Commit, compare branches, or backtrack. + +## Why This Exists + +Pinned notes inside one giant prompt are better than nothing, but they are still one token soup. This workspace treats map, scratch, and output as different artifacts with different write rules. + +That may still turn out to be elaborate prompt theater. Fine. At least it will be falsifiable prompt theater. + +## Tiny CLI + +Examples: + +```powershell +python .\tools\epiphany_state.py status +python .\tools\epiphany_state.py add-evidence --type verify --status ok --note "Map and patch still agree" +python .\tools\epiphany_state.py add-branch --id b1 --hypothesis "Smaller verifier beats elaborate verifier" +python .\tools\epiphany_state.py close-branch --id b1 --status rejected --note "More ceremony, no better signal" +``` diff --git a/eval-plan.md b/eval-plan.md new file mode 100644 index 00000000..dd8fcf33 --- /dev/null +++ b/eval-plan.md @@ -0,0 +1,37 @@ +# Eval Plan + +The first question is not whether this protocol is elegant. The first question is whether it helps. + +## Conditions + +1. Plain prompting with no explicit map +2. Plain prompting with a pinned architecture map in context +3. External typed state with no verifier stack +4. External typed state with verifier checks + +## Candidate Metrics + +- task success +- regression rate after follow-up edits +- total diff size +- revert rate +- contradiction rate between map and final patch +- branch kill rate +- human rating of architectural coherence + +## Good First Tasks + +- fix a bug without regressing adjacent behavior +- refactor a medium file while preserving invariants +- add one feature that touches multiple connected modules + +## Failure Smells + +- the map and the code disagree +- scratch grows without being summarized or deleted +- branches accumulate but are never killed +- tests pass while architectural coherence gets worse + +## Exit Criteria For The Prototype + +The prototype is worth keeping if it reliably reduces drift or makes failure easier to diagnose without adding absurd overhead. diff --git a/notes/architecture-rationale.md b/notes/architecture-rationale.md new file mode 100644 index 00000000..948a8702 --- /dev/null +++ b/notes/architecture-rationale.md @@ -0,0 +1,134 @@ +# Architecture Rationale + +## Core Epiphany + +The central failure mode is local plausibility without global coherence. + +An agent can keep making reasonable-looking edits, pass narrow tests, improve proxy metrics, and still lose the actual shape of the system. Once the mental model collapses, adding more local fixes often makes the tower worse. + +The response is to make global coherence a first-class object: + +- keep a persistent map +- isolate scratch reasoning +- treat public output and code edits as commits, not thoughts +- verify against the map and the real goal +- kill dead churn instead of preserving maybe-useful machinery + +## Proposed Model Shape + +The speculative model architecture is a typed-state controller around a language backbone: + +```mermaid +flowchart TD + U["User / Environment"] --> P["Prompt + Observations Encoder"] + P --> C["Controller Policy"] + C --> M["Persistent Map Store"] + C --> T["Scratch Thought Workspace"] + C --> O["Output Stream"] + C --> S["Search Module"] + C --> B["Sparse Backbone LLM"] + M --> B + T --> B + P --> B + S --> B + B --> D["Drafts / Plans / Patches / Critiques"] + D --> V["Verifier Stack"] + V --> C + D --> S + D --> M + D --> O + O --> U +``` + +The important split: + +- `map`: slow-changing canonical system model, invariants, decisions, data flow, rejected paths +- `scratch`: volatile local reasoning for one bounded subgoal +- `output`: user-visible replies, tool calls, and code edits +- `controller`: learned or prompted policy deciding when to model, think, verify, edit, backtrack, or speak +- `verifier`: tests plus map coherence, simplicity, and real-objective checks +- `search`: bounded branching and backtracking for hard decomposition choices + +## Why A Pinned Map Is Not Enough + +Pinning a map inside the context window is useful, but it is still one token soup: + +- it competes with everything else for attention +- it has no typed write semantics +- it can drift silently +- it does not distinguish stable knowledge from temporary hypotheses + +The cheap prototype uses external files to fake typed state before any model architecture exists. + +## Cheap Prototype Strategy + +Do not train a new backbone first. That is where money goes to die. + +Instead: + +1. Externalize state in files. +2. Use a controller protocol with a small action space. +3. Make the model choose one action at a time. +4. Record evidence after meaningful steps. +5. Compare against plain prompting. + +Current state files: + +- `state/map.yaml` +- `state/scratch.md` +- `state/branches.json` +- `state/evidence.jsonl` + +Current controller actions: + +- `update_map` +- `think_subgoal` +- `propose_patch` +- `run_verify` +- `compare_branches` +- `backtrack` +- `speak` + +## Evaluation Shape + +Compare four conditions: + +1. plain prompting +2. pinned map in normal context +3. external typed state without verifier discipline +4. external typed state with verifier discipline + +Measure: + +- task success +- regression rate after follow-up edits +- total diff size +- revert rate +- contradiction rate between map and patch +- branch kill rate +- human rating of architectural coherence + +The best informal metric: after five iterations, does the system still make sense? + +## DeepSeek Research Signals + +These sources informed the architecture direction. The repo does not depend on them directly, but they are useful background. + +- DeepSeek-R1 suggests RL can elicit reflection and self-verification, but also shows raw emergent reasoning can become repetitive or sloppy: https://github.com/deepseek-ai/DeepSeek-R1 +- Generalist Reward Modeling suggests critique and principle generation can be scaled at inference time: https://arxiv.org/abs/2504.02495 +- DeepSeek-Prover-V1.5 uses intrinsic-reward tree search, supporting the idea that search helps branching reasoning tasks: https://github.com/deepseek-ai/DeepSeek-Prover-V1.5 +- DeepSeek-Prover-V2 leans on subgoal decomposition, supporting explicit decomposition rather than pure continuation: https://arxiv.org/abs/2504.21801 +- DeepSeekMath-V2 emphasizes verification quality, not just final-answer correctness: https://github.com/deepseek-ai/DeepSeek-Math-V2 +- DeepSeek-V2/V3 show the usefulness of sparse MoE and efficient attention for cheaper large-scale reasoning: https://arxiv.org/abs/2405.04434 and https://arxiv.org/abs/2412.19437 +- Engram argues for explicit conditional memory and lookup as a separate axis of sparsity, which matches the need for map-like persistent state: https://arxiv.org/abs/2601.07372 + +## Codex-Specific Interpretation + +Codex cannot expose a new internal model architecture from this repo. The practical path is harness-level behavior: + +- add an Epiphany collaboration mode or preset +- inject developer instructions that enforce external typed state +- optionally add a tiny state helper tool +- evaluate whether that improves behavior before touching protocol enums + +This is not a new brain. It is a better harness around the existing one. diff --git a/notes/codex-epiphany-mode-plan.md b/notes/codex-epiphany-mode-plan.md new file mode 100644 index 00000000..0b60e1c1 --- /dev/null +++ b/notes/codex-epiphany-mode-plan.md @@ -0,0 +1,173 @@ +# Codex Epiphany Phase 1 Implementation Plan + +This is the actionable implementation note for the first Epiphany patch slice against the real `openai/codex` tree vendored under `vendor/codex`. + +Phase 1 is deliberately **internal/dev-usable**, not user-facing. The goal is to give Codex a real durable Epiphany state seam in core, keep the app usable, and prove replay/persistence before touching prompts, presets, protocol notifications, or GUI reflection. + +## Summary + +Land Phase 1 as an **additive core slice** that persists and rehydrates structured Epiphany thread state without changing normal Codex behavior. + +After this slice: + +- non-Epiphany sessions still behave exactly as they do now +- the app remains usable because existing clients ignore the new metadata +- Epiphany state can be exercised and validated through automated core/rollout/resume tests + +This slice does **not** include: + +- prompt integration +- app-server notifications +- GUI surfaces +- collaboration-mode presets +- mutation gates +- retrieval indexing +- specialist-agent orchestration + +## Key Changes + +### 1. Add minimal durable Epiphany protocol types + +Touch: + +- `vendor/codex/codex-rs/protocol/src/protocol.rs` + +Add: + +- `EpiphanyThreadState` +- `EpiphanyStateItem { turn_id: Option, state: EpiphanyThreadState }` +- `RolloutItem::EpiphanyState(EpiphanyStateItem)` + +Phase 1 schema should stay minimal but sufficient: + +- `revision` +- `objective` +- `active_subgoal_id` +- `subgoals` +- `invariants` +- `graphs` +- `graph_frontier` +- `graph_checkpoint` +- `scratch` +- `observations` +- `recent_evidence` +- `churn` +- `mode` +- `last_updated_turn_id` + +Defer these to later phases: + +- `agents` +- `jobs` +- `retrieval` +- separate append-only evidence/checkpoint rollout items +- separate turn intent rollout item +- any new `EventMsg` variants + +All new protocol types should derive the same serialization/schema traits as neighboring rollout types. + +### 2. Store Epiphany state in `SessionState` + +Touch: + +- `vendor/codex/codex-rs/core/src/state/session.rs` +- `vendor/codex/codex-rs/core/src/session/mod.rs` + +Add: + +- `epiphany_state: Option` to `SessionState` +- simple getter/setter helpers on session state + +Persistence rule: + +- persist one `RolloutItem::EpiphanyState(...)` per real user turn +- write it immediately after the `TurnContextItem` baseline is persisted +- skip the write when `epiphany_state` is `None` + +Do not change prompt assembly or tool behavior in this slice. + +### 3. Rehydrate Epiphany state on resume and fork + +Touch: + +- `vendor/codex/codex-rs/core/src/session/rollout_reconstruction.rs` +- the session-start/resume path that consumes `RolloutReconstruction` + +Extend reconstruction to carry: + +- `epiphany_state: Option` + +Replay rule: + +- the newest surviving `EpiphanyStateItem` wins +- rollback must discard Epiphany snapshots from rolled-back turns +- compaction must not erase the latest surviving snapshot + +Treat `EpiphanyStateItem` as turn-scoped metadata in the same reverse scan that already respects rollback and compaction boundaries. + +### 4. Update rollout readers to tolerate the new variant + +Adding a `RolloutItem` variant will create exhaustiveness fallout across the repo. Update those sites explicitly instead of letting it turn into whack-a-mole. + +At minimum touch: + +- `vendor/codex/codex-rs/app-server-protocol/src/protocol/thread_history.rs` +- `vendor/codex/codex-rs/rollout/src/policy.rs` +- `vendor/codex/codex-rs/rollout/src/metadata.rs` +- `vendor/codex/codex-rs/rollout/src/list.rs` +- `vendor/codex/codex-rs/rollout/src/recorder.rs` +- any additional exhaustive `RolloutItem` matches in `core`, `tui`, `exec`, and `protocol` + +Phase 1 behavior at those call sites: + +- thread-history/UI builders should ignore `EpiphanyState` +- rollout metadata/listing code should treat it as non-rendered metadata +- existing user-visible thread rendering must stay unchanged + +## Test Plan + +### Protocol and schema + +- serde round-trip for `EpiphanyThreadState` +- serde round-trip for `EpiphanyStateItem` +- any protocol/schema snapshots updated for the new `RolloutItem` variant + +### Core persistence + +- a session with populated Epiphany state writes `RolloutItem::EpiphanyState` +- a session with `None` Epiphany state writes nothing extra +- the Epiphany rollout item appears after the turn-context baseline for the turn + +### Replay correctness + +- resume restores the newest surviving Epiphany snapshot +- rollback drops snapshots from discarded turns +- compaction keeps the latest surviving Epiphany snapshot available after reconstruction + +### Compatibility + +- `app-server-protocol` thread reconstruction still builds the same visible turn history +- rollout list/metadata readers still work with mixed old/new rollout files +- untouched user flows remain unchanged in tests + +### Compile-fallout coverage + +- build/test the touched crates so all exhaustive `RolloutItem` matches are updated in one pass + +## Assumptions + +- Phase 1 is **internal/dev-usable**, not operator-usable or GUI-visible. +- "Testable after Phase 1" means persistence/replay behavior is verifiable through automated tests and internal session wiring, not that users can activate Epiphany from the UI yet. +- Phase 1 keeps evidence and checkpoint data embedded inside the baseline Epiphany snapshot instead of splitting them into separate rollout item variants. +- Prompt integration, app-server typed updates, GUI scene loading, retrieval, jobs, and specialist-agent scheduling are later slices. + +## Immediate Next Step After This Plan + +Implement the protocol/core/replay slice only: + +1. add the minimal protocol types +2. wire `SessionState` +3. persist one snapshot per real user turn +4. restore it through rollout reconstruction +5. patch exhaustive `RolloutItem` matches +6. add replay/persistence tests before moving on diff --git a/notes/codex-repository-algorithmic-map.md b/notes/codex-repository-algorithmic-map.md new file mode 100644 index 00000000..ae2ccdd6 --- /dev/null +++ b/notes/codex-repository-algorithmic-map.md @@ -0,0 +1,490 @@ +# Codex Repository Algorithmic Map + +This is the working map for how vendored Codex moves data. It exists because the repo is large, the crate boundaries are real, and "just keep it in your head" is how you end up chewing drywall. + +Scope: this is the main request/response loop for the open-source Codex repository in `E:\Projects\EpiphanyAgent\vendor\codex`. It focuses on the steady-state path from user input to model prompt, tool execution, UI events, and persistence. It is not a full inventory of every crate. + +## Mental model in one sentence + +Codex is a threaded agent engine in `core` wrapped by transport layers (`app-server`, `protocol`) and user interfaces (`tui`, CLI entrypoints), with durable execution history written to rollouts and indexed into SQLite metadata. + +## Main actors + +| Actor | Job | Important crates | +| --- | --- | --- | +| CLI/TUI front door | Parse user intent, bootstrap config, start or connect to the engine, render events | `cli`, `tui` | +| App server | Translate UI/CLI requests into core thread operations and stream results back over protocol transport | `app-server`, `protocol` | +| Thread manager | Own shared managers and the registry of active threads | `core` | +| Thread/session engine | Hold mutable thread state, build prompts, run turns, stream model events, persist history | `core` | +| Model client | Talk to the model provider, maintain turn-scoped streaming state, reuse websocket state inside a turn | `core`, `backend-client`, provider crates | +| Tool runtime | Expose model-visible tools, validate requests, execute shell/MCP/function calls, return tool outputs | `core`, `mcp-server`, `exec`, `exec-server` | +| Watchers and live side channels | Watch files, emit coarse change notifications, surface turn diffs and other realtime status | `core`, `app-server`, `app-server-protocol` | +| Memory and summarization | Extract rollout memories, maintain summaries, compact long history for future prompts | `core`, `rollout-trace`, `state` | +| Persistence layer | Write durable thread rollouts, mirror metadata into SQLite, support listing/resume/fork/archive | `rollout`, `thread-store`, `state` | + +## Top-level flow + +```mermaid +flowchart TD + A["User action in CLI or TUI"] --> B["CLI/TUI bootstrap"] + B --> C["App-server request or embedded app-server"] + C --> D["ThreadManager routes to CodexThread"] + D --> E["Session builds TurnContext"] + E --> F["Prompt assembly"] + F --> G["ModelClientSession streams ResponseEvents"] + G --> H["Assistant items"] + G --> I["Tool calls"] + I --> J["ToolRouter + ToolCallRuntime"] + J --> K["Tool outputs recorded as input items"] + K --> F + H --> L["Turn items + protocol events"] + L --> M["Rollout JSONL + SQLite metadata"] + L --> N["UI redraw / app-server notifications"] + L --> O["Compaction / summaries / replacement history"] + L --> P["Unified diff + watcher side channels"] +``` + +## Startup dataflow + +### 1. CLI chooses a surface + +- `cli/src/main.rs` is the top-level command parser. +- Interactive usage usually lands in the TUI path. +- Non-TUI commands branch into things like exec, review, sandbox, app server, or MCP server. + +The important bit: the CLI mostly decides which surface to launch. It is not the agent brain. + +### 2. TUI bootstraps the environment + +`tui/src/lib.rs` does the heavy bootstrap work: + +- load config +- resolve auth constraints +- set up logging and environment management +- decide whether to launch an embedded app server or connect to a remote one +- start the TUI client against that app-server target + +The TUI is a client shell around the engine, not the engine itself. + +### 3. App server becomes the transport seam + +`app-server` owns the JSON-RPC-ish transport loop and client/session coordination. + +Its job is to: + +- accept UI/client requests +- resolve config overrides and trust checks +- create or resume threads through `ThreadManager` +- subscribe to thread events +- forward thread snapshots, deltas, approvals, and status changes back to the client + +This is the cleanest seam for a custom GUI wrapper. A new GUI should talk to the app server, not reimplement the core turn loop. + +## Core runtime model + +### ThreadManager: the factory and registry + +`core/src/thread_manager.rs` owns the long-lived shared machinery: + +- models manager +- skills manager +- plugins manager +- MCP connection manager +- environment manager +- active thread registry + +It creates new threads, resumes threads from rollouts, and forks existing threads by reconstructing history from persisted rollout items. + +### CodexThread: the per-thread facade + +`core/src/codex_thread.rs` is the stable handle the rest of the system uses for a live thread. + +It delegates thread operations into the underlying session: + +- submit input +- steer input +- fetch events +- call MCP tools +- flush rollout +- expose config snapshots + +Think of it as the thread-shaped shell around a session. + +### Session: the actual mutable brain case + +`core/src/session/session.rs` is where the real per-thread state lives. + +A session owns: + +- current configuration and overrides +- conversation/thread history +- active turn state +- event channels to the UI/app server +- services bundle for auth, models, skills, plugins, MCP, analytics, and state DB +- rollout recorder and trace recorder + +This is the core locus of mutation. If a thread changes, it changes here. + +## Turn dataflow: one full loop + +This is the central algorithmic loop. + +### 1. Input enters the thread + +Typical path: + +1. user message or command enters the TUI +2. TUI sends an app-server request +3. `CodexMessageProcessor` turns that into a core op +4. `CodexThread.submit(...)` or `submit_with_trace(...)` forwards it into the session + +For a custom GUI, this is the same path you want. The GUI should produce thread operations, not hand-build prompts. + +### 2. Session creates a turn + +The session turns the incoming op into a concrete turn with: + +- sub-id / trace metadata +- collaboration mode +- approval and sandbox policy +- cwd and environment +- current date / timezone +- developer instructions +- user instructions +- personality +- feature flags +- dynamic tools +- model info and reasoning settings + +This bundle becomes `TurnContext` in `core/src/session/turn_context.rs`. + +That matters because the prompt is not built from raw user text alone. It is built from user text plus a large amount of execution policy and mode state. + +### 3. Context and history are recorded before sampling + +Before the model is asked anything, Codex records: + +- context updates +- user input +- initial turn items +- skill/plugin injections +- relevant history mutations + +This keeps the transcript, rollout, and UI state aligned even when a turn later branches into tools, retries, or cancellation. + +### 4. Tools are resolved for this turn + +`built_tools(...)` in `core/src/session/turn.rs` builds the model-visible tool surface for the turn. + +It gathers: + +- built-in tools +- MCP tools from the connection manager +- connector/tool-search surfaces when enabled +- dynamic tools +- permission-gated capabilities + +This becomes a `ToolRouter`, which is both: + +- the registry used at execution time +- the model-visible tool schema exposed in the prompt + +### 5. Prompt assembly happens + +Prompt construction is easiest to see in `core/src/prompt_debug.rs` and `core/src/session/turn.rs`. + +The prompt is assembled from: + +- formatted input/history window +- base instructions +- collaboration mode developer instructions +- optional personality +- tool definitions +- parallel-tool-call capability +- optional output schema + +Useful frame: Codex does not have one static system prompt. It synthesizes a turn-scoped prompt from history plus runtime policy. + +### 6. The model stream begins + +`ModelClientSession` is the turn-scoped provider client. + +It: + +- opens or reuses provider transport for the turn +- may prewarm websocket state +- streams `ResponseEvent`s back into the session +- retains turn-scoped routing state such as sticky turn tokens + +The stream is the heartbeat of the turn. Everything after this is driven by response items arriving incrementally. + +### 7. Stream events split into two classes + +As `ResponseEvent`s arrive, Codex handles them in `core/src/session/turn.rs` and `core/src/stream_events_utils.rs`. + +#### Class A: non-tool output + +These become visible assistant artifacts: + +- assistant messages +- reasoning items +- web search markers +- image-generation items + +They are parsed into turn items, emitted to the UI, and recorded into history/rollout. + +#### Class B: tool calls + +If a completed response item is actually a tool request, `ToolRouter::build_tool_call(...)` converts it into an executable invocation. + +Then Codex: + +1. records the tool-call item immediately +2. queues execution through `ToolCallRuntime` +3. waits for the tool result +4. turns the tool result into `ResponseInputItem` +5. records that output back into history +6. loops back into another sampling request if follow-up is needed + +This is the critical feedback loop. The model does not directly mutate the world. It emits tool requests, gets structured outputs, and continues sampling with those outputs now in-context. + +### 8. The turn repeats until follow-up stops + +Sampling can loop multiple times inside one user-visible turn: + +- model emits tool calls +- tools execute +- outputs are recorded +- model samples again + +The turn ends when the model reaches a completed state with no more required tool follow-up. + +### 9. Final outputs settle into three places + +At the end of a turn, the result exists in three overlapping representations: + +- live turn items for UI rendering +- conversation history for future prompts +- rollout/state persistence for resume, listing, fork, and analysis + +That triplicate shape matters. Codex is always balancing live UX, future context, and durable replay. + +## Tool execution dataflow + +Tool execution is not one subsystem. It is a routing stack. + +### ToolRouter decides what the model asked for + +`core/src/tools/router.rs` converts completed model response items into tool invocations: + +- shell commands +- function tools +- MCP tools +- custom tools +- tool-search calls + +It also knows which tools are model-visible and whether they can run in parallel. + +### ToolCallRuntime actually runs the call + +`ToolCallRuntime` is the execution harness: + +- dispatch built-in tools +- call MCP servers +- run shell commands with approval/sandbox policy +- handle cancellation +- return a structured result payload + +Then the session records that payload as part of the same turn's transcript. + +Short version: the model proposes, the router interprets, the runtime executes, and the session turns the result back into prompt material. + +### Important caveat: tool outputs are still mostly transcript-shaped + +This is one of the important current limitations. + +Codex does have structured tool payload types. But much of the runtime still metabolizes tool execution back into response items and transcript/history material rather than promoting the result into a stronger shared state layer. + +In practice that means: + +- useful tool output often becomes more prompt material +- later turns may need to rediscover or resummarize facts from history +- there is only a weak membrane between "raw command exhaust" and "durable understanding" + +That is part of why Epiphany needs an explicit observation/distillation layer instead of trusting transcript osmosis. + +## Auxiliary side channels that matter + +The main turn loop is the heart of the machine, but Codex also has a few auxiliary channels that shape behavior around it. + +### File watching exists, but mostly as coarse notification substrate + +Codex already has a generic file-watcher layer and app-server fs-watch support. + +Today that mostly supports things like: + +- skills change notifications +- client-visible file watch events +- coarse "something changed here" awareness + +What it does **not** natively do is maintain a typed repo-understanding model or semantically invalidate graph claims, invariants, or code-intel shards when the code moves. + +So the watcher substrate is real, but its understanding layer is thin. + +### Turn diffs exist, but they are textual + +Codex also tracks per-turn file changes and can emit an aggregated unified diff snapshot to clients. + +That is useful for UI display and review, but it is still mostly text-level impact reporting. + +It does not, by itself, answer: + +- which mapped subsystem is now stale +- which mechanism answer was invalidated +- which graph node or invariant should be reopened + +So the diff channel is a useful sensor, not yet a semantic diagnosis layer. + +### Subagents exist, but coordination is mostly message/config based + +Codex already supports spawning subagents, sending them messages, waiting on them, and closing them. + +That is real multi-agent execution substrate. + +But the dominant collaboration model is still: + +- inherit parent config and tools +- exchange prompts/messages +- report completion/status back up the tree + +There is not yet a strong shared typed state plane for map truth, evidence authority, retrieval freshness, or invalidation ownership. The agents can cooperate, but mostly by talking through the same old language pipe. + +### Startup memory exists, but it is the wrong kind of memory for repo understanding + +Codex has a memories pipeline that extracts and consolidates useful summaries from rollouts. + +That helps with: + +- user/task continuity +- startup summarization +- durable memory-like guidance + +But it is not the same thing as a canonical repo-understanding store, and it is not a substitute for a graph/evidence/invalidation plane shared by specialists. + +So the machine already has memory organs, just not the ones we actually need for Epiphany. + +## Persistence model + +Codex has two persistence stories layered together. + +### 1. Rollout files are the durable thread log + +The rollout layer writes thread history as append-oriented session/thread artifacts under `codex_home`. + +These rollouts are the source used for: + +- resume +- fork +- archive +- summary reconstruction +- durable event history + +`ThreadManager` can rebuild a thread from rollout history, which is why rollout shape matters more than any one in-memory UI state. + +### 1.5. Compaction is part of the real dataflow + +Long-running threads do not just grow forever. Codex compacts history and preserves a replacement history that future prompts may carry forward. + +That matters because compaction is not some decorative cleanup pass. It changes what later turns see as their effective past. + +Today that compaction path is still mostly transcript- and summary-shaped. It preserves continuity, but not a first-class typed model of repo understanding. + +That is why our current external map/evidence files feel useful: they are compensating for a real hole in the native harness. + +### 2. SQLite mirrors metadata for indexing and control + +The `state` crate mirrors rollout-derived metadata into SQLite. + +That database supports things like: + +- thread metadata +- backfill/extraction state +- memory-mode pollution flags +- job orchestration +- fast listing/filtering without replaying every rollout file + +So the shape is: + +- rollout = durable narrative log +- SQLite = queryable control/index layer + +One uncomfortable but useful truth: + +- rollout and SQLite are durable +- transcript and metadata replay are real +- but Codex still lacks a canonical typed repo-knowledge layer sitting above them + +So persistence is strong for history, weaker for understanding. + +### 3. Thread store abstracts local vs remote persistence + +`thread-store` defines storage-neutral thread persistence interfaces. + +That abstraction exists so callers can treat a `ThreadId` as the durable identity while storage backends decide whether the thread lives in local rollout files, remote storage, or something else. + +## UI and engine boundary + +This is the most important architectural boundary for EpiphanyAgent. + +The TUI is not where prompt assembly or tool execution fundamentally lives. + +Those live in `core`, with `app-server` acting as the transport/control seam. + +So a dedicated GUI wrapper should probably: + +- talk to app-server protocol +- consume thread snapshots and turn events +- send thread operations and settings updates +- leave prompt building, tool execution, persistence, and model streaming inside Codex core + +That keeps the wrapper thin and avoids rebuilding the dangerous part twice. + +## Where Epiphany mode naturally hooks in + +Epiphany should hook in at the turn-construction and prompt-assembly layer, not by inventing a parallel execution engine. + +Natural hook points: + +1. collaboration preset resolution in the models/config path +2. mode/preset visibility in the UI layer +3. developer-instruction injection before prompt assembly +4. possibly app-server/client settings so the chosen preset survives non-TUI flows + +The important thing is that Epiphany changes how turns are instructed and surfaced, not how the core agent loop fundamentally works. + +It also needs to patch the blind spots around that loop: + +- code-intelligence and hybrid retrieval +- semantic invalidation from repo changes +- structured distillation from tool output into durable facts +- shared knowledge across specialists without forcing every role to reread the whole transcript + +## Anti-confusion summary + +If the repo starts feeling slippery again, remember this: + +- CLI chooses the surface. +- TUI is a client shell. +- App server is the transport seam. +- ThreadManager owns shared managers and live threads. +- Session owns mutable thread state. +- TurnContext captures per-turn policy and environment. +- Prompt assembly happens per turn. +- Model streaming produces response items. +- Response items either become UI/history artifacts or executable tool calls. +- Tool outputs are fed back into the same turn. +- File watchers, unified diffs, compaction, and memory summarization are real side channels around the main loop. +- Those side channels are useful substrate, but they are not yet a semantic repo-understanding layer. +- Rollouts persist the narrative; SQLite indexes the metadata. + +That is the machine. Everything else is detail, decoration, or damage. diff --git a/notes/epiphany-core-harness-surfaces.md b/notes/epiphany-core-harness-surfaces.md new file mode 100644 index 00000000..979b2d2c --- /dev/null +++ b/notes/epiphany-core-harness-surfaces.md @@ -0,0 +1,1596 @@ +# Epiphany Core Harness Surfaces + +This note defines where Epiphany should actually live inside vendored Codex. + +Short version: Epiphany belongs in the core agent harness. The GUI is a reflection layer over that state, not the source of truth. If we make the UI the place where the map lives, we have built a pretty lie. + +## Current mechanism + +Codex already has the right architectural bones: + +- `core` owns mutable thread/session state, prompt assembly, tool execution, and rollout persistence +- `protocol` defines durable and streamed artifacts like `TurnContextItem`, `EventMsg`, and turn items +- `app-server` translates core events into client-facing notifications +- `app-server-protocol` defines the UI-facing thread/turn/item model + +There are two especially useful precedents: + +1. `TurnContextItem` + - persisted once per real user turn + - becomes the durable baseline for resume/fork/replay + - keeps structured execution context out of a purely implicit prompt + +2. `PlanUpdate` / `TurnDiff` + - emitted from core as structured events + - translated by app-server into typed notifications + - rendered by clients without making the client invent the data + +Epiphany should follow that pattern. + +## Intended change + +Add a structured Epiphany state layer to the thread/session engine so the agent can: + +- keep a canonical mental map outside the raw prompt stream +- carry invariants and active subgoal state across turns +- append evidence as durable records +- expose its current model of the system to the UI as real data +- gate broad edits on declared intent and freshness of understanding + +That means: + +- `core` owns Epiphany state +- rollout persistence stores it durably +- `app-server` exposes it over typed notifications and thread read/resume/start responses +- the GUI renders and edits that same state through protocol methods + +## Design principle + +Epiphany state should be structured first and summarized into language second. + +The model still needs language, obviously. But the source of truth should be typed state with revisioned updates, not one increasingly cursed blob of prose. + +That does **not** mean the structured state should be linguistically thin. Quite the opposite. + +The internal representation should carry rich natural-language explanations of what each component or stage is for, what role it plays in the machine, and why it belongs. Metaphors are not fluff here. They are compression aids for both the human and the model. + +So the rule is: + +- structure for shape and replay +- language for understanding +- code refs for grounding +- distillation for turning raw tool noise into durable facts + +If any one of those is missing, the map gets dumber than it needs to be. + +## Multi-agent harness principle + +Codex should not stay one overstuffed context forever. + +For nontrivial work, Epiphany should be able to run as a small society of bounded minds: + +- one supervising harness +- several specialist agents +- one shared typed state plane +- private scratch per agent + +The key rule is: + +- share artifacts, not inner monologue + +That means the mapper does **not** need to know what the worker was privately mulling over while editing a file. It needs to know: + +- what changed +- what the intended effect was +- what evidence exists +- which graph claims are now stale + +Likewise, the worker does not need the mapper's whole scratchpad. It needs: + +- the active objective and subgoal +- the relevant graph slice +- open mechanism gaps in the touched area +- invariants +- the expected effect of the current edit + +Suggested core roles: + +1. `Coordinator` + - owns objective routing, subgoal assignment, mutation gates, and final merge policy + +2. `Mapper` + - owns architecture/dataflow graph truth, freshness, coverage, mechanism questions, mechanism gaps, and code refs + +3. `Worker` + - owns implementation turns, edit intent, and code changes + +4. `Verifier` + - owns checks, measurements, evidence capture, and invariant verification status + +5. `ChurnMonitor` (optional later) + - watches for the pattern where diffs grow while understanding shrinks + +The ownership rule matters more than the role list. + +If every agent can mutate every field directly, we do not get intelligence. We get a committee with a concussion. + +## Reuse the existing substrate, do not inherit its limitations + +Codex already has useful pieces: + +- file watching +- diff streaming +- subagent lifecycle and messaging +- rollout and memory machinery + +That means Epiphany should not start from bare dirt. + +But those pieces are not yet a real repo-understanding harness. + +Today: + +- file watching mostly tells the system "something changed" +- diff streaming mostly says "here is the current unified diff text" +- subagents mostly inherit config/tools and message each other +- memory mostly summarizes rollouts and user/task context + +Those are valuable inputs, but they are not enough by themselves. + +Epiphany should reuse them as substrate while adding the missing semantic layer above them: + +- typed repo understanding +- semantic freshness invalidation +- structured tool-output distillation +- a shared durable knowledge plane for specialist agents + +Without that layer, Codex still behaves like a capable shop floor with no reliable blueprint archive. + +## Repository retrieval and code-intelligence principle + +Codex is also missing a proper repository retrieval organ. + +Today the system can lean on shell search, ripgrep, path walking, fuzzy file search, and file-by-file inspection. That is useful, but it is still a clumsy way to answer concept-level questions about a large codebase. + +It is also missing a first-class structural code-intelligence layer. + +Path matches and fuzzy file hits are not the same thing as knowing: + +- where a symbol is defined +- where it is referenced +- which local call or dependency edges connect two stages +- which module or subsystem owns a transformation +- which graph claims should be invalidated when a symbol moves + +Epiphany should add a first-class repo retrieval subsystem, because "just keep listing files until understanding happens" is a miserable use of time, attention, and tokens. + +The retrieval rule should be: + +- hybrid retrieval, not vector-only +- repo-local storage, not one giant global blob +- sharded indexes, not one monolith per machine +- incremental invalidation from diffs, not full rebuilds by reflex +- freshness-aware query results, not silent stale hits +- role-scoped query policies, not every agent blasting the whole corpus every turn + +The retrieval stack should combine: + +1. exact path and filename lookup +2. symbol and identifier lookup +3. definition/reference lookup +4. lexical text search +5. semantic chunk retrieval +6. graph-linked retrieval back to architecture/dataflow nodes +7. lightweight structural relations such as local call, read/write, emit/subscribe, or import edges when available + +That gives us both halves of the job: + +- exact search for "where is `ExpandSolar` called?" +- semantic search for "where does gravity become a rendered nebula field?" + +The vector side is only one organ in that machine. A pure vector DB is how you end up with blurry confidence and muddy grounding. + +## Compaction rule + +Compaction should squeeze the scratch, not the map. + +For large repository-mapping work, Epiphany should keep three distinct layers: + +1. `Spine` + - durable cross-turn, cross-compaction knowledge + - architecture graph + - dataflow graph shards + - invariants + - durable evidence + - graph coverage/provenance + +2. `Frontier` + - the currently investigated slice of the machine + - active shard ids + - active node ids + - unresolved questions + - likely next probe targets + +3. `Scratch` + - temporary hypotheses and local reasoning for the current bounded subgoal + +The policy is: + +- `Spine` survives compaction +- `Frontier` is checkpointed and reselected after compaction +- `Scratch` is aggressively rewritten, rotated, or discarded + +If a compaction wipes the graph spine and leaves only a prose aftertaste, Epiphany has failed. + +## Proposed state model + +Epiphany needs both thread-scoped state and turn-scoped intent. + +### 1. Thread-scoped state + +Add an `EpiphanyThreadState` owned by `SessionState`. + +Suggested shape: + +```text +EpiphanyThreadState +- revision: u64 +- objective: String +- active_subgoal_id: Option +- subgoals: Vec +- agents: EpiphanyAgentRegistry +- jobs: Vec +- invariants: Vec +- graphs: EpiphanyGraphs +- retrieval: EpiphanyRetrievalState +- graph_frontier: EpiphanyGraphFrontier +- graph_checkpoint: EpiphanyGraphCheckpoint +- scratch: Option +- observations: Vec +- recent_evidence: Vec +- churn: EpiphanyChurnState +- mode: Off | Observe | Enforce +- last_updated_turn_id: Option +``` + +#### `objective` + +The current thread-level objective in plain language. + +This is the anchor that tells the rest of the structure what "better" means. + +#### `subgoals` + +Explicit bounded work units. + +Each subgoal should carry: + +- `id` +- `title` +- `status` +- `scope_note` +- `expected_effect` + +This is stricter than the existing `update_plan` checklist. The plan tool is useful but generic; Epiphany subgoals should be part of the durable thread model. + +#### `agents` + +Role metadata for the specialist agents attached to the thread. + +Suggested shape: + +```text +EpiphanyAgentRegistry +- agents: Vec + +EpiphanyAgentState +- id: String +- role: EpiphanyAgentRole +- status: AgentRunStatus +- owned_state_domains: Vec +- subscriptions: Vec +- active_subgoal_id: Option +- frontier_hint_ids: Vec +- last_heartbeat_turn_id: Option +- last_output_summary: Option + +EpiphanyAgentRole +- Coordinator +- Mapper +- Worker +- Verifier +- ChurnMonitor + +AgentRunStatus +- Idle +- Pending +- Running +- Blocked + +AgentTrigger +- TurnStarted +- WriteTurnCompleted +- DiffTouchedPath +- EvidenceAppended +- ExplicitRemapRequest +- ExplicitVerifyRequest +- CompactionCompleted +- ResumeLoaded +``` + +This registry is for routing, ownership, and visibility. It is **not** a license to dump every agent's private scratch into every other prompt. + +#### `jobs` + +Visible long-running or background work owned by the harness. + +Suggested shape: + +```text +EpiphanyJobState +- id: String +- kind: EpiphanyJobKind +- scope: String +- owner_role: EpiphanyAgentRole +- status: EpiphanyJobStatus +- progress_note: Option +- items_processed: Option +- items_total: Option +- last_checkpoint_at: Option +- blocking_reason: Option +- linked_subgoal_id: Option +- linked_graph_node_ids: Vec + +EpiphanyJobKind +- RetrievalRefresh +- GraphRemap +- Revalidation +- VerificationBatch +- SpecialistTask + +EpiphanyJobStatus +- Pending +- Running +- Blocked +- Completed +- Failed +``` + +These jobs are for observability and steering. They are not just implementation trivia hiding behind the curtain. + +#### `invariants` + +Thread-level constraints the agent should not violate while editing. + +Each invariant should carry: + +- `id` +- `statement` +- `kind` such as `behavioral`, `dataflow`, `performance`, `safety`, `ui`, `test` +- `status` such as `assumed`, `verified`, `violated`, `stale` +- `evidence_ids` + +This gives the agent something more precise than "be careful." + +#### `graphs` + +The canonical machine map should actually be **two linked graphs**, not one. + +- `ArchitectureGraph`: what components exist, where responsibilities live, and how the codebase is partitioned +- `DataflowGraph`: how information or control moves stage by stage through the system + +Those graphs should be cross-linked so a dataflow step can point back to the architecture component that hosts it, and vice versa. + +Suggested shape: + +```text +EpiphanyGraphs +- title: String +- summary: String +- architecture: ArchitectureGraph +- dataflow: DataflowGraph +- links: Vec +- mechanism_questions: Vec +- mechanism_gaps: Vec +- graph_revision: u64 +- coverage: GraphCoverageSummary +``` + +Suggested supporting types: + +```text +CodeRef +- path: PathBuf +- start_line: Option +- end_line: Option +- symbol: Option +- note: Option +- last_seen_commit: Option +- path_fingerprint: Option + +ArchitectureGraph +- nodes: Vec +- edges: Vec + +DataflowGraph +- shards: Vec +- cross_shard_edges: Vec + +DataflowGraphShard +- id: String +- title: String +- subsystem: Option +- summary: String +- coverage_status: GraphCoverageStatus +- completion_state: MappingCompletionState +- nodes: Vec +- edges: Vec +- open_questions: Vec +- required_question_ids: Vec +- open_gap_ids: Vec +- evidence_ids: Vec +- last_validated_revision: Option + +DataflowArchitectureLink +- id: String +- dataflow_node_id: String +- architecture_node_id: String +- relationship: String +- rationale: String +- code_refs: Vec + +MappingCompletionState +- InProgress +- PassComplete +- DoneCriteriaMet + +MechanismQuestion +- id: String +- prompt: String +- scope: String +- source_node_ids: Vec +- sink_node_ids: Vec +- status: MechanismQuestionStatus +- answer_summary: Option +- evidence_ids: Vec +- gap_ids: Vec +- last_asked_turn_id: Option + +MechanismQuestionStatus +- Open +- ProvisionallyAnswered +- VerifiedAnswered +- ExposedGap + +MechanismGap +- id: String +- label: String +- description: String +- severity: String +- related_node_ids: Vec +- related_question_ids: Vec +- status: MechanismGapStatus +- evidence_ids: Vec + +MechanismGapStatus +- Open +- Investigating +- Resolved +- Stale +``` + +##### Architecture nodes + +Architecture nodes describe *where* the machine lives. + +Suggested fields: + +- `id` +- `label` +- `kind`: `crate`, `module`, `service`, `manager`, `session`, `tool_runtime`, `store`, `ui_surface`, `external_system` +- `purpose` +- `mechanism` +- `metaphor` +- `responsibilities` +- `code_refs` +- `mapping_status` +- `evidence_ids` +- `last_touched_revision` + +##### Dataflow nodes + +Dataflow nodes describe *what happens* to information or control. + +Suggested fields: + +- `id` +- `label` +- `kind`: `input`, `context_build`, `prompt_build`, `sampling`, `tool_dispatch`, `verification`, `persistence`, `render`, `external_call`, `artifact` +- `purpose` +- `mechanism` +- `metaphor` +- `inputs` +- `outputs` +- `code_refs` +- `mapping_status` +- `evidence_ids` +- `last_touched_revision` + +##### Graph edges + +Edges should also carry meaning, not just connectivity. + +Architecture edge fields: + +- `id` +- `from_node_id` +- `to_node_id` +- `relationship` +- `purpose` +- `code_refs` +- `mapping_status` + +Dataflow edge fields: + +- `id` +- `from_node_id` +- `to_node_id` +- `payload` +- `why_it_exists` +- `invariant_ids` +- `code_refs` +- `mapping_status` +- `evidence_ids` + +##### Why split the prose fields + +One `description` field is too mushy. It will end up holding everything and clarifying nothing. + +The minimum useful split is: + +- `purpose`: why this node exists at all +- `mechanism`: what it actually does to inputs, outputs, or control +- `metaphor`: vivid language that helps the model and the human hold the shape in mind + +That is where the "customs gate," "mail sorter," "pressure valve," or "ledger" style descriptions belong. + +##### Why line-anchored code refs matter + +Loose file paths are better than nothing, but they are still squishy. + +The map should be able to say not just "this happens in `turn.rs`" but "this stage is mainly implemented at these lines, in this symbol, for this reason." That makes the graph auditable and helps the model tie the story back to the actual mechanism. + +For long-lived mapping work, naked line numbers will drift. So line anchors should be treated as best-effort pointers, not sacred truth. The resilient identity for a code reference is: + +- file path +- symbol when available +- line span when known +- optional last-seen commit or file fingerprint + +That gives resumed work something sturdier than "line 847, good luck." + +##### Coverage and provenance + +Large repository maps need explicit coverage state or they turn into false confidence machines. + +Suggested coverage/provenance fields: + +```text +GraphCoverageSummary +- architecture_status: GraphCoverageStatus +- dataflow_status: GraphCoverageStatus +- shard_statuses: Vec + +GraphCoverageStatus +- Unmapped +- Surveyed +- PartiallyMapped +- DeeplyMapped +- NeedsRevalidation + +MappingStatus +- Inferred +- Verified +- Stale +``` + +This lets the resumed agent distinguish between: + +- "we have never looked here" +- "we skimmed this once" +- "this is mapped deeply and probably trustworthy" +- "this was mapped, but the code moved and needs revalidation" + +Coverage is not the same thing as completion. + +`PassComplete` only means the current sweep has stopped finding obvious new branches. +`DoneCriteriaMet` means the current shard can survive adversarial questions about how one concrete thing turns into another. + +##### Mechanism answerability and done gates + +Large maps also need an explicit bar for "done" or they will keep mistaking neat folders for understanding. + +The durable test should be a set of representative concrete transformation questions, for example: + +- how does `X` become `Y`? +- what turns this input event into that persistent state change? +- what transforms this simulation field into that visible render artifact? +- where does this message enter, mutate state, and leave the system again? + +Suggested policy: + +- each actively mapped shard owns a small set of required `MechanismQuestion` records +- if a question cannot be answered cleanly from the graph, prose, code refs, and evidence, mark it `ExposedGap` +- when a question exposes a missing seam, create or reopen a `MechanismGap`, demote shard completion state, and keep the subgoal open +- do not upgrade a shard to `DoneCriteriaMet` while required questions remain `Open`, merely `ProvisionallyAnswered`, or tied to unresolved high-severity gaps +- preserve open questions and open gaps across compaction through the frontier and checkpoint + +This is the core distinction between: + +- "we finished a mapping pass" +- "we can actually reason from this map without bluffing" + +##### Physical layout for large repos + +For big repositories, the two graphs do not need to be stored with the same granularity. + +- the `ArchitectureGraph` can usually remain one relatively stable whole +- the `DataflowGraph` should support sharding by subsystem, feature area, or execution domain + +That way a compaction or resumed turn does not have to drag the entire repo graph back into prompt context. It can reload the spine, then pull the relevant dataflow shard into the frontier. + +This is the part the GUI can render as real architecture and dataflow views without inventing anything. + +#### `retrieval` + +The durable summary of the repo retrieval subsystem. + +Suggested shape: + +```text +EpiphanyRetrievalState +- repos: Vec +- query_policies: Vec + +RepoRetrievalIndex +- repo_root: PathBuf +- index_revision: u64 +- storage_mode: RetrievalStorageMode +- overall_freshness: RetrievalFreshness +- shards: Vec +- dirty_paths: Vec +- last_indexed_commit: Option +- last_indexed_turn_id: Option +- last_indexed_at: Option + +RetrievalStorageMode +- Hybrid +- LexicalOnly +- Disabled + +RetrievalShardSummary +- id: String +- scope: String +- kind: RetrievalShardKind +- freshness: RetrievalFreshness +- chunk_count: u64 +- symbol_count: u64 +- relation_count: u64 +- linked_graph_node_ids: Vec +- last_indexed_revision: Option + +RetrievalShardKind +- Repo +- Subsystem +- Generated + +RetrievalFreshness +- Fresh +- Dirty +- Reindexing +- Stale + +RoleRetrievalPolicy +- role: EpiphanyAgentRole +- default_modes: Vec +- allow_global_queries: bool +- max_shards_per_query: u32 + +RetrievalQueryMode +- ExactPath +- Symbol +- Definition +- References +- LexicalText +- SemanticText +- Structural +- Hybrid +- GraphLinked +``` + +This state is intentionally summary-level. It should describe the health and shape of retrieval, not embed the actual chunk store into rollout. + +##### What the retrieval index should contain + +The indexed units should be richer than raw files. + +Suggested indexed surfaces: + +- code chunks with path and line anchors +- symbol records with owning file/module data +- definition/reference edges when available +- lightweight local structural relations such as calls, reads, writes, emits, subscribes, or imports when available +- graph-linked records that point back to architecture/dataflow nodes +- lightweight metadata such as language, subsystem, and generated/manual provenance + +That gives semantic search something better than "here is a 2,000-line file, good luck." + +It also gives the harness something better than "the file changed somewhere, probably panic." + +##### Physical storage rules + +The actual retrieval store should live repo-locally on disk, not in rollout and not in one cross-repo machine blob. + +Suggested storage rules: + +- one retrieval root per repo +- sharded by subsystem or execution domain for large repos +- append/update touched shards incrementally where possible +- keep lexical/symbol indexes beside semantic indexes so hybrid search can run without stitching together unrelated systems at query time + +This is partly about performance and partly about not building a giant local footgun that rewrites itself into the grave. + +##### Freshness and invalidation + +Retrieval must be freshness-aware or it becomes another source of false confidence. + +Suggested policy: + +- completed write turns mark touched shards dirty +- resumed sessions can still query dirty shards, but results should be marked as such +- the mapper should be able to demote graph claims when retrieval freshness is dirty or stale in the relevant scope +- the worker should prefer fresh or narrowly dirty shards over spraying queries across the repo +- full rebuilds should be rare and explicit, not the default reflex after every change + +Retrieval results should always carry enough metadata for the harness to say, in plain language, "this hit came from a stale shard, maybe don't build a cathedral on it yet." + +#### `graph_frontier` + +The current working slice of the graph. + +Suggested fields: + +- `active_architecture_node_ids` +- `active_dataflow_shard_ids` +- `active_dataflow_node_ids` +- `active_mechanism_question_ids` +- `open_questions` +- `open_gap_ids` +- `next_probe_targets` +- `recently_changed_node_ids` + +This is the part a resumed turn should look at first after loading the durable spine. + +#### `graph_checkpoint` + +The durable handoff record for graph work across compactions. + +Suggested fields: + +- `graph_revision` +- `active_subgoal_id` +- `frontier_shard_ids` +- `frontier_node_ids` +- `active_mechanism_question_ids` +- `recently_changed_node_ids` +- `open_questions` +- `open_gap_ids` +- `next_probe_targets` +- `checkpoint_reason` +- `created_at` + +This is not a substitute for the graph. It is the breadcrumb trail back into the graph. + +#### `scratch` + +Bounded temporary reasoning for one subgoal at a time. + +Suggested fields: + +- `subgoal_id` +- `hypothesis` +- `open_questions` +- `candidate_actions` +- `discard_after_turn: bool` + +Scratch is not the long-term store. It is allowed to be wrong and disposable. + +In a multi-agent setup, scratch should be private-by-default. + +The mapper keeps mapper scratch. The worker keeps worker scratch. The verifier keeps verifier scratch. Only promoted artifacts move into shared thread state. + +If we blindly broadcast every scratchpad to every role, we are back to one swollen context wearing a fake moustache. + +For a minimal first implementation, the single thread-level `scratch` field can act as coordinator-visible shared scratch. Role-private scratch can be added later as a separate role-scoped store rather than being stuffed into the shared thread payload. + +#### `observations` + +Structured distillation products extracted from raw tool output, verifier output, or retrieval output before they become durable graph/evidence/invariant updates. + +Suggested fields: + +- `id` +- `turn_id` +- `source_kind`: `tool_output`, `verifier_result`, `retrieval_hit`, `user_statement` +- `source_ref` +- `summary` +- `affected_paths` +- `affected_symbols` +- `candidate_graph_node_ids` +- `candidate_invariant_ids` +- `candidate_evidence_kind` +- `confidence` +- `status`: `pending`, `promoted`, `discarded` + +This is the membrane between raw transcript sludge and durable state. + +The point is not to preserve every shell belch forever. The point is to extract the parts that matter: + +- facts worth recording as evidence +- graph claims that need verification or invalidation +- invariants that may have been proved, violated, or made stale +- retrieval shards or code-intel surfaces that need refreshing + +#### `recent_evidence` + +Append-only records of what was learned. + +Each evidence record should carry: + +- `id` +- `ts` +- `turn_id` +- `kind`: `research`, `verification`, `measurement`, `rejection`, `decision` +- `claim` +- `status`: `ok`, `partial`, `rejected`, `failed` +- `source_refs` +- `notes` + +This is where tool outputs and verifier results become durable facts instead of vibes. + +Evidence should usually be promoted from structured observations, not written straight from arbitrary raw transcript chunks. + +#### Shared knowledge plane + +Specialist agents need a common world model, not just a pile of transcripts and good intentions. + +That shared durable knowledge plane should be the combination of: + +- graphs +- invariants +- retrieval summaries and freshness +- observations awaiting promotion or discard +- evidence +- checkpoints +- coordinator-visible shared scratch + +This shared plane is the collaboration substrate. + +It is **not**: + +- raw replacement history +- ad hoc prompt summaries +- the startup memories pipeline alone +- one giant merged transcript everyone is expected to inhale + +Existing memory machinery is still useful for summarization and user/task continuity, but it should not be mistaken for the authoritative repo-understanding store, especially when specialist agents are involved. + +#### Agent ownership and handoff rules + +Specialist agents should cooperate through typed state, not through prompt osmosis. + +Suggested ownership rules: + +- `Coordinator` + - may assign subgoals, open/close mutation gates, and request work from specialists + - should not silently rewrite graph truth or verifier evidence + +- `Mapper` + - is authoritative for graph structure, freshness, coverage, mechanism questions, and mechanism gaps + - may mark mapped claims stale after landed diffs or evidence + - should not silently rewrite implementation intent or test outcomes + +- `Worker` + - is authoritative for proposed edits and actual code changes + - may emit affected paths, claimed effects, and candidate impacted nodes + - should not silently mark graph claims verified or close evidence-backed gaps + +- `Verifier` + - is authoritative for check results, measurements, and invariant verification updates + - may promote or reject factual claims based on evidence + - should not silently rewrite the map just because a test passed + +- `ChurnMonitor` + - is advisory unless the coordinator chooses to turn its warnings into gates + +This lets the harness resolve conflicts by ownership instead of by whichever role wrote last. + +#### Agent subscriptions and wakeups + +The multi-agent model only helps if specialists wake up for the right reasons. + +Suggested default triggers: + +- `Mapper` + - wake on completed write turns + - wake on diffs that touch mapped files or symbols + - wake on retrieval shards becoming dirty, stale, or freshly reindexed in mapped areas + - wake on verifier evidence that invalidates or sharpens a mapped claim + - wake on explicit remap requests + - wake on resume/compaction to rebuild the frontier from checkpoints + +- `Worker` + - wake on assigned subgoals + - wake on newly exposed mechanism gaps in the target area + - wake on retrieval readiness for the target shard when mutation gates open + - wake when mutation gates open and required graph slices are fresh + +- `Verifier` + - wake on completed write turns + - wake on explicit verify requests + - wake when invariants are at risk or stale + +- `Coordinator` + - wake on user requests, specialist completion, blocked states, and churn warnings + +This keeps the mapper responsive to repo change without making it inhale the worker's whole thought process. + +#### `churn` + +Health signals about whether the work is getting sloppy. + +Suggested fields: + +- `understanding_status`: `clear`, `uncertain`, `stale` +- `diff_pressure`: low/medium/high +- `graph_freshness`: fresh/stale/missing or split freshness for architecture vs dataflow +- `unexplained_writes`: count +- `last_warning` + +This should be computed in core from observed behavior, not improvised by the UI. + +### 2. Turn-scoped intent + +Add an `EpiphanyTurnIntent` for any turn expected to mutate code or structure. + +Suggested shape: + +```text +EpiphanyTurnIntent +- turn_id: String +- subgoal_id: Option +- reason_for_edit: String +- expected_effect: String +- touched_paths: Vec +- invariants_considered: Vec +- verification_plan: Vec +``` + +This is the answer to: "why does this edit deserve to exist?" + +It should be lightweight, but it must exist before broad write actions in Epiphany mode. + +## Where the state lives in core + +### In memory + +Add Epiphany state to `core/src/state/session.rs` beside other session-scoped mutable state. + +Why there: + +- it survives across turns +- it is already the place where durable baselines and history metadata are tracked +- it is accessible from turn setup, tool handling, and event emission + +### Per turn + +Thread state should be read during `TurnContext` construction and summarized into prompt-facing context. + +The prompt should get a compact, plain-language rendering of: + +- objective +- active subgoal +- key invariants +- the relevant slice of the map +- recent evidence +- current turn intent + +That summary is derived from structured Epiphany state. The summary is not the state itself. + +## Lifecycle hooks + +These are the real patch points. + +### 0. Agent routing / subscription wakeups + +Hook point: + +- thread-level scheduler or harness coordinator +- post-turn completion path +- diff/evidence ingestion paths +- resume/compaction recovery path + +Behavior: + +- restore the agent registry and pending subscriptions on resume +- route completed write turns to mapper and verifier without sharing worker-private scratch +- route verifier evidence back to mapper when it invalidates mapped claims +- wake the worker only with the relevant graph slice, open gaps, invariants, and edit intent +- allow explicit user steering such as "remap this subsystem" or "verify this branch" to target one specialist directly + +The wakeup model should look more like typed pub/sub than one big shared diary. + +### 0.5. Retrieval indexing / invalidation + +Hook point: + +- repo-open or thread-start paths +- post-write diff ingestion +- resume/replay recovery +- explicit retrieval query path + +Behavior: + +- maintain a repo-local hybrid retrieval index per workspace or repo root +- mark touched shards dirty after completed write turns +- schedule incremental reindex work for affected shards instead of defaulting to a full rebuild +- expose freshness metadata so the harness can treat stale retrieval as a real condition, not hidden rot +- make retrieval available as a typed service to mapper, worker, and verifier with role-scoped defaults + +The retrieval subsystem should behave like a maintained utility network, not like a one-shot batch script somebody forgot to supervise. + +### 0.75. Semantic invalidation over repo changes + +Hook point: + +- generic file watcher notifications +- completed turn diff updates +- replay/startup checks against the current repo state +- retrieval shard refresh completion + +Behavior: + +- translate touched paths into impacted symbols, modules, or subsystems when possible +- mark graph nodes, mechanism answers, invariants, and retrieval shards stale in the affected scope +- reopen mechanism gaps or questions when a supporting code ref or symbol anchor moves +- emit typed freshness updates instead of silently carrying old claims forward +- let the mapper and coordinator distinguish "text changed somewhere" from "this part of the mental model is now suspect" + +This is the difference between filesystem awareness and actual epistemic hygiene. + +### 1. Thread start / resume / fork + +Hook point: + +- `core/src/thread_manager.rs` +- session initialization and rollout reconstruction + +Behavior: + +- initialize empty Epiphany state for new Epiphany-enabled threads +- recover latest durable Epiphany state from rollout when resuming/forking +- if forking, preserve the graph spine, invariants, evidence, and the latest checkpoint while clearing or rebasing scratch/frontier as needed + +This should feel similar to how `TurnContextItem` provides a durable baseline today. + +### 2. Turn construction + +Hook point: + +- `core/src/session/turn_context.rs` +- `core/src/session/turn.rs` + +Behavior: + +- derive the prompt-visible Epiphany summary from structured state +- derive the frontier slice from the graph checkpoint and active shard selection +- attach current turn intent if one exists +- mark whether the graph spine and frontier are fresh, stale, or missing + +This is where Epiphany becomes part of the model-facing harness instead of an afterthought. + +### 3. Pre-mutation gate + +Hook point: + +- file-changing tool paths in `core` +- especially shell/patch/file-change-producing operations + +Behavior in Epiphany `Enforce` mode: + +- reject or pause broad write actions when: + - no active subgoal is set + - no turn intent exists + - the relevant architecture or dataflow graph slice is marked stale for the affected subsystem + - invariants are known violated and not acknowledged + +Behavior in `Observe` mode: + +- emit warnings and churn signals but do not block + +This is the difference between "please think harder" and an actual harness. + +### 3.5. Tool-output distillation + +Hook point: + +- tool handlers after completion +- verifier-result ingestion paths +- retrieval-result ingestion paths +- turn-completion summarization paths + +Behavior: + +- convert raw tool outputs into `EpiphanyObservation` records instead of relying on transcript persistence alone +- extract affected paths, symbols, candidate graph nodes, candidate invariant impacts, and promotion candidates +- promote only the useful parts into evidence, graph updates, invariant updates, or retrieval invalidation +- discard low-value noise instead of letting every command transcript become future context +- make the promoted artifacts visible to specialist agents through typed state instead of forcing each one to reread the raw output independently + +This is the layer that keeps "I ran a command" from becoming "everyone now has to stare at 500 lines of shell exhaust forever." + +### 4. Post-tool / post-verifier updates + +Hook point: + +- tool handlers +- turn-completion path +- verifier-result ingestion paths + +Behavior: + +- append evidence when shell/tests/review/measurements produce useful results +- update invariant status when evidence proves or violates something +- update coverage and mapping status when graph claims are verified, inferred, or invalidated +- update retrieval freshness and dirty-shard state from completed writes and index refreshes +- update mechanism question status and gap status when evidence closes or exposes a seam +- demote shard completion state when a concrete question reveals that the current map still cannot explain a required transformation +- update churn state from write pressure and verification quality + +This stage should prefer promoted observations over raw transcript text whenever possible. + +The agent should not have to remember to manually carry every fact forward. The harness should help. + +### 5. Turn completion / compaction + +Hook point: + +- turn completion path +- compaction path +- rollout persistence + +Behavior: + +- persist the latest Epiphany snapshot before the turn fully settles +- persist or refresh the graph checkpoint before any compaction finalizes +- preserve the graph spine and frontier metadata across compaction +- preserve retrieval freshness summaries and dirty-shard metadata across compaction +- preserve open mechanism questions, open gaps, and shard completion state across compaction +- clear or rotate scratch as needed +- keep evidence append-only +- distinguish `PassComplete` from `DoneCriteriaMet` in mapping work +- do not mark a mapping subgoal done while required mechanism questions remain open or unresolved as gaps +- preserve enough structure that compaction does not erase the machine's understanding or lose the next re-entry point + +If compaction wipes the map, we have built a goldfish with a dashboard. + +## Persistence strategy + +Epiphany state should follow the same broad pattern as turn context: + +- durable baseline snapshots +- append-only factual records where appropriate +- replayable on resume/fork + +### Proposed rollout additions + +Add new `RolloutItem` variants in `protocol`: + +```text +RolloutItem::EpiphanyState(EpiphanyStateItem) +RolloutItem::EpiphanyEvidence(EpiphanyEvidenceItem) +RolloutItem::EpiphanyIntent(EpiphanyIntentItem) +RolloutItem::EpiphanyCheckpoint(EpiphanyGraphCheckpointItem) +``` + +Why separate variants instead of stuffing everything into one blob: + +- `EpiphanyState` is the latest baseline snapshot +- `EpiphanyEvidence` is naturally append-only +- `EpiphanyIntent` is turn-scoped and useful for replay/audit +- `EpiphanyCheckpoint` is the re-entry breadcrumb for graph work after compaction or resume + +That split keeps replay sane and lets the UI show provenance. + +Do not treat `replacement_history` or the existing memory pipeline as substitutes for this state. + +Those are useful summary channels, but they are not a safe place to keep the machine's canonical repo understanding, and they are especially not enough for specialist-agent coordination. + +`EpiphanyState` should include the agent registry and any coordinator-visible shared scratch, but not automatically dump role-private scratch into the shared baseline. + +The durable state may include retrieval summaries and freshness metadata, but the actual lexical/vector indexes should stay in the repo-local retrieval store rather than being serialized into rollout. + +### State DB mirror + +Do not start with deep SQLite mirroring. + +First make rollout replay and resume work. After that, mirror small summary fields into the state DB if we want: + +- latest objective +- active subgoal +- invariant counts by status +- graph freshness +- active frontier shard ids +- churn level + +That is enough for thread lists, badges, and search. + +## Protocol and app-server exposure + +The existing `PlanUpdate` and `TurnDiff` path is the model here: + +- core emits structured events +- app-server translates them into typed notifications +- clients render them + +### Core protocol additions + +Add event variants to `protocol/src/protocol.rs`: + +```text +EventMsg::EpiphanyStateUpdated(EpiphanyStateDeltaEvent) +EventMsg::EpiphanyEvidenceAppended(EpiphanyEvidenceEvent) +EventMsg::EpiphanyIntentUpdated(EpiphanyIntentEvent) +EventMsg::EpiphanyChurnWarning(EpiphanyChurnWarningEvent) +EventMsg::EpiphanyAgentsUpdated(EpiphanyAgentsEvent) +EventMsg::EpiphanyRetrievalUpdated(EpiphanyRetrievalEvent) +EventMsg::EpiphanyJobsUpdated(EpiphanyJobsEvent) +``` + +These are live-update signals, not the durable baseline by themselves. + +### App-server notifications + +Add matching notifications in `app-server-protocol`: + +```text +thread/epiphany/stateUpdated +thread/epiphany/evidenceAppended +turn/epiphany/intentUpdated +thread/epiphany/churnUpdated +thread/epiphany/agentsUpdated +thread/epiphany/retrievalUpdated +thread/epiphany/jobsUpdated +``` + +Use the same translation pattern already present in `bespoke_event_handling.rs`. + +### Thread read/start/resume surfaces + +Thread-scoped Epiphany data should also be available on initial load. + +Best option: + +- add an optional `epiphany` field to the app-server `Thread` type + +Fallback option: + +- add dedicated `thread/epiphany/read` and `thread/epiphany/update` methods if we want to avoid widening `Thread` immediately + +My preference is: + +- `Thread.epiphany_summary` for light thread list / thread start payloads +- a dedicated `thread/epiphany/read` for the full graph spine, frontier, evidence, agent registry, and coordinator-visible shared scratch + +If we later expose role-private scratch at all, it should be through a role-targeted inspection surface, not mixed into the default thread payload. + +For retrieval itself, prefer a dedicated typed query surface such as `thread/epiphany/retrieve` or `thread/epiphany/search` rather than shoving ad hoc search blobs into the baseline thread payload. + +That query surface should let callers specify: + +- role or query policy +- repo root or shard scope +- query mode (`ExactPath`, `Symbol`, `LexicalText`, `SemanticText`, `Hybrid`, `GraphLinked`) +- optional graph node or subsystem constraints + +And it should return: + +- hit text or symbol summary +- file path and line anchors when available +- linked graph node ids +- shard freshness and index revision metadata + +That keeps normal thread payloads from getting obese while still making Epiphany a first-class surface. + +## GUI-first implications + +Epiphany is not terminal-first. + +That should change architectural decisions, not just presentation. + +If we keep treating the linear transcript as the primary artifact and everything else as optional garnish, we will smuggle terminal bias straight back into the harness. + +The GUI-first rules should be: + +- the primary artifact is structured thread state, not the chat log +- chat is one client of Epiphany state, not the throne +- typed snapshots and typed deltas are first-class product surfaces +- user steering should update typed state directly, not only by throwing more prose into the prompt +- background work should be visible as jobs with progress, not as mysterious turns that maybe did something +- resume should restore the working scene, not merely enough text to improvise from + +### Primary artifact + +For Epiphany, the main thing the user should be working with is the thread scene: + +- graph spine and frontier +- subgoals +- invariants +- evidence +- retrieval freshness +- specialist-agent status +- current edit intent +- pending jobs and blockers + +The transcript still matters, but it becomes one inspection surface among several, not the canonical center of gravity. + +### Snapshot and delta requirements + +GUI-first clients need more than a stream of text and a prayer. + +The app-server surface should support: + +- a typed initial scene load +- typed incremental updates +- targeted reads for large surfaces such as graph shards, evidence ranges, retrieval state, and agent registry +- resumable identity for selections such as active shard ids, active question ids, and blocked job ids + +This is stricter than terminal-oriented "read thread, then infer the rest." + +### Steering and mutation requirements + +In a terminal-first system, user steering mostly means more words. + +In a GUI-first Epiphany, the user should be able to steer the machine through stateful actions such as: + +- pinning or switching the active frontier +- marking a graph claim stale +- promoting or rejecting an observation +- assigning or rerouting a subgoal +- opening or closing a mutation gate +- requesting remap, retrieval refresh, or verification on a scoped target + +Those should be typed update surfaces, not only prompt injections wearing a fake badge. + +### Background jobs and progress + +GUI users will expect indexing, remapping, verification, and specialist-agent work to be observable. + +That means the harness should expose typed job state for long-running work such as: + +- retrieval shard refresh +- remap or revalidation pass +- verification batches +- specialist-agent tasks that are materially longer than one quick turn + +Suggested job fields: + +- `id` +- `kind` +- `scope` +- `owner_role` +- `status` +- `progress_note` +- `items_processed` +- `items_total` +- `last_checkpoint_at` +- `blocking_reason` +- `linked_subgoal_id` +- `linked_graph_node_ids` + +Without that, the GUI will have to pretend that invisible background labor is a crisp user experience. Grim. + +### Approval and mutation review + +GUI-first approval should be inspectable before the user blesses a risky write. + +The approval surface should be able to show: + +- declared intent +- touched scope +- stale-map or stale-retrieval warnings +- invariant risk +- affected graph nodes or subsystems +- suggested verification plan + +That is more useful than a bare transcript-era permission bump. + +### Resume semantics + +GUI-first resume should restore the working scene: + +- active frontier and selected shard +- open mechanism questions and gaps +- dirty retrieval shards +- blocked specialists and jobs +- current graph/evidence/invariant freshness + +If resume restores only enough text to keep talking, the GUI will feel hollow even if the engine technically survived. + +### Shell de-centering + +The shell remains important, but it should stop acting like the only sensory organ in the room. + +GUI-first Epiphany should expect dedicated surfaces for: + +- retrieval and code-intel inspection +- graph browsing +- diff impact review +- evidence and observation review +- specialist-agent status and blockers + +The shell and transcript become compatibility surfaces, not the sole worldview. + +## UI responsibilities + +The GUI should render and steer this state. It should not manufacture it. + +Primary views: + +1. graph views + - render the `ArchitectureGraph` + - render the `DataflowGraph` by shard when needed + - show links from dataflow stages back to architecture components + - show frontier shard selection and checkpoint re-entry information + - let the user inspect purpose, mechanism, metaphor, mapping status, evidence links, and line-anchored code refs + +2. invariants view + - show status, evidence links, and violated/stale items prominently + +3. subgoal + scratch view + - show active subgoal + - show bounded scratchpad attached to that subgoal + +4. evidence log + - timeline of verified/rejected/partial findings + +5. edit-intent view + - current turn intent + - expected effect + - touched paths + - verification plan + +6. churn / understanding view + - warnings when the graph spine or frontier is stale + - warnings when diff pressure is high with weak evidence + +7. coverage / provenance view + - show which architecture areas and dataflow shards are unmapped, surveyed, deeply mapped, or stale + - show where claims are inferred versus verified + +8. mechanism-question view + - list the required concrete transformation questions for the active frontier + - show which questions are answered cleanly, only provisionally answered, or exposing a gap + - show the linked nodes, code refs, and evidence for each answer or missing seam + +9. agent orchestration view + - show the active specialist roles, their ownership domains, subscriptions, and current status + - show which role is waiting on which artifact or freshness condition + - make it obvious when the mapper is stale, the worker is blocked, or the verifier has unanswered evidence + +10. retrieval view + - show repo index status, shard freshness, dirty scopes, and last indexed revision + - show whether the current role is querying lexical, semantic, hybrid, or graph-linked retrieval + - surface stale-hit warnings instead of letting them hide in the plumbing + +11. jobs and blockers view + - show long-running retrieval, remap, verification, and specialist tasks as explicit jobs + - show progress, checkpoints, blocking reasons, and ownership + - make it obvious which work is background maintenance versus active foreground execution + +The UI can also provide direct editing affordances for map nodes, invariants, and subgoals. But those edits should go through typed app-server methods, not hidden local-only UI state. + +## What the UI should not do + +- It should not keep a private graph separate from core. +- It should not collapse architecture and dataflow into one ambiguous blob. +- It should not infer invariants from vibes. +- It should not decide on its own whether an edit was justified. +- It should not be the only place where evidence exists. + +Otherwise a resumed thread, headless run, or alternate client becomes blind again. + +## Recommended MVP patch order + +Do this in layers. + +### Phase 1: structured state in core + +- add Epiphany structs +- store them in `SessionState` +- include observations as the first structured distillation surface +- add rollout persistence, replay, and graph checkpointing + +No fancy UI yet. Just make the machine capable of remembering what it thinks. + +### Phase 2: prompt integration + +- derive a compact Epiphany summary during turn construction +- inject that summary into the developer/context path for Epiphany mode + +This makes the state actually matter to the agent loop. + +### Phase 3: event and protocol exposure + +- add protocol events and app-server notifications +- expose read/update methods +- make sure those methods support GUI-first scene loading and stateful steering rather than assuming chat is the only control surface + +Now the GUI can see the real state. + +### Phase 4: repo retrieval subsystem + +- add repo-local hybrid retrieval summaries to Epiphany state +- add code-intelligence summaries for symbols, references, and lightweight structural relations +- add diff-driven invalidation and incremental shard refresh +- add typed retrieval query surfaces with role-scoped defaults +- keep lexical/symbol search and semantic retrieval under one roof instead of pretending one can replace the other + +Now the agents can ask better questions without paying the full file-by-file tax every time. + +### Phase 5: semantic invalidation and distillation + +- wire watcher and diff inputs into scoped graph/retrieval/invariant invalidation +- promote raw tool outputs through structured observations instead of transcript osmosis +- make freshness and promoted facts visible to the coordinator and specialists + +This is where the harness stops trusting raw history as its only memory of what happened. + +### Phase 6: GUI reflection + +- graph view +- invariants +- evidence +- intent +- churn +- jobs and blockers +- stateful steering for frontier, observations, and scoped maintenance actions + +By this point the UI is reflecting a real engine, not a hallucinated one. + +### Phase 7: hard mutation gates + +- require intent before broad writes +- warn or block when map freshness is stale +- enforce invariant acknowledgment before risky edits + +This is where Epiphany graduates from "better instructions" to an actual harness discipline. + +### Phase 8: specialist-agent scheduling + +- add the agent registry and role ownership model +- add role-scoped private scratch +- add subscription-driven wakeups for mapper, worker, verifier, and coordinator +- keep the shared state plane authoritative while keeping role prompts narrow + +This is where Epiphany stops being one careful mind and becomes a coordinated pack. + +## Recommendation + +The next implementation note should be a patch-slice plan for Phase 1 only: + +- exact Rust types +- exact files to touch +- exact replay/persistence path +- exact minimal prompt summary format + +Do not start with the GUI. Do not start with a massive protocol expansion. Start by teaching the engine to remember its own understanding in a form the rest of the stack can trust. diff --git a/notes/fresh-workspace-handoff.md b/notes/fresh-workspace-handoff.md new file mode 100644 index 00000000..56bbfb5e --- /dev/null +++ b/notes/fresh-workspace-handoff.md @@ -0,0 +1,91 @@ +# Fresh Workspace Handoff + +## What This Repo Is + +`EpiphanyAgent` is a prototype workspace for testing whether AI coding agents behave better when their mental model is externalized into typed state: + +- `map`: canonical project/system understanding +- `scratch`: temporary local reasoning +- `evidence`: verification, acceptance, and rejection trail + +The project grew from a concrete failure mode: an agent can keep making plausible local changes while the global design quietly collapses. The intended fix is not "more prompting" in the abstract, but explicit state channels and a controller discipline around when to model, think, verify, edit, revert, and speak. + +## Current State + +The workspace has: + +- a root git repo +- a vendored clone of `openai/codex` at `vendor/codex` +- a small state CLI at `tools/epiphany_state.py` +- typed state files under `state/` +- controller action rules in `protocol/controller-actions.md` +- an eval sketch in `eval-plan.md` +- architecture rationale in `notes/architecture-rationale.md` +- a Codex patch plan in `notes/codex-epiphany-mode-plan.md` + +The latest canonical status is in `state/map.yaml`. + +## Key Conclusion From Codex Repo Research + +The best insertion point for Epiphany mode is Codex's existing collaboration-mode preset system. + +Relevant files: + +- `vendor/codex/codex-rs/protocol/src/config_types.rs` +- `vendor/codex/codex-rs/models-manager/src/collaboration_mode_presets.rs` +- `vendor/codex/codex-rs/collaboration-mode-templates/src/lib.rs` +- `vendor/codex/codex-rs/core/src/context/collaboration_mode_instructions.rs` +- `vendor/codex/codex-rs/app-server/src/codex_message_processor.rs` +- `vendor/codex/codex-rs/tui/src/collaboration_modes.rs` +- `vendor/codex/codex-rs/tui/src/chatwidget.rs` +- `vendor/codex/codex-rs/tui/src/bottom_pane/footer.rs` + +## Recommended Next Implementation + +Build Phase 1, not the full enum-backed mode: + +1. Add `codex-rs/collaboration-mode-templates/templates/epiphany.md`. +2. Export it as `EPIPHANY` from `codex-rs/collaboration-mode-templates/src/lib.rs`. +3. Add an `Epiphany` preset in `codex-rs/models-manager/src/collaboration_mode_presets.rs`. +4. Keep the preset backed by `ModeKind::Default` initially. +5. Patch the TUI to display the preset name so Epiphany is visible even though it uses Default semantics. +6. Test manually before adding a real `ModeKind::Epiphany`. + +Why: + +- avoids protocol/schema churn +- avoids SDK regeneration churn +- keeps tool gating predictable +- tests the real behavioral hypothesis first + +## What Not To Do First + +Do not immediately add `ModeKind::Epiphany` unless the user explicitly asks for the full protocol-level mode. That path touches more of the stack: protocol types, analytics, UI indicators, tool gating, generated schemas, SDK types, and snapshots. + +Do not build a large state-management subsystem before testing whether the instruction/preset path helps. A tiny helper tool may be useful later, but the first value test is the mode discipline itself. + +## Prototype Philosophy + +The useful behavior is: + +- keep a canonical map for nontrivial work +- use scratch for one bounded subgoal +- explain the current mechanism before broad edits +- prefer one hypothesis per iteration +- verify against real goals and map coherence +- revert churn that does not move the needle +- stop coding when understanding degrades + +The mode should not pretend the model has native typed channels. It should use explicit external files and write rules until a real architecture exists. + +## Useful Verification + +From the repo root: + +```powershell +& 'C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe' '.\tools\epiphany_state.py' status +Get-Content -Tail 5 '.\state\evidence.jsonl' +git status --short +``` + +The vendored Codex repo is its own clone under `vendor/codex`. Treat it as source material for now unless the user asks to patch it. diff --git a/protocol/controller-actions.md b/protocol/controller-actions.md new file mode 100644 index 00000000..6686135f --- /dev/null +++ b/protocol/controller-actions.md @@ -0,0 +1,119 @@ +# Controller Actions + +The controller chooses exactly one primary action per step. This is meant to reduce the classic failure mode where the model is "thinking", rewriting the plan, editing files, and rationalizing the edits all at once. + +## Actions + +### `update_map` + +Purpose: +Write stable understanding into `state/map.yaml`. + +Allowed writes: +- objectives +- constraints +- invariants +- accepted design +- rejected paths +- current status + +Rules: +- Do not copy raw scratch into the map. +- Do not write guesses as settled architecture. +- If the map changes, say why. + +### `think_subgoal` + +Purpose: +Use `state/scratch.md` for one bounded problem only. + +Allowed writes: +- temporary decomposition +- open questions +- candidate checks +- local reasoning that may be wrong + +Rules: +- Scope it to one subgoal. +- Prefer deletion or summarization over accumulation. +- Scratch is disposable by design. + +### `propose_patch` + +Purpose: +Make one bounded code or document change tied to a clear hypothesis. + +Allowed writes: +- workspace files +- `state/branches.json` + +Rules: +- Name the hypothesis. +- Keep the patch narrow enough to verify. +- Do not silently redefine the map through code. + +### `run_verify` + +Purpose: +Check whether the proposed change actually improved the thing we care about. + +Allowed writes: +- `state/evidence.jsonl` +- `state/branches.json` + +Checks may include: +- tests +- lints +- benchmark deltas +- invariant checks +- coherence review against `state/map.yaml` +- simplicity review + +### `compare_branches` + +Purpose: +Choose between competing changes or kill all of them. + +Allowed writes: +- `state/branches.json` +- `state/evidence.jsonl` + +Rules: +- Prefer the smallest branch that clearly satisfies the objective. +- If two branches both feel dubious, reject both. + +### `backtrack` + +Purpose: +Abandon a bad line of attack instead of decorating it further. + +Allowed writes: +- `state/branches.json` +- `state/evidence.jsonl` +- `state/map.yaml` + +Rules: +- Record why the branch was rejected. +- Preserve lessons that belong in the map. +- Do not keep dead machinery out of sentiment. + +### `speak` + +Purpose: +Produce user-visible output. + +Rules: +- Speak after verification when possible. +- If speaking before verification, be explicit that the result is provisional. + +## Promotion Rule + +Information moves from scratch to map only if it survives verification, comparison, or repeated re-use without contradiction. + +## Stop Conditions + +Stop implementation and switch back to diagnosis when: +- the diff is growing and understanding is shrinking +- tests are green but the map and code disagree +- the agent cannot explain the data flow stage by stage +- new abstraction is compensating for confusion instead of reducing it diff --git a/state/branches.json b/state/branches.json new file mode 100644 index 00000000..b472d5cc --- /dev/null +++ b/state/branches.json @@ -0,0 +1,15 @@ +{ + "branches": [ + { + "id": "main", + "hypothesis": "External typed state will reduce drift relative to plain prompting on iterative coding tasks.", + "status": "active", + "artifacts": [ + "state/map.yaml", + "state/scratch.md", + "state/evidence.jsonl" + ], + "notes": "Initial scaffold plus tiny CLI. No evaluated task yet." + } + ] +} diff --git a/state/evidence.jsonl b/state/evidence.jsonl new file mode 100644 index 00000000..041e209f --- /dev/null +++ b/state/evidence.jsonl @@ -0,0 +1,26 @@ +{"ts":"2026-04-23","type":"scaffold","status":"ok","note":"Initialized EpiphanyAgent workspace with typed state files, controller action rules, and an initial experiment map."} +{"ts": "2026-04-23T03:42:27+00:00", "type": "bootstrap", "status": "ok", "note": "Initialized repo, added typed-state scaffold, and verified the tiny CLI runs with bundled Python."} +{"ts": "2026-04-23T03:48:53+00:00", "type": "research", "status": "ok", "note": "Cloned openai/codex, identified collaboration-mode presets as the clean insertion point, and wrote a concrete patch plan note for preset-backed versus enum-backed Epiphany mode."} +{"ts": "2026-04-23T15:28:33+00:00", "type": "handoff", "status": "ok", "note": "Added root AGENTS.md and notes/fresh-workspace-handoff.md so a fresh workspace can continue from repo-local context."} +{"ts": "2026-04-23T15:29:25+00:00", "type": "handoff", "status": "ok", "note": "Added architecture-rationale.md with the typed-state architecture, cheap prototype strategy, eval shape, and DeepSeek research context."} +{"ts": "2026-04-23T15:47:17+00:00", "type": "research", "status": "ok", "note": "Orientation confirmed the preset-backed Codex insertion point and recorded that Phase 1 TUI selection/labeling must compare preset identity, not only ModeKind, because Epiphany rides on Default."} +{"ts": "2026-04-23T15:48:11+00:00", "type": "research", "status": "ok", "note": "App-server collaboration mode list preserves preset name but drops developer_instructions; turn-start fallback currently matches only ModeKind, so duplicate Default-backed presets need explicit preset identity or explicit instructions in non-TUI flows."} +{"ts": "2026-04-23T15:55:06+00:00", "type": "research", "status": "ok", "note": "Cloned stellarlinkco/codex to vendor/opencodex and ymichael/open-codex to vendor/open-codex-cli. The exact-name open-codex fork is terminal-only; vendor/opencodex is the GUI-relevant Rust/Web UI fork served by codex serve."} +{"ts": "2026-04-23T16:17:47+00:00", "type": "research", "status": "ok", "note": "Checked OpenCodex Anthropic/Claude GUI claims. web README and MODEL_OPTIONS mention claude/sonnet/opus, but AgentSelector only exposes codex and serve backend rejects any spawned agent other than codex; Anthropic support appears limited to model_provider config, not GUI agent selection."} +{"ts": "2026-04-23T16:26:11+00:00", "type": "research", "status": "rejected", "note": "Audited imdigitalashish/OpenCodex: it is a Tauri desktop wrapper around vendored opencode, not a Codex fork; the core coding-agent path depends on an external opencode binary and non-interactive CLI mode."} +{"ts": "2026-04-23T16:26:11+00:00", "type": "verification", "status": "partial", "note": "Frontend npm ci and npm run build succeeded for imdigitalashish/OpenCodex; npm audit reports 3 high-severity advisories. Go and Cargo toolchains are not on PATH, so opencode and Tauri backend were not compiled."} +{"ts": "2026-04-23T16:26:11+00:00", "type": "research", "status": "rejected", "note": "imdigitalashish/OpenCodex GUI model IDs are inconsistent with vendored opencode SupportedModels: the GUI advertises/writes Anthropic API IDs like claude-sonnet-4-5-20250929 while opencode accepts symbolic IDs such as claude-4-sonnet."} +{"ts": "2026-04-23T16:31:48+00:00", "type": "decision", "status": "ok", "note": "Removed third-party OpenCodex comparison clones from vendor/opencodex, vendor/open-codex-cli, and vendor/opencodex-imdigitalashish. Decision: build a dedicated GUI wrapper around customized vendored Codex instead of trusting unaudited forks."} +{"ts": "2026-04-23T18:02:11+01:00", "type": "research", "status": "ok", "note": "Mapped vendored Codex end-to-end dataflow in notes/codex-repository-algorithmic-map.md: CLI/TUI choose a surface, app-server is the transport seam, core/session owns prompt assembly and tool execution, and rollouts plus SQLite provide durable thread history and metadata."} +{"ts": "2026-04-23T18:16:47+01:00", "type": "design", "status": "ok", "note": "Defined Epiphany as core-owned structured thread state in notes/epiphany-core-harness-surfaces.md. The spec anchors on existing TurnContextItem and PlanUpdate/TurnDiff patterns: persist Epiphany baselines in rollouts, stream typed updates through app-server, and let the GUI render graphs, invariants, evidence, intent, and churn without becoming the source of truth."} +{"ts": "2026-04-23T18:24:36+01:00", "type": "design", "status": "ok", "note": "Refined the Epiphany harness spec so the machine map is two linked graphs rather than one: an ArchitectureGraph and a DataflowGraph, joined by explicit cross-links and grounded by line-anchored code refs. Replaced the vague node description field with purpose, mechanism, and metaphor so the internal representation carries richer language for both model alignment and human inspection."} +{"ts": "2026-04-23T18:36:52+01:00", "type": "design", "status": "ok", "note": "Updated the Epiphany harness spec to handle repeated compactions during large repo mapping. Added an explicit spine/frontier/scratch model, graph checkpoints as re-entry breadcrumbs, dataflow sharding for large repositories, coverage/provenance status on graph elements, and more resilient code refs with symbol and last-seen revision metadata so resume/compaction does not collapse the map back into prose mush."} +{"ts": "2026-04-23T19:41:43+00:00", "type": "research", "status": "ok", "note": "Inspected Codex session 019db84b-7edd-7843-abc9-76ad2c402bb4 rollout. Failure mode was repeated premature closure on mapping work: assistant repeatedly framed local pass completion as done while user had to restate bar (missing leaves, control flow not file map, hand-waved mechanisms). Four context compactions preserved replacement_history but likely weakened the persistent evaluation bar. Quality improved only when user asked a concrete mechanism question (planetary gravity to visible nebula terrain), which forced an honest admission of a missing seam. Implication for Epiphany: mapping tasks need explicit done criteria based on answering concrete transformation questions, plus durable coverage/frontier/gap state across compactions."} +{"ts": "2026-04-23T19:54:42+00:00", "type": "decision", "status": "ok", "note": "Patched epiphany-core-harness-surfaces.md to make mapping done-criteria explicit. Added durable MechanismQuestion and MechanismGap concepts, separate MappingCompletionState from coverage, preserved active questions/gaps through frontier and checkpoint, and required concrete transformation-question answerability before marking mapping work done. Updated state/map.yaml to carry the same rule into accepted design and next action."} +{"ts": "2026-04-23T20:01:26+00:00", "type": "decision", "status": "ok", "note": "Documented a multi-agent Epiphany harness model in epiphany-core-harness-surfaces.md. Added supervising-harness plus specialist-agent framing, agent registry and role metadata, ownership boundaries for mapper/worker/verifier/coordinator, private-by-default role scratch, subscription-triggered wakeups, protocol/UI surfaces for agent status, and phased specialist-agent scheduling. Updated state/map.yaml to record the shared-state-plane plus role-ownership design and to defer the patch-slice plan until after this broader design pass."} +{"ts": "2026-04-23T20:08:53+00:00", "type": "decision", "status": "ok", "note": "Extended epiphany-core-harness-surfaces.md with a first-class repository retrieval subsystem. Added hybrid retrieval principle (exact path/symbol/lexical plus semantic chunk and graph-linked lookup), repo-local sharded index summaries, diff-driven freshness/invalidation, role-scoped retrieval policies, retrieval lifecycle hooks, protocol/app-server update events, typed retrieval query surface guidance, UI retrieval view, and phased rollout positioning before GUI reflection and specialist-agent scheduling. Updated state/map.yaml to treat retrieval as a core design decision and to keep the next action on broader harness design before a Phase 1 patch-slice plan."} +{"ts": "2026-04-23T20:13:55+00:00", "type": "research", "status": "ok", "note": "Scanned vendored Codex for additional harness blind spots before patch planning. Findings: repo retrieval is still fuzzy-file/CLI-centric; file watchers are wired mainly to skills cache invalidation and fs notifications, not repo-understanding freshness; subagents inherit config and tools but do not share a typed collaboration state plane; the startup memories pipeline explicitly excludes sub-agent sessions; most tool outputs still persist into history as text payloads; and turn diff tracking exposes aggregated unified diff, not impacted symbols/subsystems or semantic invalidation. Implication: Epiphany should reuse watcher/diff/subagent substrate but add typed distillation, freshness tracking, and shared repo knowledge above it."} +{"ts": "2026-04-23T20:23:25+00:00", "type": "decision", "status": "ok", "note": "Patched notes/epiphany-core-harness-surfaces.md after the Codex blind-spot scan. The spec now explicitly reuses existing Codex substrate (watchers, diff stream, subagent lifecycle, memory pipeline) while naming the missing semantic layer above it: code-intelligence-aware hybrid retrieval, semantic invalidation over repo changes, a structured observation/distillation layer between raw tool output and durable state, and a shared knowledge plane for specialist agents that is not replacement_history or prompt osmosis. Synced state/map.yaml with the same observation-layer design."} +{"ts": "2026-04-23T20:33:19+00:00", "type": "decision", "status": "ok", "note": "Revised notes/codex-repository-algorithmic-map.md so the Codex machine map reflects both the main turn loop and the surrounding side channels/limitations. Added watcher and memory actors, extended the top-level flow with compaction and watcher/diff side channels, documented that tool outputs are still mostly transcript-shaped, and added explicit sections for file watching, textual diff tracking, message/config-based subagent coordination, startup memory summarization, and compaction as real dataflow. Conclusion: Codex has useful substrate around the loop, but still lacks a canonical typed repo-understanding layer above it."} +{"ts": "2026-04-23T20:38:04+00:00", "type": "decision", "status": "ok", "note": "Patched epiphany-core-harness-surfaces.md with explicit GUI-first implications. The spec now states that structured thread state, not the transcript, is the primary artifact; chat is one client of that state; GUI steering should use typed updates; resume must restore a working scene; and long-running retrieval/remap/verification/specialist work should surface as typed jobs with progress. Added EpiphanyJobState to the thread model plus EpiphanyJobsUpdated protocol/app-server events, and expanded UI responsibilities and phase notes accordingly. Synced state/map.yaml with the same GUI-first design correction."} +{"ts": "2026-04-23T20:56:56+00:00", "type": "decision", "status": "ok", "note": "Replaced notes/codex-epiphany-mode-plan.md with the actionable Phase 1 implementation plan. The repo now records an internal/dev-usable first slice focused on minimal Epiphany protocol types, SessionState storage, one Epiphany rollout snapshot per real user turn, replay through rollout reconstruction, exhaustive RolloutItem compatibility updates, and persistence/replay tests before prompt integration, presets, protocol notifications, or GUI work. Synced state/map.yaml so the next action is implementation rather than more design passes."} diff --git a/state/map.yaml b/state/map.yaml new file mode 100644 index 00000000..eacf2b04 --- /dev/null +++ b/state/map.yaml @@ -0,0 +1,90 @@ +experiment: + name: EpiphanyAgent + status: scaffolded + phase: manual_external_typed_state + premise: > + Test whether separating map, scratch, and output reduces architectural drift + for coding tasks when using a strong general model. + +objective: + primary: > + Build a model-agnostic protocol that Codex can use in this workspace and that + can later be evaluated on open models without requiring frontier-scale + training. + immediate: + - Finish the initial workspace scaffold. + - Keep controller actions and write rules explicit. + - Ground the Codex epiphany-mode idea against the real open-source codebase. + - Use the first minimal runner or manual operating loop on a small real task. + - Try the protocol on a small real task. + +constraints: + - No frontier-scale training budget. + - Start with external state and prompting. + - Keep the protocol light enough to use during normal work. + - Prefer evidence over vibes. + +invariants: + - The map is canonical and slow-changing. + - Scratch is temporary and allowed to be wrong. + - Output must not silently redefine the map. + - Accepted changes should have recorded evidence. + - Rejected paths stay recorded. + +data_flow: + observe: + - task request + - repository state + - verifier results + process: + - update map only with stable understanding + - think about one subgoal at a time + - propose one bounded change + - verify against task goals and invariants + - record evidence + - commit or backtrack + +active_subgoals: + - Implement the Phase 1 internal/dev-usable Epiphany core state slice in vendored Codex: protocol types, SessionState storage, rollout persistence, replay, and compatibility tests. + - Use the tiny CLI and typed state files on a small real task. + - Decide whether the next executable artifact should be a patch runner or a verifier helper. + - Run the protocol on a small real task. + +accepted_design: + - external typed state files + - bounded controller actions + - verifier-guided acceptance + - manual first, automation later + - patch Codex via collaboration-mode presets before touching protocol enums + - make Phase 1 preset selection, labeling, and instruction lookup preset-name-aware where needed because Epiphany initially rides on ModeKind::Default + - for a GUI-centered experiment, build a dedicated wrapper around customized vendored Codex instead of adopting third-party OpenCodex forks + - treat Epiphany as core-owned structured thread state in Codex, with the GUI as a reflection and steering layer rather than the source of truth + - represent the Epiphany machine map as two linked graphs (architecture and dataflow) with rich natural-language node fields such as purpose, mechanism, and metaphor, plus line-anchored code references + - make the graph model compaction-aware: preserve a durable spine, checkpoint and reselect a frontier, and allow scratch to be rewritten or discarded + - for mapping work, distinguish a completed pass from actual understanding: require representative concrete transformation questions, record unresolved mechanism gaps durably, and carry those gates across compaction + - treat Epiphany as a supervising harness over specialist agents sharing a typed state plane, with private role-scoped scratch by default and explicit ownership for map, evidence, and code-change domains + - add a repo-local hybrid retrieval subsystem to Epiphany: combine exact path/symbol/lexical search with semantic chunk retrieval and graph-linked lookup, shard it by repo/subsystem, and track freshness from diffs + - treat Codex's existing file watcher, subagent lifecycle, memory pipeline, and diff stream as reusable substrate, but add the missing semantic layer above them: typed repo understanding, freshness invalidation, structured tool-output distillation, and shared state across specialist agents + - insert a structured observation layer between raw tool output and durable Epiphany state so evidence, invalidation, and graph updates are promoted from typed distillation rather than transcript osmosis + - design Epiphany as GUI-first rather than transcript-first: the primary artifact is typed thread state, chat is one client of that state, user steering should go through typed updates, and long-running retrieval/remap/verification work should surface as explicit observable jobs + - preserve handoff context in repo-local AGENTS and notes files + - keep a durable repository-level algorithmic map of vendored Codex so GUI and Epiphany changes hook into the existing app-server/core boundary instead of reimplementing the turn loop + - keep the Codex algorithmic map honest about auxiliary side channels and limitations: file watching, unified diffs, compaction, subagent messaging, and memory summarization are real substrate, but they do not by themselves amount to semantic repo understanding + +rejected_paths: + - name: pinned_map_only + reason: still one token stream with no typed write semantics + - name: train_new_base_model_first + reason: too expensive for a first falsifiable experiment + - name: exact_name_open_codex_cli_as_gui_surface + reason: ymichael/open-codex is terminal-focused and has no Web UI or desktop surface + - name: read_opencodex_web_readme_claude_models_as_current_gui_support + reason: the Web UI has stale/aspirational Claude/Gemini model definitions, but the new-session selector only exposes Codex and the serve backend rejects non-Codex agents + - name: imdigitalashish_opencodex_as_primary_gui_surface + reason: the repo is a Tauri wrapper around an archived/vendored opencode CLI, has mismatched GUI/API model IDs versus the CLI model registry, runs the non-interactive tool path with broad auto-approval, and commits generated/binary/local-state artifacts + - name: third_party_opencodex_forks_as_gui_base + reason: audited candidates were either terminal-only, stale/aspirational, or incoherent wrappers; the comparison clones were removed from vendor to avoid building on untrusted drift + +current_status: + summary: Workspace initialized with protocol files, typed state, a tiny CLI, a git repo, fresh-workspace handoff notes, architecture rationale, a Codex algorithmic map note, an Epiphany core-harness surface spec, and an actionable Phase 1 implementation plan against vendored Codex. Third-party OpenCodex comparison clones were audited and removed. + next_action: Implement the Phase 1 internal/dev-usable Epiphany core state slice: add the minimal protocol types, store Epiphany state in SessionState, persist one rollout snapshot per real user turn, rehydrate it through rollout reconstruction, patch exhaustive RolloutItem matches, and add persistence/replay compatibility tests before touching prompts, presets, or GUI surfaces. diff --git a/state/scratch.md b/state/scratch.md new file mode 100644 index 00000000..40969c75 --- /dev/null +++ b/state/scratch.md @@ -0,0 +1,26 @@ +# Scratch + +This file is intentionally disposable. + +## Current Subgoal + +- Confirm the scaffold is coherent enough to use. +- Decide whether to patch vendored Codex with the lightweight Epiphany preset now. +- Decide whether the next executable artifact should be a patch runner or a verifier helper. + +## Working Notes + +- The protocol should be usable by any capable model because most of the structure lives outside the model. +- The first test should be small enough to run repeatedly without becoming its own research program. +- If the protocol becomes too ceremonial, it will lose to plain prompting for the most boring reason imaginable: friction. +- The current CLI is intentionally small. It exists to keep state honest, not to impersonate a full agent runtime. +- The Codex repo already has a cleaner-than-expected collaboration-mode preset seam, so a preset-backed Epiphany mode is a tractable first implementation. + +## Open Questions + +- What is the smallest useful verifier stack? +- Which coding task will expose drift quickly? +- How much branch management is enough before it turns into paperwork? +- Should the first in-Codex implementation be instructions-only, or should it also add a tiny state helper tool? + +Do not promote anything from here into the map unless it survives verification or repeated reuse without contradiction. diff --git a/tools/epiphany_state.py b/tools/epiphany_state.py new file mode 100644 index 00000000..cc6fbb5a --- /dev/null +++ b/tools/epiphany_state.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +import argparse +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parents[1] +STATE_DIR = ROOT / "state" +MAP_PATH = STATE_DIR / "map.yaml" +BRANCHES_PATH = STATE_DIR / "branches.json" +EVIDENCE_PATH = STATE_DIR / "evidence.jsonl" + + +def read_text(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +def write_text(path: Path, content: str) -> None: + path.write_text(content, encoding="utf-8") + + +def load_branches() -> dict[str, Any]: + return json.loads(read_text(BRANCHES_PATH)) + + +def save_branches(data: dict[str, Any]) -> None: + write_text(BRANCHES_PATH, json.dumps(data, indent=2) + "\n") + + +def append_evidence(record: dict[str, Any]) -> None: + with EVIDENCE_PATH.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(record, ensure_ascii=True) + "\n") + + +def utc_stamp() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + +def extract_map_field(name: str) -> str | None: + prefix = f" {name}:" + for line in read_text(MAP_PATH).splitlines(): + if line.startswith(prefix): + return line.split(":", 1)[1].strip() + return None + + +def extract_active_subgoals() -> list[str]: + lines = read_text(MAP_PATH).splitlines() + results: list[str] = [] + in_section = False + for line in lines: + if not in_section: + if line.strip() == "active_subgoals:": + in_section = True + continue + if line.startswith(" - "): + results.append(line[4:].strip()) + continue + if line and not line.startswith(" "): + break + return results + + +def cmd_status(_: argparse.Namespace) -> int: + branches = load_branches().get("branches", []) + active = [branch for branch in branches if branch.get("status") == "active"] + summary = extract_map_field("summary") or "(missing)" + next_action = extract_map_field("next_action") or "(missing)" + subgoals = extract_active_subgoals() + + print(f"Workspace: {ROOT}") + print(f"Summary: {summary}") + print(f"Next action: {next_action}") + print(f"Active branches: {len(active)} / {len(branches)}") + if subgoals: + print("Active subgoals:") + for item in subgoals: + print(f"- {item}") + return 0 + + +def cmd_add_evidence(args: argparse.Namespace) -> int: + record: dict[str, Any] = { + "ts": utc_stamp(), + "type": args.type, + "status": args.status, + "note": args.note, + } + if args.branch: + record["branch"] = args.branch + append_evidence(record) + print("Appended evidence record.") + return 0 + + +def cmd_add_branch(args: argparse.Namespace) -> int: + data = load_branches() + branches = data.setdefault("branches", []) + if any(branch.get("id") == args.id for branch in branches): + raise SystemExit(f"Branch '{args.id}' already exists.") + branches.append( + { + "id": args.id, + "hypothesis": args.hypothesis, + "status": "active", + "artifacts": args.artifact or [], + "notes": args.note or "", + } + ) + save_branches(data) + print(f"Added branch '{args.id}'.") + return 0 + + +def cmd_close_branch(args: argparse.Namespace) -> int: + data = load_branches() + for branch in data.get("branches", []): + if branch.get("id") != args.id: + continue + branch["status"] = args.status + if args.note: + branch["notes"] = args.note + save_branches(data) + print(f"Updated branch '{args.id}' to status '{args.status}'.") + return 0 + raise SystemExit(f"Branch '{args.id}' was not found.") + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Inspect and update EpiphanyAgent state files." + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + status_parser = subparsers.add_parser("status", help="Show a compact state summary.") + status_parser.set_defaults(func=cmd_status) + + evidence_parser = subparsers.add_parser( + "add-evidence", help="Append one JSONL evidence record." + ) + evidence_parser.add_argument("--type", required=True) + evidence_parser.add_argument("--status", required=True) + evidence_parser.add_argument("--note", required=True) + evidence_parser.add_argument("--branch") + evidence_parser.set_defaults(func=cmd_add_evidence) + + add_branch_parser = subparsers.add_parser( + "add-branch", help="Create a new active branch entry." + ) + add_branch_parser.add_argument("--id", required=True) + add_branch_parser.add_argument("--hypothesis", required=True) + add_branch_parser.add_argument("--artifact", action="append") + add_branch_parser.add_argument("--note") + add_branch_parser.set_defaults(func=cmd_add_branch) + + close_branch_parser = subparsers.add_parser( + "close-branch", help="Close an existing branch." + ) + close_branch_parser.add_argument("--id", required=True) + close_branch_parser.add_argument( + "--status", required=True, choices=["accepted", "rejected", "archived"] + ) + close_branch_parser.add_argument("--note") + close_branch_parser.set_defaults(func=cmd_close_branch) + + return parser + + +def main() -> int: + parser = build_parser() + args = parser.parse_args() + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/codex b/vendor/codex new file mode 160000 index 00000000..e8ba912f --- /dev/null +++ b/vendor/codex @@ -0,0 +1 @@ +Subproject commit e8ba912fcc9971d899808870821b83614ced7508