From c744a9367f5d349e712dc9d0c167348ef8bf3554 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 14 May 2026 08:41:11 +0100 Subject: [PATCH] Extract token usage projection to bridge --- epiphany-codex-bridge/src/lib.rs | 1 + epiphany-codex-bridge/src/token_usage.rs | 77 ++++++++++++++++++ notes/fresh-workspace-handoff.md | 6 ++ state/ledgers.msgpack | Bin 115671 -> 115933 bytes state/map.yaml | 13 ++- vendor/codex/codex-rs/Cargo.lock | 2 + .../token_usage_replay.rs | 77 +----------------- 7 files changed, 95 insertions(+), 81 deletions(-) create mode 100644 epiphany-codex-bridge/src/token_usage.rs diff --git a/epiphany-codex-bridge/src/lib.rs b/epiphany-codex-bridge/src/lib.rs index 28760345..35ca6ebc 100644 --- a/epiphany-codex-bridge/src/lib.rs +++ b/epiphany-codex-bridge/src/lib.rs @@ -12,6 +12,7 @@ pub mod retrieve; pub mod runtime_results; pub mod scene; pub mod state; +pub mod token_usage; pub mod view; pub use epiphany_openai_adapter as openai_adapter; diff --git a/epiphany-codex-bridge/src/token_usage.rs b/epiphany-codex-bridge/src/token_usage.rs new file mode 100644 index 00000000..c79adaec --- /dev/null +++ b/epiphany-codex-bridge/src/token_usage.rs @@ -0,0 +1,77 @@ +use codex_app_server_protocol::Thread; +use codex_app_server_protocol::ThreadHistoryBuilder; +use codex_app_server_protocol::TurnStatus; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::RolloutItem; +use codex_protocol::protocol::TokenUsageInfo; + +pub fn latest_token_usage_info_from_rollout_items( + rollout_items: &[RolloutItem], +) -> Option { + rollout_items.iter().rev().find_map(|item| match item { + RolloutItem::EventMsg(EventMsg::TokenCount(event)) => event.info.clone(), + _ => None, + }) +} + +/// Identifies the turn that was active when a `TokenCount` record appeared. +/// +/// The id is preferred when it still appears in the rebuilt thread. The position +/// is a fallback for histories whose implicit turn ids are regenerated during +/// reconstruction. +struct TokenUsageTurnOwner { + id: String, + position: Option, +} + +pub fn latest_token_usage_turn_id_from_rollout_items( + rollout_items: &[RolloutItem], + thread: &Thread, +) -> Option { + let owner = latest_token_usage_turn_owner_from_rollout_items(rollout_items)?; + if thread.turns.iter().any(|turn| turn.id == owner.id) { + return Some(owner.id); + } + owner + .position + .and_then(|position| thread.turns.get(position)) + .map(|turn| turn.id.clone()) +} + +fn latest_token_usage_turn_owner_from_rollout_items( + rollout_items: &[RolloutItem], +) -> Option { + let mut builder = ThreadHistoryBuilder::new(); + let mut token_usage_turn_owner = None; + + for item in rollout_items { + if matches!(item, RolloutItem::EventMsg(EventMsg::TokenCount(_))) { + token_usage_turn_owner = + builder + .active_turn_snapshot() + .map(|turn| TokenUsageTurnOwner { + id: turn.id, + position: builder.active_turn_position(), + }); + } + builder.handle_rollout_item(item); + } + + token_usage_turn_owner +} + +/// Chooses a fallback turn id that should own a replayed token usage update. +/// +/// Normal replay derives the owner from the rollout position of the latest +/// `TokenCount` event. This fallback only preserves a stable wire shape for +/// unusual histories where that rollout information cannot be read. +pub fn latest_token_usage_turn_id(thread: &Thread) -> String { + thread + .turns + .iter() + .rev() + .find(|turn| matches!(turn.status, TurnStatus::Completed | TurnStatus::Failed)) + .or_else(|| thread.turns.last()) + .map(|turn| turn.id.clone()) + .unwrap_or_default() +} diff --git a/notes/fresh-workspace-handoff.md b/notes/fresh-workspace-handoff.md index 389381f4..636688b0 100644 --- a/notes/fresh-workspace-handoff.md +++ b/notes/fresh-workspace-handoff.md @@ -781,6 +781,12 @@ bridge now: app-server passes `EpiphanyInvalidationSnapshot` to `epiphany_freshness_watcher_snapshot` instead of defining that mapping beside `codex_message_processor.rs`. +Token-usage rollout projection has also left the app-server helper. +`epiphany-codex-bridge::token_usage` owns the pure logic that finds the latest +persisted token-usage snapshot and maps it back to the rebuilt turn owner. +`token_usage_replay.rs` is now just the host shell: read rollout items, choose +whether to replay, and send a connection-scoped JSON-RPC notification. + The MCP config path is now product-quarantined. `Config::to_mcp_config` keeps user-declared MCP servers but no longer accepts a `PluginsManager`, folds plugin-provided MCP servers into the runtime, enables Codex apps, enables skill diff --git a/state/ledgers.msgpack b/state/ledgers.msgpack index 107157d89255a0f2b7fb52096305bfd83b580c2d..b12826e77d1cc8f1b6fad72bcc8acac0f9038140 100644 GIT binary patch delta 230 zcmccK&VIL(eS$L6k;NNTr!g`bHSb~EzK4;K_0aURgN(u~CI(g}(*rj!YDwQb?3-Vf znxasWpPibgTUwl$o~lrkpOcecS~C5QE~8q#l@(Yj9xP?8P?nmPl3xVWkXTTlTbx={ zmRbbS3sDeXlv` or `` prompt compatibility, memory usage classification, snapshot normalization, or cleanup paths. Remaining product-shaped skill residue is also cut from memory templates, app-server docs/tests/warnings, TUI tooltips/placeholders/warnings/comments, app-server v1 docs, agent-tool schema text, stale `UserInput::Skill` test arms, and obsolete feature flags/schema entries. App-instruction prompt compatibility is cut too: no `APPS_INSTRUCTIONS` protocol constants, no `` snapshot canonicalization, and MCP fixture tests now use neutral `demo_mcp` / `calendar_mcp` names instead of `codex_apps` / `/api/codex/apps` branding. The TUI app connector product path is also deleted: no `/apps` slash command, app-link view, connector AppEvent variants, app connector cache/popup/refresh loop, app mention insertion, `app://` mention codec support, stale app-popup tests, or TUI browser-opening shim remains. The `app/list` route and `app/list/updated` notification are gone from app-server protocol, generated schemas, app-server dispatch, test helpers, and docs; `AppInfo`/app-list metadata schema structs are deleted; the orphaned `codex-connectors` crate and `core::connectors` shim are deleted from the workspace; Codex app analytics no longer tracks `codex_app_mentioned` / `codex_app_used` facts, events, reducer arms, client APIs, or dedupe cache state; `[apps.*]`, `Feature::Apps`, `include_apps_instructions`, app config/read schemas, app managed requirements, app prompt tests, and app file-MCP tests are gone; `Feature::Plugins`, `Feature::RemotePlugin`, `plugin://` structured mentions, plugin instruction tags, and stale plugin popup snapshots are gone too; `[plugins]` / `[marketplaces]`, `PluginConfig`, `MarketplaceConfig`, marketplace edit helpers, stale plugin/external-agent README route claims, and generated schema entries are removed from config; and the bundled `codex-skills` sample-skill crate plus embedded imagegen/openai-docs/plugin-creator/skill-creator/skill-installer payloads are deleted from the workspace. The vendored `vendor/codex/sdk` subtree is deleted as non-spine app-server client/release scaffolding, including stale generated Python SDK route types for removed product surfaces; the root pnpm workspace no longer lists `sdk/typescript`. Live MCP tool approval is named `McpToolApproval`, not `AppToolApproval`, and remains under MCP server config; live structured user-input mentions are MCP-shaped only. `codex_message_processor.rs` is about 9,482 lines; root `message_processor.rs` is about 1,292 lines after the external-agent/config-product cuts; `epiphany_read_routes.rs` is about 579 lines after the full view and read-response extractions; `epiphany_mutation_routes.rs` is about 694 lines after route-load/notification helper cuts and product-surface starvation; the old vendored `epiphany_state_helpers.rs` rollout-state loader is deleted and its remaining function lives in `epiphany-codex-bridge::state`; and `codex_thread.rs` is about 1,425 lines after state-update law moved native. `epiphany-openai-adapter` defines CultCache `DatabaseEntry` request/event/receipt/auth-status documents with no Codex app-server or generic JSON dependency. `epiphany-openai-codex-spine` owns the native OpenAI model transport path: it exposes `default_codex_home` and `auth_manager`, projects `epiphany-openai-auth-spine` `AuthManager` / `CodexAuth` into typed `EpiphanyOpenAiAdapterStatus`, builds a local serializable OpenAI Responses request body instead of Codex protocol request/item/config types, opens HTTP Responses streams through `codex-client`, parses Responses SSE frames into typed stream events and receipts, and no longer directly or transitively depends on `codex-login`, `codex-app-server-protocol`, `codex-protocol`, `codex-api`, `codex-model-provider`, or `codex-model-provider-info`. Native auth currently covers env/file/keyring/auto credentials plus ChatGPT token refresh; external bearer command auth and agent identity remain explicit compatibility gaps. `epiphany-openai-runtime` no longer depends directly on `codex-login`; it writes OpenAI status/request/stream-event/receipt records plus runtime sessions/jobs/results/events into `epiphany-core::runtime_spine_cache`, can emit the assistant text artifact for legacy result consumers, is now the default executor for startup-only repo birth specialists, and now has `run-worker` to load a durable `EpiphanyRuntimeWorkerLaunchRequest`, call the typed OpenAI runtime through the spine, persist typed `EpiphanyRuntimeRoleWorkerResult` / `EpiphanyRuntimeReorientWorkerResult` documents, and complete the original heartbeat/specialist runtime job without Codex worker execution; `codex exec` survives only as explicit fallback in repo-birth startup. `epiphany-mvp-coordinator` now invokes `epiphany-openai-runtime run-worker` for launched runtime job ids and no longer creates a second coordinator-owned shadow runtime job/result as a compensator. `epiphany-core::plan_runtime_spine_heartbeat_launch` validates launch contracts/conflicts and projects durable job bindings/runtime links; `epiphany-core::open_runtime_spine_heartbeat_job` owns the initialize-runtime/ensure-session/create-job sequence and now persists `EpiphanyRuntimeWorkerLaunchRequest` CultCache records keyed by runtime job id, with indexed kind/contract fields plus MessagePack-encoded typed launch documents instead of discarding work intent after queue receipt creation; `roleResult` and `reorientResult` now require typed worker result documents; generic job summaries are lifecycle receipts, not reviewable findings; `epiphany-core::EpiphanyStateUpdate`, `epiphany_state_update_validation_errors`, and `apply_epiphany_state_update` own state-update document shape and mutation law while `codex-core` calls them for compatibility. Vendored `CodexThread::epiphany_launch_job` is now mostly revision check, persistence validation, launch request handoff, and rollout/session writeback for the compatibility route. Vendored `codex-core` no longer has `epiphany_distillation`, `epiphany_promotion`, `epiphany_proposal`, or `epiphany_retrieval` husk modules; it re-exports native surfaces from `epiphany-core` directly, and now owns only the real Codex rollout adapter because it is a host seam. `epiphany-runtime-spine` advertises OpenAI adapter status/model-request/stream-event/receipt CultNet schemas and mutation contracts, so the native caller now has a real storage route instead of only JSON-RPC. MCP itself is correctly JSON at its protocol edge; the target is typed Epiphany intent/result/receipt wrappers around MCP, not replacing MCP JSON. The current OpenAI request path no longer pulls `codex-login` or `codex-api`; the next OpenAI spine decision is whether external bearer or agent-identity auth deserves native implementation or remains sealed legacy Codex compatibility. latest_cut: > - The watcher invalidation-to-freshness adapter moved out of the vendored - app-server root and into `epiphany-codex-bridge::invalidation`. App-server - still owns host lifecycle: creating/snapshotting watches, reading loaded - threads, and sending JSON-RPC responses. The bridge now owns the pure mapping - from `EpiphanyInvalidationSnapshot` to `EpiphanyFreshnessWatcherSnapshot`, - reducing one more Epiphany policy/projection helper inside - `codex_message_processor.rs`. + Token-usage rollout projection moved out of the vendored app-server helper + and into `epiphany-codex-bridge::token_usage`. App-server still owns host + lifecycle: reading rollout files, replaying connection-scoped JSON-RPC + token usage notifications, and preserving response ordering. The bridge now + owns the pure projection from Codex rollout items to latest token-usage + pressure/turn-owner facts used by Epiphany views. next_action: Continue starving coordinator/runtime execution and operator surfaces off app-server JSON-RPC and out of `codex_message_processor.rs`; next suspicion is physical extraction of Epiphany route/helper logic from vendored app-server without inventing compensator traits. diff --git a/vendor/codex/codex-rs/Cargo.lock b/vendor/codex/codex-rs/Cargo.lock index ef653a28..69de126e 100644 --- a/vendor/codex/codex-rs/Cargo.lock +++ b/vendor/codex/codex-rs/Cargo.lock @@ -5138,9 +5138,11 @@ dependencies = [ "base64 0.22.1", "chrono", "codex-client", + "codex-keyring-store", "reqwest", "serde", "serde_json", + "sha2 0.10.9", "thiserror 2.0.18", "tracing", ] diff --git a/vendor/codex/codex-rs/app-server/src/codex_message_processor/token_usage_replay.rs b/vendor/codex/codex-rs/app-server/src/codex_message_processor/token_usage_replay.rs index 752b4d81..44cd3908 100644 --- a/vendor/codex/codex-rs/app-server/src/codex_message_processor/token_usage_replay.rs +++ b/vendor/codex/codex-rs/app-server/src/codex_message_processor/token_usage_replay.rs @@ -14,15 +14,14 @@ use std::sync::Arc; use codex_app_server_protocol::ServerNotification; use codex_app_server_protocol::Thread; -use codex_app_server_protocol::ThreadHistoryBuilder; use codex_app_server_protocol::ThreadTokenUsage; use codex_app_server_protocol::ThreadTokenUsageUpdatedNotification; -use codex_app_server_protocol::TurnStatus; use codex_core::CodexThread; use codex_protocol::ThreadId; -use codex_protocol::protocol::EventMsg; -use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::TokenUsageInfo; +use epiphany_codex_bridge::token_usage::latest_token_usage_info_from_rollout_items; +use epiphany_codex_bridge::token_usage::latest_token_usage_turn_id; +pub(super) use epiphany_codex_bridge::token_usage::latest_token_usage_turn_id_from_rollout_items; use crate::codex_message_processor::read_rollout_items_from_rollout; use crate::outgoing_message::ConnectionId; @@ -78,73 +77,3 @@ pub(super) async fn latest_token_usage_info_from_rollout_path( let rollout_items = read_rollout_items_from_rollout(rollout_path).await.ok()?; latest_token_usage_info_from_rollout_items(&rollout_items) } - -pub(super) fn latest_token_usage_info_from_rollout_items( - rollout_items: &[RolloutItem], -) -> Option { - rollout_items.iter().rev().find_map(|item| match item { - RolloutItem::EventMsg(EventMsg::TokenCount(event)) => event.info.clone(), - _ => None, - }) -} - -/// Identifies the turn that was active when a `TokenCount` record appeared. -/// -/// The id is preferred when it still appears in the rebuilt thread. The position is a -/// fallback for histories whose implicit turn ids are regenerated during reconstruction. -struct TokenUsageTurnOwner { - id: String, - position: Option, -} - -pub(super) fn latest_token_usage_turn_id_from_rollout_items( - rollout_items: &[RolloutItem], - thread: &Thread, -) -> Option { - let owner = latest_token_usage_turn_owner_from_rollout_items(rollout_items)?; - if thread.turns.iter().any(|turn| turn.id == owner.id) { - return Some(owner.id); - } - owner - .position - .and_then(|position| thread.turns.get(position)) - .map(|turn| turn.id.clone()) -} - -fn latest_token_usage_turn_owner_from_rollout_items( - rollout_items: &[RolloutItem], -) -> Option { - let mut builder = ThreadHistoryBuilder::new(); - let mut token_usage_turn_owner = None; - - for item in rollout_items { - if matches!(item, RolloutItem::EventMsg(EventMsg::TokenCount(_))) { - token_usage_turn_owner = - builder - .active_turn_snapshot() - .map(|turn| TokenUsageTurnOwner { - id: turn.id, - position: builder.active_turn_position(), - }); - } - builder.handle_rollout_item(item); - } - - token_usage_turn_owner -} - -/// Chooses a fallback turn id that should own a replayed token usage update. -/// -/// Normal replay derives the owner from the rollout position of the latest -/// `TokenCount` event. This fallback only preserves a stable wire shape for -/// unusual histories where that rollout information cannot be read. -fn latest_token_usage_turn_id(thread: &Thread) -> String { - thread - .turns - .iter() - .rev() - .find(|turn| matches!(turn.status, TurnStatus::Completed | TurnStatus::Failed)) - .or_else(|| thread.turns.last()) - .map(|turn| turn.id.clone()) - .unwrap_or_default() -}