Epiphany/epiphany-core/src/retrieval.rs

use anyhow::Context;
use anyhow::Result;
use bm25::Document;
use bm25::Language;
use bm25::SearchEngineBuilder;
use codex_file_search as file_search;
use codex_protocol::protocol::EpiphanyRetrievalShardSummary;
use codex_protocol::protocol::EpiphanyRetrievalState;
use codex_protocol::protocol::EpiphanyRetrievalStatus;
use ignore::WalkBuilder;
use reqwest::StatusCode;
use reqwest::blocking::Client;
use reqwest::blocking::ClientBuilder;
use reqwest::header::HeaderMap;
use reqwest::header::HeaderValue;
use serde::Deserialize;
use serde::Serialize;
use serde_json::Value;
use serde_json::json;
use sha1::Digest;
use sha1::Sha1;
use std::cmp::Ordering;
use std::collections::BTreeMap;
use std::collections::BTreeSet;
use std::fs;
use std::num::NonZero;
use std::path::Component;
use std::path::Path;
use std::path::PathBuf;
use std::time::Duration;
use std::time::SystemTime;
use std::time::UNIX_EPOCH;
use uuid::Uuid;

pub const EPIPHANY_RETRIEVAL_DEFAULT_LIMIT: usize = 10;
pub const EPIPHANY_RETRIEVAL_MAX_LIMIT: usize = 50;

const EXACT_SEARCH_CANDIDATE_MULTIPLIER: usize = 5;
const EXACT_SEARCH_MAX_CANDIDATES: usize = 250;
const FILE_SEARCH_MAX_THREADS: usize = 12;
const QUERY_TIME_INDEX_REVISION: &str = "query-time-bm25-v1";
const QDRANT_INDEX_REVISION_PREFIX: &str = "qdrant-ollama-v1";
const SEMANTIC_MANIFEST_SCHEMA_VERSION: u32 = 1;
const MAX_TEXT_FILE_BYTES: u64 = 256 * 1024;
const CHUNK_LINE_COUNT: usize = 24;
const CHUNK_LINE_OVERLAP: usize = 8;
const WORKSPACE_SHARD_ID: &str = "workspace";
const MANIFESTS_DIR: &str = "epiphany/retrieval/manifests";
const QDRANT_COLLECTION_PREFIX: &str = "epiphany_workspace";
const QDRANT_POINT_BATCH_SIZE: usize = 128;
const OLLAMA_EMBED_BATCH_SIZE: usize = 32;
const DEFAULT_QDRANT_URL: &str = "http://127.0.0.1:6333";
const DEFAULT_QDRANT_TIMEOUT_MS: u64 = 30_000;
const DEFAULT_OLLAMA_BASE_URL: &str = "http://127.0.0.1:11434";
const DEFAULT_OLLAMA_MODEL: &str = "qwen3-embedding:0.6b";
const DEFAULT_OLLAMA_TIMEOUT_MS: u64 = 30_000;
const DEFAULT_QUERY_INSTRUCTION: &str = "Given a source-tree or codebase question, retrieve relevant files and code snippets that answer it.";

#[derive(Clone, Debug, PartialEq, Eq)]
pub struct EpiphanyRetrieveQuery {
    pub query: String,
    pub limit: usize,
    pub path_prefixes: Vec<PathBuf>,
}

impl EpiphanyRetrieveQuery {
    pub fn new(query: String) -> Self {
        Self {
            query,
            limit: EPIPHANY_RETRIEVAL_DEFAULT_LIMIT,
            path_prefixes: Vec::new(),
        }
    }
}

#[derive(Clone, Debug, PartialEq, Eq)]
pub enum EpiphanyRetrieveResultKind {
    ExactFile,
    ExactDirectory,
    SemanticChunk,
}

#[derive(Clone, Debug, PartialEq)]
pub struct EpiphanyRetrieveResult {
    pub kind: EpiphanyRetrieveResultKind,
    pub path: PathBuf,
    pub score: f32,
    pub line_start: Option<u32>,
    pub line_end: Option<u32>,
    pub excerpt: Option<String>,
}

#[derive(Clone, Debug, PartialEq)]
pub struct EpiphanyRetrieveResponse {
    pub query: String,
    pub index_summary: EpiphanyRetrievalState,
    pub results: Vec<EpiphanyRetrieveResult>,
}

pub fn retrieval_state_for_workspace(
    workspace_root: &Path,
    codex_home: &Path,
) -> EpiphanyRetrievalState {
    if !workspace_root.is_dir() {
        return unavailable_retrieval_state(workspace_root);
    }

    let config = SemanticBackendConfig::from_env();
    let manifest_path = config.manifest_path(codex_home, workspace_root);
    let Ok(Some(manifest)) = load_semantic_manifest(&manifest_path) else {
        return baseline_retrieval_state(workspace_root);
    };

    if manifest.index_revision != config.index_revision()
        || manifest.collection_name != config.collection_name(workspace_root)
    {
        return manifest_to_retrieval_state(
            workspace_root,
            &manifest,
            EpiphanyRetrievalStatus::Stale,
            Vec::new(),
        );
    }

    match collect_workspace_index_snapshot(workspace_root) {
        Ok(snapshot) => {
            let dirty_paths = dirty_paths_from_manifest(&manifest, &snapshot);
            let status = if dirty_paths.is_empty() {
                EpiphanyRetrievalStatus::Ready
            } else {
                EpiphanyRetrievalStatus::Stale
            };
            manifest_to_retrieval_state(workspace_root, &manifest, status, dirty_paths)
        }
        Err(_) => manifest_to_retrieval_state(
            workspace_root,
            &manifest,
            EpiphanyRetrievalStatus::Stale,
            Vec::new(),
        ),
    }
}

pub fn index_workspace(
    workspace_root: &Path,
    codex_home: &Path,
    force_full_rebuild: bool,
) -> Result<EpiphanyRetrievalState> {
    if !workspace_root.is_dir() {
        anyhow::bail!(
            "workspace root does not exist: {}",
            workspace_root.display()
        );
    }

    let config = SemanticBackendConfig::from_env();
    index_workspace_with_config(workspace_root, codex_home, force_full_rebuild, &config)
}

pub fn retrieve_workspace(
    workspace_root: &Path,
    codex_home: &Path,
    query: EpiphanyRetrieveQuery,
) -> Result<EpiphanyRetrieveResponse> {
    if !workspace_root.is_dir() {
        anyhow::bail!(
            "workspace root does not exist: {}",
            workspace_root.display()
        );
    }

    let query_text = query.query.trim();
    if query_text.is_empty() {
        anyhow::bail!("query must not be empty");
    }

    let limit = query.limit.clamp(1, EPIPHANY_RETRIEVAL_MAX_LIMIT);
    let path_prefixes = normalize_path_prefixes(workspace_root, &query.path_prefixes);

    let exact_results = run_exact_search(workspace_root, query_text, limit, &path_prefixes)
        .context("exact retrieval failed")?;

    let semantic_limit = semantic_candidate_limit(limit, &path_prefixes);
    let persistent_search = match search_persistent_semantic_chunks(
        workspace_root,
        codex_home,
        query_text,
        semantic_limit,
    ) {
        Ok(search) => search,
        Err(err) => {
            tracing::warn!(
                workspace_root = %workspace_root.display(),
                error = %err,
                "Epiphany persistent semantic search failed; falling back to query-time BM25"
            );
            PersistentSemanticSearch::Fallback {
                summary: stale_summary_from_manifest(workspace_root, codex_home),
            }
        }
    };

    let (index_summary, semantic_results) = match persistent_search {
        PersistentSemanticSearch::Ready { summary, results } => (
            summary,
            results
                .into_iter()
                .filter(|result| matches_path_prefixes(&result.path, &path_prefixes))
                .take(limit)
                .collect::<Vec<_>>(),
        ),
        PersistentSemanticSearch::Fallback { summary } => {
            let semantic_corpus = collect_semantic_corpus(workspace_root, &path_prefixes)
                .context("semantic corpus collection failed")?;
            let semantic_results =
                search_semantic_chunks(query_text, semantic_limit, &semantic_corpus);
            (
                summary.unwrap_or_else(|| {
                    query_time_retrieval_state(
                        workspace_root,
                        semantic_corpus.searchable_file_count,
                        semantic_corpus.chunks.len(),
                    )
                }),
                semantic_results,
            )
        }
        PersistentSemanticSearch::None => {
            let semantic_corpus = collect_semantic_corpus(workspace_root, &path_prefixes)
                .context("semantic corpus collection failed")?;
            let semantic_results =
                search_semantic_chunks(query_text, semantic_limit, &semantic_corpus);
            (
                query_time_retrieval_state(
                    workspace_root,
                    semantic_corpus.searchable_file_count,
                    semantic_corpus.chunks.len(),
                ),
                semantic_results,
            )
        }
    };

    let mut results = Vec::with_capacity(exact_results.len() + semantic_results.len());
    results.extend(exact_results);
    results.extend(semantic_results);
    sort_results(&mut results);
    results.truncate(limit);

    Ok(EpiphanyRetrieveResponse {
        query: query_text.to_string(),
        index_summary,
        results,
    })
}

fn index_workspace_with_config(
    workspace_root: &Path,
    codex_home: &Path,
    force_full_rebuild: bool,
    config: &SemanticBackendConfig,
) -> Result<EpiphanyRetrievalState> {
    let manifest_path = config.manifest_path(codex_home, workspace_root);
    let snapshot = collect_workspace_index_snapshot(workspace_root)?;
    let existing_manifest = load_semantic_manifest(&manifest_path)?;
    let expected_collection = config.collection_name(workspace_root);
    let qdrant = QdrantClient::new(config.qdrant.clone())?;

    let manifest_compatible = existing_manifest.as_ref().is_some_and(|manifest| {
        manifest.index_revision == config.index_revision()
            && manifest.collection_name == expected_collection
    });
    let collection_exists = qdrant.collection_exists(&expected_collection)?;

    let full_rebuild = force_full_rebuild || !manifest_compatible || !collection_exists;
    if full_rebuild {
        if let Some(manifest) = existing_manifest.as_ref()
            && manifest.collection_name != expected_collection
        {
            qdrant.delete_collection(&manifest.collection_name)?;
        }
        if collection_exists {
            qdrant.delete_collection(&expected_collection)?;
        }
    }

    if snapshot.files.is_empty() {
        let manifest = SemanticIndexManifest {
            schema_version: SEMANTIC_MANIFEST_SCHEMA_VERSION,
            workspace_root: workspace_root.to_path_buf(),
            collection_name: expected_collection,
            index_revision: config.index_revision(),
            indexed_at_unix_seconds: unix_timestamp_seconds(SystemTime::now())?,
            files: Vec::new(),
        };
        write_semantic_manifest(&manifest_path, &manifest)?;
        return Ok(manifest_to_retrieval_state(
            workspace_root,
            &manifest,
            EpiphanyRetrievalStatus::Ready,
            Vec::new(),
        ));
    }

    let (paths_to_reindex, deleted_point_ids, carried_files) = if full_rebuild {
        (
            snapshot.files.keys().cloned().collect::<BTreeSet<_>>(),
            Vec::new(),
            BTreeMap::new(),
        )
    } else {
        compute_incremental_plan(existing_manifest.as_ref(), &snapshot)
    };

    let indexed_files = collect_indexed_files(workspace_root, &paths_to_reindex)?;

    let mut new_manifest_files = carried_files;
    for indexed_file in &indexed_files {
        let snapshot_file = snapshot.files.get(&indexed_file.path).with_context(|| {
            format!("missing index snapshot for {}", indexed_file.path.display())
        })?;
        new_manifest_files.insert(
            indexed_file.path.clone(),
            SemanticIndexedFile {
                path: indexed_file.path.clone(),
                size_bytes: snapshot_file.size_bytes,
                modified_unix_nanos: snapshot_file.modified_unix_nanos,
                chunk_count: indexed_file.chunks.len() as u32,
                point_ids: indexed_file
                    .chunks
                    .iter()
                    .map(|chunk| chunk_point_id(&expected_collection, chunk))
                    .collect(),
            },
        );
    }

    if !deleted_point_ids.is_empty() {
        qdrant.delete_points(&expected_collection, &deleted_point_ids)?;
    }

    if !indexed_files.is_empty() {
        let ollama = OllamaEmbedder::new(config.ollama.clone())?;
        let documents = indexed_files
            .iter()
            .flat_map(|file| file.chunks.iter().cloned())
            .collect::<Vec<_>>();
        let texts = documents
            .iter()
            .map(|chunk| chunk.search_text.clone())
            .collect::<Vec<_>>();
        let embeddings = ollama.embed_documents(&texts)?;
        validate_embedding_batch(&embeddings, texts.len())?;

        if full_rebuild {
            let vector_size = embeddings
                .first()
                .map(Vec::len)
                .context("embedding backend returned no vectors for rebuild")?;
            qdrant.create_collection(
                &expected_collection,
                vector_size,
                &config.index_revision(),
                workspace_root,
                &config.ollama.model,
            )?;
        }

        let points = documents
            .into_iter()
            .zip(embeddings)
            .map(|(chunk, vector)| QdrantPointInput {
                id: chunk_point_id(&expected_collection, &chunk),
                vector,
                payload: json!({
                    "path": normalize_payload_path(&chunk.path),
                    "line_start": chunk.line_start,
                    "line_end": chunk.line_end,
                    "excerpt": chunk.excerpt,
                }),
            })
            .collect::<Vec<_>>();
        qdrant.upsert_points(&expected_collection, &points)?;
    }

    let manifest = SemanticIndexManifest {
        schema_version: SEMANTIC_MANIFEST_SCHEMA_VERSION,
        workspace_root: workspace_root.to_path_buf(),
        collection_name: expected_collection,
        index_revision: config.index_revision(),
        indexed_at_unix_seconds: unix_timestamp_seconds(SystemTime::now())?,
        files: new_manifest_files.into_values().collect(),
    };
    write_semantic_manifest(&manifest_path, &manifest)?;

    Ok(manifest_to_retrieval_state(
        workspace_root,
        &manifest,
        EpiphanyRetrievalStatus::Ready,
        Vec::new(),
    ))
}

fn search_persistent_semantic_chunks(
    workspace_root: &Path,
    codex_home: &Path,
    query: &str,
    limit: usize,
) -> Result<PersistentSemanticSearch> {
    let config = SemanticBackendConfig::from_env();
    let manifest_path = config.manifest_path(codex_home, workspace_root);
    let Some(manifest) = load_semantic_manifest(&manifest_path)? else {
        return Ok(PersistentSemanticSearch::None);
    };

    if manifest.index_revision != config.index_revision()
        || manifest.collection_name != config.collection_name(workspace_root)
    {
        return Ok(PersistentSemanticSearch::Fallback {
            summary: Some(manifest_to_retrieval_state(
                workspace_root,
                &manifest,
                EpiphanyRetrievalStatus::Stale,
                Vec::new(),
            )),
        });
    }

    let snapshot = collect_workspace_index_snapshot(workspace_root)?;
    let dirty_paths = dirty_paths_from_manifest(&manifest, &snapshot);
    if !dirty_paths.is_empty() {
        return Ok(PersistentSemanticSearch::Fallback {
            summary: Some(manifest_to_retrieval_state(
                workspace_root,
                &manifest,
                EpiphanyRetrievalStatus::Stale,
                dirty_paths,
            )),
        });
    }

    let qdrant = QdrantClient::new(config.qdrant.clone())?;
    if !qdrant.collection_exists(&manifest.collection_name)? {
        return Ok(PersistentSemanticSearch::Fallback {
            summary: Some(manifest_to_retrieval_state(
                workspace_root,
                &manifest,
                EpiphanyRetrievalStatus::Stale,
                Vec::new(),
            )),
        });
    }

    let ollama = OllamaEmbedder::new(config.ollama.clone())?;
    let query_vector = ollama.embed_query(query)?;
    let search_hits = qdrant.query_points(&manifest.collection_name, &query_vector, limit)?;

    let results = search_hits
        .into_iter()
        .filter_map(map_qdrant_hit_to_result)
        .collect::<Vec<_>>();
    let summary = manifest_to_retrieval_state(
        workspace_root,
        &manifest,
        EpiphanyRetrievalStatus::Ready,
        Vec::new(),
    );

    Ok(PersistentSemanticSearch::Ready { summary, results })
}

fn stale_summary_from_manifest(
    workspace_root: &Path,
    codex_home: &Path,
) -> Option<EpiphanyRetrievalState> {
    let config = SemanticBackendConfig::from_env();
    let manifest_path = config.manifest_path(codex_home, workspace_root);
    let manifest = load_semantic_manifest(&manifest_path).ok().flatten()?;
    Some(manifest_to_retrieval_state(
        workspace_root,
        &manifest,
        EpiphanyRetrievalStatus::Stale,
        Vec::new(),
    ))
}

fn run_exact_search(
    workspace_root: &Path,
    query: &str,
    limit: usize,
    path_prefixes: &[PathBuf],
) -> Result<Vec<EpiphanyRetrieveResult>> {
    let candidate_limit = if path_prefixes.is_empty() {
        limit
    } else {
        limit
            .saturating_mul(EXACT_SEARCH_CANDIDATE_MULTIPLIER)
            .min(EXACT_SEARCH_MAX_CANDIDATES)
    };
    #[expect(clippy::expect_used)]
    let candidate_limit =
        NonZero::new(candidate_limit.max(1)).expect("candidate limit should be non-zero");

    let search_results = file_search::run(
        query,
        vec![workspace_root.to_path_buf()],
        file_search::FileSearchOptions {
            limit: candidate_limit,
            threads: search_threads(),
            compute_indices: false,
            ..Default::default()
        },
        /*cancel_flag*/ None,
    )?;

    let mut results = search_results
        .matches
        .into_iter()
        .filter_map(|file_match| {
            let relative_path = file_match
                .full_path()
                .strip_prefix(workspace_root)
                .ok()?
                .to_path_buf();
            if !matches_path_prefixes(&relative_path, path_prefixes) {
                return None;
            }

            Some(EpiphanyRetrieveResult {
                kind: match file_match.match_type {
                    file_search::MatchType::File => EpiphanyRetrieveResultKind::ExactFile,
                    file_search::MatchType::Directory => EpiphanyRetrieveResultKind::ExactDirectory,
                },
                path: relative_path,
                score: file_match.score as f32,
                line_start: None,
                line_end: None,
                excerpt: None,
            })
        })
        .collect::<Vec<_>>();

    results.truncate(limit);
    Ok(results)
}

fn search_threads() -> NonZero<usize> {
    let cores = std::thread::available_parallelism()
        .map(NonZero::get)
        .unwrap_or(1);
    #[expect(clippy::expect_used)]
    NonZero::new(cores.min(FILE_SEARCH_MAX_THREADS).max(1))
        .expect("file-search threads should be non-zero")
}

#[derive(Clone, Debug)]
struct SemanticChunk {
    path: PathBuf,
    line_start: u32,
    line_end: u32,
    excerpt: String,
    search_text: String,
}

struct SemanticCorpus {
    searchable_file_count: usize,
    chunks: Vec<SemanticChunk>,
}

struct WorkspaceIndexSnapshot {
    files: BTreeMap<PathBuf, WorkspaceIndexSnapshotFile>,
}

#[derive(Clone, Debug, PartialEq, Eq)]
struct WorkspaceIndexSnapshotFile {
    size_bytes: u64,
    modified_unix_nanos: i64,
    chunk_count: u32,
}

#[derive(Clone, Debug)]
struct IndexedFile {
    path: PathBuf,
    chunks: Vec<SemanticChunk>,
}

#[derive(Clone, Debug, Serialize, Deserialize)]
struct SemanticIndexManifest {
    schema_version: u32,
    workspace_root: PathBuf,
    collection_name: String,
    index_revision: String,
    indexed_at_unix_seconds: i64,
    #[serde(default)]
    files: Vec<SemanticIndexedFile>,
}

#[derive(Clone, Debug, Serialize, Deserialize)]
struct SemanticIndexedFile {
    path: PathBuf,
    size_bytes: u64,
    modified_unix_nanos: i64,
    chunk_count: u32,
    #[serde(default)]
    point_ids: Vec<String>,
}

enum PersistentSemanticSearch {
    Ready {
        summary: EpiphanyRetrievalState,
        results: Vec<EpiphanyRetrieveResult>,
    },
    Fallback {
        summary: Option<EpiphanyRetrievalState>,
    },
    None,
}

#[derive(Clone, Debug)]
struct SemanticBackendConfig {
    qdrant: QdrantConfig,
    ollama: OllamaConfig,
}

impl SemanticBackendConfig {
    fn from_env() -> Self {
        Self {
            qdrant: QdrantConfig {
                url: env_override("EPIPHANY_QDRANT_URL", "QDRANT_URL")
                    .unwrap_or_else(|| DEFAULT_QDRANT_URL.to_string()),
                api_key: env_override("EPIPHANY_QDRANT_API_KEY", "QDRANT_API_KEY"),
                timeout_ms: env_override("EPIPHANY_QDRANT_TIMEOUT_MS", "QDRANT_TIMEOUT_MS")
                    .and_then(|value| value.parse::<u64>().ok())
                    .unwrap_or(DEFAULT_QDRANT_TIMEOUT_MS),
            },
            ollama: OllamaConfig {
                base_url: env_override("EPIPHANY_OLLAMA_BASE_URL", "RAG_OLLAMA_BASE_URL")
                    .unwrap_or_else(|| DEFAULT_OLLAMA_BASE_URL.to_string()),
                model: env_override("EPIPHANY_OLLAMA_MODEL", "RAG_OLLAMA_MODEL")
                    .unwrap_or_else(|| DEFAULT_OLLAMA_MODEL.to_string()),
                timeout_ms: env_override("EPIPHANY_OLLAMA_TIMEOUT_MS", "RAG_OLLAMA_TIMEOUT_MS")
                    .and_then(|value| value.parse::<u64>().ok())
                    .unwrap_or(DEFAULT_OLLAMA_TIMEOUT_MS),
                query_instruction: env_override(
                    "EPIPHANY_OLLAMA_QUERY_INSTRUCTION",
                    "RAG_SOURCE_QUERY_INSTRUCTION",
                )
                .unwrap_or_else(|| DEFAULT_QUERY_INSTRUCTION.to_string()),
            },
        }
    }

    fn index_revision(&self) -> String {
        format!("{QDRANT_INDEX_REVISION_PREFIX}:{}", self.ollama.model)
    }

    fn collection_name(&self, workspace_root: &Path) -> String {
        let mut hasher = Sha1::new();
        hasher.update(workspace_root.to_string_lossy().as_bytes());
        hasher.update(b"\n");
        hasher.update(self.index_revision().as_bytes());
        let digest = hasher.finalize();
        format!("{QDRANT_COLLECTION_PREFIX}_{}", hex_lower(&digest[..8]))
    }

    fn manifest_path(&self, codex_home: &Path, workspace_root: &Path) -> PathBuf {
        let mut hasher = Sha1::new();
        hasher.update(workspace_root.to_string_lossy().as_bytes());
        let digest = hasher.finalize();
        codex_home
            .join(MANIFESTS_DIR)
            .join(format!("{}.json", hex_lower(&digest[..10])))
    }
}

#[derive(Clone, Debug)]
struct QdrantConfig {
    url: String,
    api_key: Option<String>,
    timeout_ms: u64,
}

#[derive(Clone, Debug)]
struct OllamaConfig {
    base_url: String,
    model: String,
    timeout_ms: u64,
    query_instruction: String,
}

struct QdrantClient {
    base_url: String,
    timeout_seconds: u64,
    client: Client,
}

impl QdrantClient {
    fn new(config: QdrantConfig) -> Result<Self> {
        let mut headers = HeaderMap::new();
        if let Some(api_key) = config.api_key
            && !api_key.is_empty()
        {
            headers.insert(
                "api-key",
                HeaderValue::from_str(&api_key).context("invalid Qdrant api key")?,
            );
        }

        let client = ClientBuilder::new()
            .default_headers(headers)
            .timeout(Duration::from_millis(config.timeout_ms))
            .build()
            .context("failed to build Qdrant client")?;

        Ok(Self {
            base_url: normalize_base_url(&config.url),
            timeout_seconds: timeout_seconds(config.timeout_ms),
            client,
        })
    }

    fn collection_exists(&self, collection_name: &str) -> Result<bool> {
        let response = self
            .client
            .get(format!(
                "{}/collections/{collection_name}/exists",
                self.base_url
            ))
            .send()
            .with_context(|| format!("failed to query Qdrant collection {collection_name}"))?;

        let payload: QdrantEnvelope<QdrantCollectionExistsResult> = parse_qdrant_response(response)
            .with_context(|| {
                format!("failed to decode Qdrant collection-exists response for {collection_name}")
            })?;
        Ok(payload.result.exists)
    }

    fn delete_collection(&self, collection_name: &str) -> Result<()> {
        let response = self
            .client
            .delete(format!("{}/collections/{collection_name}", self.base_url))
            .query(&[("timeout", self.timeout_seconds)])
            .send()
            .with_context(|| format!("failed to delete Qdrant collection {collection_name}"))?;

        if response.status() == StatusCode::NOT_FOUND {
            return Ok(());
        }
        parse_qdrant_response::<bool>(response).with_context(|| {
            format!("failed to decode Qdrant delete-collection response for {collection_name}")
        })?;
        Ok(())
    }

    fn create_collection(
        &self,
        collection_name: &str,
        vector_size: usize,
        index_revision: &str,
        workspace_root: &Path,
        embedding_model: &str,
    ) -> Result<()> {
        let body = json!({
            "vectors": {
                "size": vector_size,
                "distance": "Cosine",
                "on_disk": true,
            },
            "on_disk_payload": true,
            "metadata": {
                "managedBy": "epiphany",
                "schemaVersion": SEMANTIC_MANIFEST_SCHEMA_VERSION,
                "indexRevision": index_revision,
                "workspaceRoot": workspace_root.to_string_lossy(),
                "embeddingModel": embedding_model,
                "vectorSize": vector_size,
            }
        });

        let response = self
            .client
            .put(format!("{}/collections/{collection_name}", self.base_url))
            .query(&[("timeout", self.timeout_seconds)])
            .json(&body)
            .send()
            .with_context(|| format!("failed to create Qdrant collection {collection_name}"))?;

        parse_qdrant_response::<bool>(response).with_context(|| {
            format!("failed to decode Qdrant create-collection response for {collection_name}")
        })?;
        Ok(())
    }

    fn upsert_points(&self, collection_name: &str, points: &[QdrantPointInput]) -> Result<()> {
        for chunk in points.chunks(QDRANT_POINT_BATCH_SIZE) {
            let body = json!({ "points": chunk });
            let response = self
                .client
                .put(format!(
                    "{}/collections/{collection_name}/points",
                    self.base_url
                ))
                .query(&[
                    ("wait", "true"),
                    ("timeout", &self.timeout_seconds.to_string()),
                ])
                .json(&body)
                .send()
                .with_context(|| {
                    format!("failed to upsert Qdrant points into {collection_name}")
                })?;

            parse_qdrant_response::<Value>(response).with_context(|| {
                format!("failed to decode Qdrant upsert response for {collection_name}")
            })?;
        }
        Ok(())
    }

    fn delete_points(&self, collection_name: &str, point_ids: &[String]) -> Result<()> {
        if point_ids.is_empty() {
            return Ok(());
        }

        for batch in point_ids.chunks(QDRANT_POINT_BATCH_SIZE) {
            let body = json!({ "points": batch });
            let response = self
                .client
                .post(format!(
                    "{}/collections/{collection_name}/points/delete",
                    self.base_url
                ))
                .query(&[
                    ("wait", "true"),
                    ("timeout", &self.timeout_seconds.to_string()),
                ])
                .json(&body)
                .send()
                .with_context(|| {
                    format!("failed to delete Qdrant points from {collection_name}")
                })?;

            parse_qdrant_response::<Value>(response).with_context(|| {
                format!("failed to decode Qdrant delete-points response for {collection_name}")
            })?;
        }
        Ok(())
    }

    fn query_points(
        &self,
        collection_name: &str,
        query_vector: &[f32],
        limit: usize,
    ) -> Result<Vec<QdrantQueryPoint>> {
        let body = json!({
            "query": query_vector,
            "limit": limit,
            "with_payload": true,
            "with_vector": false,
        });

        let response = self
            .client
            .post(format!(
                "{}/collections/{collection_name}/points/query",
                self.base_url
            ))
            .query(&[("timeout", self.timeout_seconds)])
            .json(&body)
            .send()
            .with_context(|| format!("failed to query Qdrant collection {collection_name}"))?;

        let payload: QdrantEnvelope<QdrantQueryResultEnvelope> = parse_qdrant_response(response)
            .with_context(|| {
                format!("failed to decode Qdrant query response for {collection_name}")
            })?;
        Ok(payload.result.points)
    }
}

struct OllamaEmbedder {
    base_url: String,
    model: String,
    query_instruction: String,
    client: Client,
}

impl OllamaEmbedder {
    fn new(config: OllamaConfig) -> Result<Self> {
        let client = ClientBuilder::new()
            .timeout(Duration::from_millis(config.timeout_ms))
            .build()
            .context("failed to build Ollama client")?;

        Ok(Self {
            base_url: normalize_base_url(&config.base_url),
            model: config.model,
            query_instruction: config.query_instruction,
            client,
        })
    }

    fn embed_documents(&self, texts: &[String]) -> Result<Vec<Vec<f32>>> {
        let mut embeddings = Vec::new();
        for batch in texts.chunks(OLLAMA_EMBED_BATCH_SIZE) {
            let payload = self.embed_batch(batch)?;
            embeddings.extend(payload);
        }
        Ok(embeddings)
    }

    fn embed_query(&self, query: &str) -> Result<Vec<f32>> {
        let formatted = format!("Instruct: {}\nQuery: {}", self.query_instruction, query);
        let mut payload = self.embed_batch(&[formatted])?;
        payload
            .pop()
            .context("Ollama embedding backend returned no vector for query")
    }

    fn embed_batch(&self, texts: &[String]) -> Result<Vec<Vec<f32>>> {
        let response = self
            .client
            .post(format!("{}/api/embed", self.base_url))
            .json(&json!({
                "model": self.model,
                "input": texts,
            }))
            .send()
            .with_context(|| {
                format!(
                    "failed to contact Ollama at {} using model {}",
                    self.base_url, self.model
                )
            })?;

        if !response.status().is_success() {
            let status = response.status();
            let body = response.text().unwrap_or_default();
            anyhow::bail!(
                "Ollama embedding request failed with {status}: {body}. Make sure Ollama is running at {} and model {} is available",
                self.base_url,
                self.model
            );
        }

        let payload: OllamaEmbedResponse = response.json().with_context(|| {
            format!(
                "failed to decode Ollama embedding response from {}",
                self.base_url
            )
        })?;
        payload
            .embeddings
            .context("Ollama embedding response did not include embeddings")
    }
}

#[derive(Serialize)]
struct QdrantPointInput {
    id: String,
    vector: Vec<f32>,
    payload: Value,
}

#[derive(Debug, Deserialize)]
struct QdrantEnvelope<T> {
    result: T,
}

#[derive(Debug, Deserialize)]
struct QdrantCollectionExistsResult {
    exists: bool,
}

#[derive(Debug, Deserialize)]
struct QdrantQueryResultEnvelope {
    points: Vec<QdrantQueryPoint>,
}

#[derive(Clone, Debug, Deserialize)]
struct QdrantQueryPoint {
    score: f32,
    #[serde(default)]
    payload: BTreeMap<String, Value>,
}

#[derive(Debug, Deserialize)]
struct OllamaEmbedResponse {
    embeddings: Option<Vec<Vec<f32>>>,
}

fn parse_qdrant_response<T: for<'de> Deserialize<'de>>(
    response: reqwest::blocking::Response,
) -> Result<QdrantEnvelope<T>> {
    let status = response.status();
    if !status.is_success() {
        let body = response.text().unwrap_or_default();
        anyhow::bail!("Qdrant request failed with {status}: {body}");
    }
    response
        .json()
        .context("failed to decode Qdrant response JSON")
}

fn collect_semantic_corpus(
    workspace_root: &Path,
    path_prefixes: &[PathBuf],
) -> Result<SemanticCorpus> {
    let mut builder = WalkBuilder::new(workspace_root);
    builder.hidden(false);

    let mut searchable_file_count = 0usize;
    let mut chunks = Vec::new();
    for entry in builder.build() {
        let Ok(entry) = entry else {
            continue;
        };
        if !entry
            .file_type()
            .is_some_and(|file_type| file_type.is_file())
        {
            continue;
        }

        let path = entry.path();
        let Ok(relative_path) = path.strip_prefix(workspace_root) else {
            continue;
        };
        if should_skip_relative_path(relative_path) {
            continue;
        }
        if !matches_path_prefixes(relative_path, path_prefixes) {
            continue;
        }

        let Ok(metadata) = fs::metadata(path) else {
            continue;
        };
        if metadata.len() > MAX_TEXT_FILE_BYTES {
            continue;
        }

        let Ok(contents) = fs::read_to_string(path) else {
            continue;
        };
        let file_chunks = build_semantic_chunks(relative_path, &contents);
        if file_chunks.is_empty() {
            continue;
        }

        searchable_file_count += 1;
        chunks.extend(file_chunks);
    }

    Ok(SemanticCorpus {
        searchable_file_count,
        chunks,
    })
}

fn collect_workspace_index_snapshot(workspace_root: &Path) -> Result<WorkspaceIndexSnapshot> {
    let mut builder = WalkBuilder::new(workspace_root);
    builder.hidden(false);

    let mut files = BTreeMap::new();
    for entry in builder.build() {
        let Ok(entry) = entry else {
            continue;
        };
        if !entry
            .file_type()
            .is_some_and(|file_type| file_type.is_file())
        {
            continue;
        }

        let path = entry.path();
        let Ok(relative_path) = path.strip_prefix(workspace_root) else {
            continue;
        };
        if should_skip_relative_path(relative_path) {
            continue;
        }

        let metadata = match fs::metadata(path) {
            Ok(metadata) if metadata.len() <= MAX_TEXT_FILE_BYTES => metadata,
            _ => continue,
        };

        let Ok(contents) = fs::read_to_string(path) else {
            continue;
        };
        let chunk_count = build_semantic_chunks(relative_path, &contents).len() as u32;
        if chunk_count == 0 {
            continue;
        }

        files.insert(
            relative_path.to_path_buf(),
            WorkspaceIndexSnapshotFile {
                size_bytes: metadata.len(),
                modified_unix_nanos: unix_timestamp_nanos(metadata.modified()?)?,
                chunk_count,
            },
        );
    }

    Ok(WorkspaceIndexSnapshot { files })
}

fn collect_indexed_files(
    workspace_root: &Path,
    relative_paths: &BTreeSet<PathBuf>,
) -> Result<Vec<IndexedFile>> {
    let mut files = Vec::with_capacity(relative_paths.len());
    for relative_path in relative_paths {
        let absolute_path = workspace_root.join(relative_path);
        let contents = fs::read_to_string(&absolute_path).with_context(|| {
            format!("failed to read indexable file {}", absolute_path.display())
        })?;
        let chunks = build_semantic_chunks(relative_path, &contents);
        if chunks.is_empty() {
            continue;
        }
        files.push(IndexedFile {
            path: relative_path.clone(),
            chunks,
        });
    }
    Ok(files)
}

fn build_semantic_chunks(relative_path: &Path, contents: &str) -> Vec<SemanticChunk> {
    let lines = contents.lines().collect::<Vec<_>>();
    if lines.is_empty() {
        return Vec::new();
    }

    let step = CHUNK_LINE_COUNT.saturating_sub(CHUNK_LINE_OVERLAP).max(1);
    let mut chunks = Vec::new();
    let mut start = 0usize;
    while start < lines.len() {
        let end = (start + CHUNK_LINE_COUNT).min(lines.len());
        let excerpt = lines[start..end].join("\n").trim().to_string();
        if !excerpt.is_empty() {
            chunks.push(SemanticChunk {
                path: relative_path.to_path_buf(),
                line_start: (start + 1) as u32,
                line_end: end as u32,
                search_text: format!("{}\n{}", relative_path.display(), excerpt),
                excerpt,
            });
        }

        if end == lines.len() {
            break;
        }
        start += step;
    }

    chunks
}

fn compute_incremental_plan(
    manifest: Option<&SemanticIndexManifest>,
    snapshot: &WorkspaceIndexSnapshot,
) -> (
    BTreeSet<PathBuf>,
    Vec<String>,
    BTreeMap<PathBuf, SemanticIndexedFile>,
) {
    let Some(manifest) = manifest else {
        return (
            snapshot.files.keys().cloned().collect(),
            Vec::new(),
            BTreeMap::new(),
        );
    };

    let manifest_map = manifest_file_map(manifest);
    let mut paths_to_reindex = BTreeSet::new();
    let mut deleted_point_ids = Vec::new();
    let mut carried_files = BTreeMap::new();

    for (path, manifest_file) in &manifest_map {
        match snapshot.files.get(path) {
            Some(snapshot_file) if same_snapshot_file(manifest_file, snapshot_file) => {
                carried_files.insert(path.clone(), manifest_file.clone());
            }
            Some(_) => {
                paths_to_reindex.insert(path.clone());
                deleted_point_ids.extend(manifest_file.point_ids.clone());
            }
            None => {
                deleted_point_ids.extend(manifest_file.point_ids.clone());
            }
        }
    }

    for path in snapshot.files.keys() {
        if !manifest_map.contains_key(path) {
            paths_to_reindex.insert(path.clone());
        }
    }

    (paths_to_reindex, deleted_point_ids, carried_files)
}

fn manifest_file_map(manifest: &SemanticIndexManifest) -> BTreeMap<PathBuf, SemanticIndexedFile> {
    manifest
        .files
        .iter()
        .cloned()
        .map(|file| (file.path.clone(), file))
        .collect()
}

fn same_snapshot_file(
    manifest_file: &SemanticIndexedFile,
    snapshot_file: &WorkspaceIndexSnapshotFile,
) -> bool {
    manifest_file.size_bytes == snapshot_file.size_bytes
        && manifest_file.modified_unix_nanos == snapshot_file.modified_unix_nanos
        && manifest_file.chunk_count == snapshot_file.chunk_count
}

fn dirty_paths_from_manifest(
    manifest: &SemanticIndexManifest,
    snapshot: &WorkspaceIndexSnapshot,
) -> Vec<PathBuf> {
    let manifest_map = manifest_file_map(manifest);
    let mut dirty_paths = BTreeSet::new();

    for (path, manifest_file) in &manifest_map {
        match snapshot.files.get(path) {
            Some(snapshot_file) if same_snapshot_file(manifest_file, snapshot_file) => {}
            _ => {
                dirty_paths.insert(path.clone());
            }
        }
    }

    for path in snapshot.files.keys() {
        if !manifest_map.contains_key(path) {
            dirty_paths.insert(path.clone());
        }
    }

    dirty_paths.into_iter().collect()
}

fn load_semantic_manifest(path: &Path) -> Result<Option<SemanticIndexManifest>> {
    if !path.is_file() {
        return Ok(None);
    }

    let bytes = fs::read(path)
        .with_context(|| format!("failed to read semantic index manifest {}", path.display()))?;
    let manifest: SemanticIndexManifest = serde_json::from_slice(&bytes).with_context(|| {
        format!(
            "failed to decode semantic index manifest {}",
            path.display()
        )
    })?;
    if manifest.schema_version != SEMANTIC_MANIFEST_SCHEMA_VERSION {
        return Ok(None);
    }

    Ok(Some(manifest))
}

fn write_semantic_manifest(path: &Path, manifest: &SemanticIndexManifest) -> Result<()> {
    if let Some(parent) = path.parent() {
        fs::create_dir_all(parent).with_context(|| {
            format!(
                "failed to create semantic index manifest directory {}",
                parent.display()
            )
        })?;
    }

    let payload =
        serde_json::to_vec_pretty(manifest).context("failed to encode semantic manifest")?;
    fs::write(path, payload)
        .with_context(|| format!("failed to write semantic manifest {}", path.display()))
}

fn search_semantic_chunks(
    query: &str,
    limit: usize,
    semantic_corpus: &SemanticCorpus,
) -> Vec<EpiphanyRetrieveResult> {
    if semantic_corpus.chunks.is_empty() {
        return Vec::new();
    }

    let documents = semantic_corpus
        .chunks
        .iter()
        .enumerate()
        .map(|(index, chunk)| Document::new(index, chunk.search_text.clone()))
        .collect::<Vec<_>>();
    let search_engine =
        SearchEngineBuilder::<usize>::with_documents(Language::English, documents).build();

    search_engine
        .search(query, limit)
        .into_iter()
        .filter_map(|result| {
            semantic_corpus
                .chunks
                .get(result.document.id)
                .map(|chunk| EpiphanyRetrieveResult {
                    kind: EpiphanyRetrieveResultKind::SemanticChunk,
                    path: chunk.path.clone(),
                    score: result.score,
                    line_start: Some(chunk.line_start),
                    line_end: Some(chunk.line_end),
                    excerpt: Some(chunk.excerpt.clone()),
                })
        })
        .collect()
}

fn map_qdrant_hit_to_result(point: QdrantQueryPoint) -> Option<EpiphanyRetrieveResult> {
    let path = point.payload.get("path")?.as_str()?;
    let excerpt = point.payload.get("excerpt")?.as_str()?.to_string();
    let line_start = point.payload.get("line_start")?.as_u64()? as u32;
    let line_end = point.payload.get("line_end")?.as_u64()? as u32;

    Some(EpiphanyRetrieveResult {
        kind: EpiphanyRetrieveResultKind::SemanticChunk,
        path: PathBuf::from(path),
        score: point.score,
        line_start: Some(line_start),
        line_end: Some(line_end),
        excerpt: Some(excerpt),
    })
}

fn unavailable_retrieval_state(workspace_root: &Path) -> EpiphanyRetrievalState {
    EpiphanyRetrievalState {
        workspace_root: workspace_root.to_path_buf(),
        index_revision: Some(QUERY_TIME_INDEX_REVISION.to_string()),
        status: EpiphanyRetrievalStatus::Unavailable,
        semantic_available: false,
        last_indexed_at_unix_seconds: None,
        indexed_file_count: None,
        indexed_chunk_count: None,
        shards: vec![EpiphanyRetrievalShardSummary {
            shard_id: WORKSPACE_SHARD_ID.to_string(),
            path_prefix: PathBuf::from("."),
            indexed_file_count: None,
            indexed_chunk_count: None,
            status: EpiphanyRetrievalStatus::Unavailable,
            exact_available: false,
            semantic_available: false,
        }],
        dirty_paths: Vec::new(),
    }
}

fn baseline_retrieval_state(workspace_root: &Path) -> EpiphanyRetrievalState {
    EpiphanyRetrievalState {
        workspace_root: workspace_root.to_path_buf(),
        index_revision: Some(QUERY_TIME_INDEX_REVISION.to_string()),
        status: EpiphanyRetrievalStatus::Ready,
        semantic_available: true,
        last_indexed_at_unix_seconds: None,
        indexed_file_count: None,
        indexed_chunk_count: None,
        shards: vec![EpiphanyRetrievalShardSummary {
            shard_id: WORKSPACE_SHARD_ID.to_string(),
            path_prefix: PathBuf::from("."),
            indexed_file_count: None,
            indexed_chunk_count: None,
            status: EpiphanyRetrievalStatus::Ready,
            exact_available: true,
            semantic_available: true,
        }],
        dirty_paths: Vec::new(),
    }
}

fn query_time_retrieval_state(
    workspace_root: &Path,
    searchable_file_count: usize,
    chunk_count: usize,
) -> EpiphanyRetrievalState {
    let indexed_file_count = u32::try_from(searchable_file_count).ok();
    let indexed_chunk_count = u32::try_from(chunk_count).ok();

    EpiphanyRetrievalState {
        workspace_root: workspace_root.to_path_buf(),
        index_revision: Some(QUERY_TIME_INDEX_REVISION.to_string()),
        status: EpiphanyRetrievalStatus::Ready,
        semantic_available: chunk_count > 0,
        last_indexed_at_unix_seconds: Some(chrono::Utc::now().timestamp()),
        indexed_file_count,
        indexed_chunk_count,
        shards: vec![EpiphanyRetrievalShardSummary {
            shard_id: WORKSPACE_SHARD_ID.to_string(),
            path_prefix: PathBuf::from("."),
            indexed_file_count,
            indexed_chunk_count,
            status: EpiphanyRetrievalStatus::Ready,
            exact_available: true,
            semantic_available: chunk_count > 0,
        }],
        dirty_paths: Vec::new(),
    }
}

fn manifest_to_retrieval_state(
    workspace_root: &Path,
    manifest: &SemanticIndexManifest,
    status: EpiphanyRetrievalStatus,
    dirty_paths: Vec<PathBuf>,
) -> EpiphanyRetrievalState {
    let indexed_file_count = u32::try_from(manifest.files.len()).ok();
    let indexed_chunk_count = manifest
        .files
        .iter()
        .try_fold(0u32, |count, file| count.checked_add(file.chunk_count));
    let semantic_available = indexed_chunk_count.unwrap_or(0) > 0;

    EpiphanyRetrievalState {
        workspace_root: workspace_root.to_path_buf(),
        index_revision: Some(manifest.index_revision.clone()),
        status,
        semantic_available,
        last_indexed_at_unix_seconds: Some(manifest.indexed_at_unix_seconds),
        indexed_file_count,
        indexed_chunk_count,
        shards: vec![EpiphanyRetrievalShardSummary {
            shard_id: WORKSPACE_SHARD_ID.to_string(),
            path_prefix: PathBuf::from("."),
            indexed_file_count,
            indexed_chunk_count,
            status,
            exact_available: true,
            semantic_available,
        }],
        dirty_paths,
    }
}

fn sort_results(results: &mut [EpiphanyRetrieveResult]) {
    results.sort_by(|left, right| {
        result_priority(&right.kind)
            .cmp(&result_priority(&left.kind))
            .then_with(|| {
                right
                    .score
                    .partial_cmp(&left.score)
                    .unwrap_or(Ordering::Equal)
            })
            .then_with(|| left.path.cmp(&right.path))
            .then_with(|| {
                left.line_start
                    .unwrap_or(u32::MAX)
                    .cmp(&right.line_start.unwrap_or(u32::MAX))
            })
    });
}

fn result_priority(kind: &EpiphanyRetrieveResultKind) -> u8 {
    match kind {
        EpiphanyRetrieveResultKind::ExactFile => 3,
        EpiphanyRetrieveResultKind::SemanticChunk => 2,
        EpiphanyRetrieveResultKind::ExactDirectory => 1,
    }
}

fn semantic_candidate_limit(limit: usize, path_prefixes: &[PathBuf]) -> usize {
    if path_prefixes.is_empty() {
        limit
    } else {
        limit
            .saturating_mul(EXACT_SEARCH_CANDIDATE_MULTIPLIER)
            .min(EXACT_SEARCH_MAX_CANDIDATES)
    }
}

fn normalize_path_prefixes(workspace_root: &Path, path_prefixes: &[PathBuf]) -> Vec<PathBuf> {
    let mut normalized = path_prefixes
        .iter()
        .filter_map(|prefix| normalize_path_prefix(workspace_root, prefix.as_path()))
        .collect::<Vec<_>>();
    normalized.sort();
    normalized.dedup();
    normalized
}

fn normalize_path_prefix(workspace_root: &Path, prefix: &Path) -> Option<PathBuf> {
    if prefix.as_os_str().is_empty() || prefix == Path::new(".") {
        return None;
    }

    let relative = if prefix.is_absolute() {
        prefix.strip_prefix(workspace_root).ok()?
    } else {
        prefix
    };

    let mut normalized = PathBuf::new();
    for component in relative.components() {
        match component {
            Component::Normal(value) => normalized.push(value),
            Component::CurDir => {}
            Component::ParentDir | Component::Prefix(_) | Component::RootDir => return None,
        }
    }

    if normalized.as_os_str().is_empty() || normalized == Path::new(".") {
        None
    } else {
        Some(normalized)
    }
}

fn matches_path_prefixes(path: &Path, path_prefixes: &[PathBuf]) -> bool {
    path_prefixes.is_empty() || path_prefixes.iter().any(|prefix| path.starts_with(prefix))
}

fn should_skip_relative_path(relative_path: &Path) -> bool {
    relative_path
        .components()
        .any(|component| matches!(component, Component::Normal(value) if value == ".git"))
}

fn chunk_point_id(collection_name: &str, chunk: &SemanticChunk) -> String {
    let point_key = format!(
        "{collection_name}\n{}\n{}\n{}",
        normalize_payload_path(&chunk.path),
        chunk.line_start,
        chunk.line_end
    );
    Uuid::new_v5(&Uuid::NAMESPACE_URL, point_key.as_bytes()).to_string()
}

fn normalize_payload_path(path: &Path) -> String {
    path.components()
        .filter_map(|component| match component {
            Component::Normal(value) => Some(value.to_string_lossy().to_string()),
            Component::CurDir => None,
            _ => None,
        })
        .collect::<Vec<_>>()
        .join("/")
}

fn validate_embedding_batch(embeddings: &[Vec<f32>], expected_count: usize) -> Result<()> {
    if embeddings.len() != expected_count {
        anyhow::bail!(
            "embedding backend returned {} vectors for {} chunks",
            embeddings.len(),
            expected_count
        );
    }

    let Some(vector_length) = embeddings.first().map(Vec::len) else {
        return Ok(());
    };
    if vector_length == 0 {
        anyhow::bail!("embedding backend returned an empty vector");
    }

    for (index, embedding) in embeddings.iter().enumerate() {
        if embedding.len() != vector_length {
            anyhow::bail!("embedding backend returned inconsistent vector length at item {index}");
        }
    }

    Ok(())
}

fn unix_timestamp_seconds(time: SystemTime) -> Result<i64> {
    Ok(i64::try_from(
        time.duration_since(UNIX_EPOCH)
            .context("timestamp before unix epoch")?
            .as_secs(),
    )?)
}

fn unix_timestamp_nanos(time: SystemTime) -> Result<i64> {
    let duration = time
        .duration_since(UNIX_EPOCH)
        .context("timestamp before unix epoch")?;
    let nanos = duration
        .as_secs()
        .checked_mul(1_000_000_000)
        .and_then(|seconds| seconds.checked_add(u64::from(duration.subsec_nanos())))
        .context("timestamp overflow")?;
    Ok(i64::try_from(nanos)?)
}

fn normalize_base_url(input: &str) -> String {
    input.trim_end_matches('/').to_string()
}

fn timeout_seconds(timeout_ms: u64) -> u64 {
    timeout_ms.div_ceil(1000).max(1)
}

fn env_override(primary: &str, fallback: &str) -> Option<String> {
    std::env::var(primary)
        .ok()
        .filter(|value| !value.trim().is_empty())
        .or_else(|| {
            std::env::var(fallback)
                .ok()
                .filter(|value| !value.trim().is_empty())
        })
}

fn hex_lower(bytes: &[u8]) -> String {
    bytes.iter().map(|byte| format!("{byte:02x}")).collect()
}

#[cfg(test)]
mod tests {
    use super::*;
    use anyhow::Result;
    use pretty_assertions::assert_eq;
    use tempfile::TempDir;
    use wiremock::Mock;
    use wiremock::MockServer;
    use wiremock::ResponseTemplate;
    use wiremock::matchers::method;
    use wiremock::matchers::path;

    #[test]
    fn retrieve_workspace_prioritizes_exact_hits_without_losing_semantic_chunks() -> Result<()> {
        let temp_dir = TempDir::new()?;
        let codex_home = TempDir::new()?;
        fs::create_dir_all(temp_dir.path().join("src"))?;
        fs::create_dir_all(temp_dir.path().join("notes"))?;
        fs::write(
            temp_dir.path().join("src").join("session_checkpoint.rs"),
            "pub fn route_session() {}\n",
        )?;
        fs::write(
            temp_dir.path().join("notes").join("design.md"),
            "The compaction checkpoint keeps the field notebook alive.\n\
             Retrieval should honor the checkpoint before the next pass.\n",
        )?;

        let response = retrieve_workspace(
            temp_dir.path(),
            codex_home.path(),
            EpiphanyRetrieveQuery {
                query: "session checkpoint".to_string(),
                limit: 5,
                path_prefixes: Vec::new(),
            },
        )?;

        assert_eq!(
            response.results.first().map(|result| &result.kind),
            Some(&EpiphanyRetrieveResultKind::ExactFile)
        );
        assert!(
            response
                .results
                .iter()
                .any(|result| result.kind == EpiphanyRetrieveResultKind::SemanticChunk),
            "expected at least one semantic chunk result"
        );
        Ok(())
    }

    #[test]
    fn retrieve_workspace_finds_semantic_match_without_path_match() -> Result<()> {
        let temp_dir = TempDir::new()?;
        let codex_home = TempDir::new()?;
        fs::create_dir_all(temp_dir.path().join("src"))?;
        fs::write(
            temp_dir.path().join("src").join("engine.rs"),
            "The frontier checkpoint preserves the machine map before compaction.\n\
             Keep the durable spine and discard disposable scratch.\n",
        )?;

        let response = retrieve_workspace(
            temp_dir.path(),
            codex_home.path(),
            EpiphanyRetrieveQuery {
                query: "durable spine compaction checkpoint".to_string(),
                limit: 3,
                path_prefixes: Vec::new(),
            },
        )?;

        assert!(
            response.results.iter().any(|result| {
                result.kind == EpiphanyRetrieveResultKind::SemanticChunk
                    && result.path == PathBuf::from("src").join("engine.rs")
            }),
            "expected semantic retrieval to find engine.rs"
        );
        Ok(())
    }

    #[test]
    fn retrieve_workspace_respects_path_prefixes() -> Result<()> {
        let temp_dir = TempDir::new()?;
        let codex_home = TempDir::new()?;
        fs::create_dir_all(temp_dir.path().join("src"))?;
        fs::create_dir_all(temp_dir.path().join("docs"))?;
        fs::write(
            temp_dir.path().join("src").join("router.rs"),
            "checkpoint frontier checkpoint frontier\n",
        )?;
        fs::write(
            temp_dir.path().join("docs").join("guide.md"),
            "checkpoint frontier checkpoint frontier\n",
        )?;

        let response = retrieve_workspace(
            temp_dir.path(),
            codex_home.path(),
            EpiphanyRetrieveQuery {
                query: "checkpoint frontier".to_string(),
                limit: 10,
                path_prefixes: vec![PathBuf::from("docs")],
            },
        )?;

        assert!(
            response
                .results
                .iter()
                .all(|result| result.path.starts_with("docs")),
            "all results should stay inside the requested path prefix"
        );
        Ok(())
    }

    #[test]
    fn index_workspace_builds_manifest_and_retrieve_uses_qdrant_when_fresh() -> Result<()> {
        let temp_dir = TempDir::new()?;
        let codex_home = TempDir::new()?;
        fs::create_dir_all(temp_dir.path().join("src"))?;
        fs::write(
            temp_dir.path().join("src").join("router.rs"),
            "The checkpoint frontier keeps the durable spine intact.\n",
        )?;

        let runtime = tokio::runtime::Builder::new_current_thread()
            .enable_all()
            .build()?;
        let qdrant = runtime.block_on(MockServer::start());
        let ollama = runtime.block_on(MockServer::start());
        let config = SemanticBackendConfig {
            qdrant: QdrantConfig {
                url: qdrant.uri(),
                api_key: None,
                timeout_ms: 5_000,
            },
            ollama: OllamaConfig {
                base_url: ollama.uri(),
                model: "qwen3-embedding:0.6b".to_string(),
                timeout_ms: 5_000,
                query_instruction: DEFAULT_QUERY_INSTRUCTION.to_string(),
            },
        };

        runtime.block_on(async {
            Mock::given(method("GET"))
                .and(path(format!(
                    "/collections/{}/exists",
                    config.collection_name(temp_dir.path())
                )))
                .respond_with(ResponseTemplate::new(200).set_body_json(json!({
                    "result": { "exists": false }
                })))
                .expect(1)
                .mount(&qdrant)
                .await;

            Mock::given(method("PUT"))
                .and(path(format!(
                    "/collections/{}",
                    config.collection_name(temp_dir.path())
                )))
                .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "result": true })))
                .expect(1)
                .mount(&qdrant)
                .await;

            Mock::given(method("PUT"))
                .and(path(format!(
                    "/collections/{}/points",
                    config.collection_name(temp_dir.path())
                )))
                .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "result": true })))
                .expect(1)
                .mount(&qdrant)
                .await;

            Mock::given(method("POST"))
                .and(path("/api/embed"))
                .respond_with(ResponseTemplate::new(200).set_body_json(json!({
                    "embeddings": [[0.1_f32, 0.2_f32, 0.3_f32]]
                })))
                .expect(1)
                .mount(&ollama)
                .await;
        });

        let summary =
            index_workspace_with_config(temp_dir.path(), codex_home.path(), false, &config)?;
        assert_eq!(summary.status, EpiphanyRetrievalStatus::Ready);
        assert_eq!(summary.indexed_file_count, Some(1));

        runtime.block_on(async {
            qdrant.reset().await;
            ollama.reset().await;

            Mock::given(method("GET"))
                .and(path(format!(
                    "/collections/{}/exists",
                    config.collection_name(temp_dir.path())
                )))
                .respond_with(ResponseTemplate::new(200).set_body_json(json!({
                    "result": { "exists": true }
                })))
                .expect(1)
                .mount(&qdrant)
                .await;

            Mock::given(method("POST"))
                .and(path("/api/embed"))
                .respond_with(ResponseTemplate::new(200).set_body_json(json!({
                    "embeddings": [[0.1_f32, 0.2_f32, 0.3_f32]]
                })))
                .expect(1)
                .mount(&ollama)
                .await;

            Mock::given(method("POST"))
                .and(path(format!(
                    "/collections/{}/points/query",
                    config.collection_name(temp_dir.path())
                )))
                .respond_with(ResponseTemplate::new(200).set_body_json(json!({
                    "result": {
                        "points": [
                            {
                                "score": 0.93,
                                "payload": {
                                    "path": "src/router.rs",
                                    "line_start": 1,
                                    "line_end": 1,
                                    "excerpt": "The checkpoint frontier keeps the durable spine intact."
                                }
                            }
                        ]
                    }
                })))
                .expect(1)
                .mount(&qdrant)
                .await;
        });

        let response = retrieve_workspace_with_config(
            temp_dir.path(),
            codex_home.path(),
            EpiphanyRetrieveQuery::new("durable spine".to_string()),
            &config,
        )?;
        assert_eq!(
            response.index_summary.index_revision,
            Some(config.index_revision())
        );
        assert!(response.results.iter().any(|result| {
            result.kind == EpiphanyRetrieveResultKind::SemanticChunk
                && result.path == PathBuf::from("src/router.rs")
        }));
        drop(qdrant);
        drop(ollama);
        drop(runtime);
        Ok(())
    }

    #[test]
    fn retrieval_state_marks_manifest_stale_after_workspace_change() -> Result<()> {
        let temp_dir = TempDir::new()?;
        let codex_home = TempDir::new()?;
        fs::create_dir_all(temp_dir.path().join("src"))?;
        fs::write(
            temp_dir.path().join("src").join("router.rs"),
            "The checkpoint frontier keeps the durable spine intact.\n",
        )?;

        let config = SemanticBackendConfig::from_env();
        let snapshot = collect_workspace_index_snapshot(temp_dir.path())?;
        let manifest = SemanticIndexManifest {
            schema_version: SEMANTIC_MANIFEST_SCHEMA_VERSION,
            workspace_root: temp_dir.path().to_path_buf(),
            collection_name: config.collection_name(temp_dir.path()),
            index_revision: config.index_revision(),
            indexed_at_unix_seconds: 1_744_500_000,
            files: snapshot
                .files
                .iter()
                .map(|(path, file)| SemanticIndexedFile {
                    path: path.clone(),
                    size_bytes: file.size_bytes,
                    modified_unix_nanos: file.modified_unix_nanos,
                    chunk_count: file.chunk_count,
                    point_ids: vec!["point-1".to_string()],
                })
                .collect(),
        };
        write_semantic_manifest(
            &config.manifest_path(codex_home.path(), temp_dir.path()),
            &manifest,
        )?;

        fs::write(
            temp_dir.path().join("src").join("router.rs"),
            "The checkpoint frontier moved and now the file changed.\n",
        )?;

        let state = retrieval_state_for_workspace(temp_dir.path(), codex_home.path());
        assert_eq!(state.status, EpiphanyRetrievalStatus::Stale);
        assert_eq!(state.dirty_paths, vec![PathBuf::from("src/router.rs")]);
        Ok(())
    }

    fn retrieve_workspace_with_config(
        workspace_root: &Path,
        codex_home: &Path,
        query: EpiphanyRetrieveQuery,
        config: &SemanticBackendConfig,
    ) -> Result<EpiphanyRetrieveResponse> {
        let query_text = query.query.trim();
        let limit = query.limit.clamp(1, EPIPHANY_RETRIEVAL_MAX_LIMIT);
        let path_prefixes = normalize_path_prefixes(workspace_root, &query.path_prefixes);
        let exact_results = run_exact_search(workspace_root, query_text, limit, &path_prefixes)?;
        let semantic_limit = semantic_candidate_limit(limit, &path_prefixes);
        let persistent_search = search_persistent_semantic_chunks_with_config(
            workspace_root,
            codex_home,
            query_text,
            semantic_limit,
            config,
        )?;

        let (index_summary, semantic_results) = match persistent_search {
            PersistentSemanticSearch::Ready { summary, results } => (summary, results),
            PersistentSemanticSearch::Fallback { summary } => (summary.unwrap(), Vec::new()),
            PersistentSemanticSearch::None => {
                (baseline_retrieval_state(workspace_root), Vec::new())
            }
        };

        let mut results = exact_results;
        results.extend(semantic_results);
        sort_results(&mut results);
        Ok(EpiphanyRetrieveResponse {
            query: query_text.to_string(),
            index_summary,
            results,
        })
    }

    fn search_persistent_semantic_chunks_with_config(
        workspace_root: &Path,
        codex_home: &Path,
        query: &str,
        limit: usize,
        config: &SemanticBackendConfig,
    ) -> Result<PersistentSemanticSearch> {
        let manifest_path = config.manifest_path(codex_home, workspace_root);
        let Some(manifest) = load_semantic_manifest(&manifest_path)? else {
            return Ok(PersistentSemanticSearch::None);
        };
        let snapshot = collect_workspace_index_snapshot(workspace_root)?;
        let dirty_paths = dirty_paths_from_manifest(&manifest, &snapshot);
        if !dirty_paths.is_empty() {
            return Ok(PersistentSemanticSearch::Fallback {
                summary: Some(manifest_to_retrieval_state(
                    workspace_root,
                    &manifest,
                    EpiphanyRetrievalStatus::Stale,
                    dirty_paths,
                )),
            });
        }

        let qdrant = QdrantClient::new(config.qdrant.clone())?;
        if !qdrant.collection_exists(&manifest.collection_name)? {
            return Ok(PersistentSemanticSearch::Fallback {
                summary: Some(manifest_to_retrieval_state(
                    workspace_root,
                    &manifest,
                    EpiphanyRetrievalStatus::Stale,
                    Vec::new(),
                )),
            });
        }

        let ollama = OllamaEmbedder::new(config.ollama.clone())?;
        let query_vector = ollama.embed_query(query)?;
        let hits = qdrant.query_points(&manifest.collection_name, &query_vector, limit)?;
        Ok(PersistentSemanticSearch::Ready {
            summary: manifest_to_retrieval_state(
                workspace_root,
                &manifest,
                EpiphanyRetrievalStatus::Ready,
                Vec::new(),
            ),
            results: hits
                .into_iter()
                .filter_map(map_qdrant_hit_to_result)
                .collect(),
        })
    }
}