Merge pull request 'Source search takes up to eight texts, fused by reciprocal rank (k=60)' (#52) from gamecult-chat/source-query-texts into main

crates/eureka-state/src/bin/huginn/main.rs

@@ -99,8 +99,11 @@ enum Command {
     },
     /// Search the public forge repos' code and docs by meaning.
     Sources {
-        /// What to find, in words.
+        /// What to find, in words: one to eight texts. Each is a topic; the
-        text: String,
+        /// ranked lists of several are fused, so a brief with many topics
+        /// retrieves by every one.
+        #[arg(required = true, num_args = 1..=8)]
+        texts: Vec<String>,
         /// Look in this repo alone (its forge name).
         #[arg(long)]
         repo: Option<String>,
@@ -275,12 +278,12 @@ fn run(settings: &Settings, command: Command) -> Result<u8, Trouble> {
             query(settings, selection, semantic, None, documents)
         }
         Command::Sources {
-            text,
+            texts,
             repo,
             front,
             top_k,
             json,
-        } => sources(settings, text, repo, front, top_k, json),
+        } => sources(settings, texts, repo, front, top_k, json),
         Command::Recipe(Recipe::Blocked { spec_id, page }) => {
             blocked(settings, &spec_id, page.cursor, page.documents)
         }
@@ -438,14 +441,15 @@ fn blocked(
 
 fn sources(
     settings: &Settings,
-    text: String,
+    texts: Vec<String>,
     repo: Option<String>,
     front: bool,
     top_k: Option<u32>,
     json: bool,
 ) -> Result<u8, Trouble> {
     let query = SourceQuery {
-        text: Line(text),
+        text: Line::default(),
+        texts: texts.into_iter().map(Line).collect(),
         top_k: top_k.unwrap_or(DEFAULT_TOP_K),
         repo: repo.map(Short),
         front,

crates/eureka-state/tests/cli.rs

@@ -1383,6 +1383,31 @@ fn sources_front_prints_only_the_front_page_with_its_lines() {
     assert_eq!(found.out.trim_end(), "Demo:README.md:1-2 @abcdef1 0.900\n# Demo", "{}", found.out);
 }
 
+/// `huginn sources <TEXT>...` sends every text in `texts` and none in `text`, the
+/// daemon fuses them into one answer, and a ninth or no text never leaves the CLI.
+#[test]
+fn sources_sends_each_text_as_a_topic_and_prints_the_fused_answer() {
+    let (server, seen) = replying(
+        "search_sources",
+        HuginnMindResponse::Sources(huginn_mind::SourceHits { indexed_under: huginn_mind::SourceModel { name: "m".into(), digest: "d".into() }, hits: vec![] }),
+    );
+    let ran = at(server.addr, &["sources", "qdrant filter", "reciprocal rank", "--top-k", "5"], None);
+    assert_eq!(ran.code, 0, "{}", ran.err);
+    let HuginnMindRequest::SearchSources { query, .. } = seen.lock().unwrap().remove(0) else { panic!("a source search") };
+    let texts: Vec<&str> = query.texts.iter().map(|text| text.0.as_str()).collect();
+    assert_eq!((texts, query.text.0.as_str(), query.top_k), (vec!["qdrant filter", "reciprocal rank"], "", 5));
+
+    let daemon = source_daemon("d1", "d1");
+    let found = at(daemon.addr, &["sources", "cursor", "readme", "--repo", "Demo"], None);
+    assert_eq!(found.code, 0, "{}", found.err);
+    assert!(found.out.contains("Demo:src/lib.rs:3-9") && found.out.contains("Demo:README.md:1-2"), "{}", found.out);
+
+    let nine = ["sources", "a", "b", "c", "d", "e", "f", "g", "h", "i"];
+    assert_ne!(at(server.addr, &nine, None).code, 0);
+    assert_ne!(at(server.addr, &["sources"], None).code, 0);
+    assert!(seen.lock().unwrap().is_empty(), "a refused command asked nothing");
+}
+
 /// A daemon whose embedder is not the model the index was built under refuses
 /// the search naming both, and the CLI exits 1 with that word.
 #[test]

crates/huginn-daemon/src/daemon.rs

@@ -655,7 +655,7 @@ pub(crate) mod tests {
         let mut daemon = Daemon::new(mind, SourcesIndex { asked: Arc::clone(&asked), hits: hits.clone() });
         let request = |instance: &str, top_k: u32| HuginnMindRequest::SearchSources {
             instance: slug(instance),
-            query: SourceQuery { text: huginn_mind::eureka_pipeline::Line("find".into()), top_k, repo: None, front: false },
+            query: SourceQuery { text: huginn_mind::eureka_pipeline::Line("find".into()), texts: vec![], top_k, repo: None, front: false },
         };
 
         let Handled::Searching(search) = daemon.handle(request(INSTANCE, 5), now()) else { panic!("a well-formed source search is a search") };
@@ -673,7 +673,16 @@ pub(crate) mod tests {
             let answer = daemon.handle(refused, now()).answered();
             assert!(matches!(answer, HuginnMindResponse::Refused(_)), "{answer:?}");
         }
-        assert_eq!(*asked.lock().unwrap(), 2, "a refused query never reached the index");
+        let texts = |texts: &[&str]| HuginnMindRequest::SearchSources {
+            instance: slug(INSTANCE),
+            query: SourceQuery { text: huginn_mind::eureka_pipeline::Line::default(), texts: texts.iter().map(|text| huginn_mind::eureka_pipeline::Line((*text).into())).collect(), top_k: 5, repo: None, front: false },
+        };
+        for refused in [texts(&[]), texts(&["a"; 9]), texts(&["a", " "])] {
+            let answer = daemon.handle(refused, now()).answered();
+            assert!(matches!(answer, HuginnMindResponse::Refused(MindRefusal::SelectionInvalid { ref field, .. }) if field == "texts"), "{answer:?}");
+        }
+        assert!(matches!(daemon.handle(texts(&["a", "b"]), now()), Handled::Searching(_)), "two texts are one queued search");
+        assert_eq!(*asked.lock().unwrap(), 3, "a refused query never reached the index");
     }
 
     /// A daemon over the instance's mind under `state_root`, with no index.

crates/huginn-daemon/src/index/tests.rs

@@ -1995,7 +1995,7 @@ fn the_sink_owns_the_searchs_deadline_and_tells_the_loop() {
 }
 
 fn ask_sources(sink: &mut WorkerSink, text: &str) -> Result<SearchTicket, String> {
-    let query = SourceQuery { text: huginn_mind::eureka_pipeline::Line(text.into()), top_k: 5, repo: None, front: false };
+    let query = SourceQuery { text: huginn_mind::eureka_pipeline::Line(text.into()), texts: vec![], top_k: 5, repo: None, front: false };
     <WorkerSink as IndexSink<OwnedRedbMessagePackBackingStore>>::search_sources(sink, &query)
 }
 

crates/huginn-daemon/src/sources/mod.rs

@@ -24,12 +24,13 @@ pub mod git;
 #[cfg(test)]
 mod tests;
 
-use std::collections::{BTreeMap, BTreeSet};
+use std::collections::hash_map::Entry;
+use std::collections::{BTreeMap, BTreeSet, HashMap};
 use std::path::Path;
 use std::time::{Duration, Instant};
 
 use anyhow::{Context, Result, bail, ensure};
-use huginn_mind::{SourceHit, SourceHits, SourceModel, SourceQuery};
+use huginn_mind::{SOURCE_TOP_K_MAX, SourceHit, SourceHits, SourceModel, SourceQuery};
 
 use crate::index::{BATCH, BULK_EMBED_TIMEOUT, Budget, Embedder, INDEX_CALL_TIMEOUT, MANAGED_BY, ModelIdentity, point_id};
 use chunk::{CHUNKER_VERSION, embedded_text};
@@ -49,6 +50,14 @@ pub const CORPUS: &str = "forge:GameCult";
 pub const SOURCE_QUERY_INSTRUCTION: &str =
     "Given a question about a codebase, retrieve the source code and documentation passages that answer it";
 
+/// The constant of reciprocal rank fusion, score = sum over lists of
+/// 1 / (FUSE_K + rank), rank 1-based: Cormack, Clarke and Buttcher, SIGIR 2009,
+/// whose value this is. The one calibration constant of a multi-text search.
+pub const FUSE_K: u32 = 60;
+
+/// How many chunks each text's list holds before the lists are fused.
+pub const FUSE_DEPTH: u32 = SOURCE_TOP_K_MAX;
+
 /// Paths named in one delete call.
 const DELETE_PATHS_PER_CALL: usize = 256;
 
@@ -438,6 +447,44 @@ impl Batch {
     }
 }
 
+/// Reciprocal rank fusion of ranked lists (best first): an item's score is the
+/// sum over the lists that hold it of `1 / (k + rank)`, rank 1-based, and items
+/// with the same `key` are one item (the first list's copy is kept). Best first,
+/// the first-seen item ahead on a tie, cut to `top_k`. Pure: the one fuse, which
+/// every search that ranks across lists calls and none copies.
+pub fn fuse<T>(lists: Vec<Vec<T>>, key: impl Fn(&T) -> String, k: u32, top_k: usize) -> Vec<(f32, T)> {
+    let mut seen: HashMap<String, usize> = HashMap::new();
+    let mut fused: Vec<(f32, T)> = Vec::new();
+    for list in lists {
+        for (rank, item) in list.into_iter().enumerate() {
+            let share = 1.0 / (k as f32 + rank as f32 + 1.0);
+            match seen.entry(key(&item)) {
+                Entry::Occupied(at) => fused[*at.get()].0 += share,
+                Entry::Vacant(at) => {
+                    at.insert(fused.len());
+                    fused.push((share, item));
+                }
+            }
+        }
+    }
+    fused.sort_by(|a, b| b.0.total_cmp(&a.0));
+    fused.truncate(top_k);
+    fused
+}
+
+/// The texts a query searches by: `texts`, or an old client's `text` as a list
+/// of one, trimmed, and each distinct text once, in the order asked.
+fn query_texts(query: &SourceQuery) -> Vec<&str> {
+    let asked: Vec<&str> = if query.texts.is_empty() { vec![query.text.0.as_str()] } else { query.texts.iter().map(|text| text.0.as_str()).collect() };
+    let mut distinct: Vec<&str> = Vec::with_capacity(asked.len());
+    for text in asked {
+        if !distinct.iter().any(|seen| seen.trim() == text.trim()) {
+            distinct.push(text);
+        }
+    }
+    distinct
+}
+
 /// A source search, which a daemon answers from the collection as it finds it:
 /// it embeds the query with its own embedder, and refuses if the collection
 /// was built under another model, because vectors of two models do not compare.
@@ -459,12 +506,23 @@ pub fn search<E: Embedder, S: SourceIndex>(embedder: &mut E, index: &mut S, quer
         identity.digest,
         identity.dimensions
     );
-    let text = format!("Instruct: {SOURCE_QUERY_INSTRUCTION}\nQuery: {}", query.text.0);
+    let topics = query_texts(query);
-    let mut vectors = embedder.embed(&[text], budget.left()?).context("embedding the query")?;
+    let prompts: Vec<String> = topics.iter().map(|topic| format!("Instruct: {SOURCE_QUERY_INSTRUCTION}\nQuery: {topic}")).collect();
-    ensure!(vectors.len() == 1, "the embedder returned {} vectors for one query", vectors.len());
+    let vectors = embedder.embed(&prompts, budget.left()?).context("embedding the query")?;
-    let vector = vectors.remove(0);
+    ensure!(vectors.len() == prompts.len(), "the embedder returned {} vectors for {} query texts", vectors.len(), prompts.len());
-    ensure!(vector.len() == identity.dimensions as usize, "the embedder returned a query vector of {} where the collection holds {}", vector.len(), identity.dimensions);
+    let repo = query.repo.as_ref().map(|repo| repo.0.as_str());
-    let rows = index.search_sources(&vector, query.repo.as_ref().map(|repo| repo.0.as_str()), query.front, query.top_k, budget.left()?)?;
+    let depth = if vectors.len() == 1 { query.top_k } else { FUSE_DEPTH };
+    let mut lists = Vec::with_capacity(vectors.len());
+    for vector in &vectors {
+        ensure!(vector.len() == identity.dimensions as usize, "the embedder returned a query vector of {} where the collection holds {}", vector.len(), identity.dimensions);
+        lists.push(index.search_sources(vector, repo, query.front, depth, budget.left()?)?);
+    }
+    let rows = if lists.len() == 1 {
+        lists.remove(0)
+    } else {
+        let key = |row: &SourceRow| chunk_id(&row.payload.repo, &row.payload.path, &row.payload.blob, row.payload.chunk_index);
+        fuse(lists, key, FUSE_K, query.top_k as usize).into_iter().map(|(score, row)| SourceRow { score, ..row }).collect()
+    };
     Ok(SourceHits {
         indexed_under: SourceModel { name: built.model, digest: built.model_digest },
         hits: rows

crates/huginn-daemon/src/sources/tests.rs

@@ -505,7 +505,7 @@ fn front_query(text: &str, top_k: u32, repo: Option<&str>) -> SourceQuery {
 }
 
 fn query(text: &str, top_k: u32, repo: Option<&str>) -> SourceQuery {
-    SourceQuery { text: Line(text.into()), top_k, repo: repo.map(|repo| Short(repo.into())), front: false }
+    SourceQuery { text: Line(text.into()), texts: vec![], top_k, repo: repo.map(|repo| Short(repo.into())), front: false }
 }
 
 /// A daemon whose embedder is not the one the collection was built under
@@ -606,3 +606,152 @@ fn chunks_are_embedded_in_bounded_batches() {
     assert_eq!(sizes, [BATCH, BATCH, 6]);
     assert_eq!(rig.index.held().len(), 70);
 }
+
+fn fused(lists: &[&[&str]], top_k: usize) -> Vec<(f32, String)> {
+    fuse(lists.iter().map(|list| list.iter().map(|item| (*item).to_owned()).collect()).collect(), |item| item.clone(), FUSE_K, top_k)
+}
+
+/// A chunk that two lists put first outranks one that a single list puts first:
+/// 1/61 + 1/61 against 1/61, with k = 60 and ranks counted from 1.
+#[test]
+fn a_chunk_first_in_two_lists_outranks_a_chunk_first_in_one() {
+    let found = fused(&[&["both", "x"], &["both", "y"]], 10);
+    let names: Vec<&str> = found.iter().map(|(_, item)| item.as_str()).collect();
+    assert_eq!(names[0], "both");
+    assert!((found[0].0 - 2.0 / 61.0).abs() < 1e-6, "{}", found[0].0);
+    assert!((found[1].0 - 1.0 / 62.0).abs() < 1e-6, "second place in one list is 1/(60+2)");
+    assert_eq!(names, ["both", "x", "y"], "a tie keeps the earlier list's item ahead");
+    let deeper = fused(&[&["a", "b", "c"], &["c"]], 10);
+    assert_eq!(deeper[0].1, "c", "1/63 + 1/61 beats 1/61");
+}
+
+/// The fused list is cut to `top_k`, and an empty input fuses to nothing.
+#[test]
+fn a_fused_list_is_cut_to_top_k() {
+    assert_eq!(fused(&[&["a", "b", "c"]], 2).len(), 2);
+    assert!(fused(&[], 5).is_empty());
+}
+
+/// An embedder whose vector is the axis of the topic a text names (`topic-N`
+/// anywhere in it), so a test says which chunk each text is near.
+struct TopicEmbedder(FakeEmbedder);
+
+impl Embedder for TopicEmbedder {
+    fn model_identity(&mut self, within: Duration) -> Result<crate::index::ModelIdentity> {
+        self.0.model_identity(within)
+    }
+
+    fn embed(&mut self, texts: &[String], within: Duration) -> Result<Vec<Vec<f32>>> {
+        self.0.embed(texts, within)?;
+        let dimensions = self.0.state.lock().unwrap().identity.dimensions as usize;
+        Ok(texts
+            .iter()
+            .map(|text| {
+                let axis: usize = text.split("topic-").nth(1).and_then(|rest| rest.get(..1)).and_then(|digit| digit.parse().ok()).expect("a text names topic-N");
+                (0..dimensions).map(|at| if at == axis { 1.0 } else { 0.0 }).collect()
+            })
+            .collect())
+    }
+}
+
+/// A rig whose collection holds one chunk per axis, `chunk-N` near `topic-N`,
+/// built under an embedder of `dimensions`.
+fn topic_rig(dimensions: u32) -> (Rig, TopicEmbedder) {
+    let remote = Remote::new();
+    remote.commit(&[("seed.rs", b"seed\n")], &[]);
+    let mut rig = Rig::new();
+    rig.embedder.state.lock().unwrap().identity.dimensions = dimensions;
+    rig.converged(&mut FakeForge::of(&[("Demo", &remote)]));
+    {
+        let mut state = rig.index.state.lock().unwrap();
+        state.points.clear();
+        for axis in 0..dimensions as usize {
+            let payload = SourcePayload {
+                repo: "Demo".into(),
+                path: format!("chunk-{axis}.rs"),
+                commit: "c".into(),
+                blob: format!("b{axis}"),
+                line_start: 1,
+                line_end: 1,
+                chunk_index: 0,
+                ext: "rs".into(),
+                text: format!("chunk {axis}"),
+            };
+            let id = chunk_id(&payload.repo, &payload.path, &payload.blob, 0);
+            state.points.insert(id.clone(), SourcePoint { id, vector: (0..dimensions as usize).map(|at| if at == axis { 1.0 } else { 0.0 }).collect(), payload });
+        }
+        state.searches.clear();
+    }
+    rig.embedder.state.lock().unwrap().embedded.clear();
+    let embedder = TopicEmbedder(rig.embedder.clone());
+    (rig, embedder)
+}
+
+fn queries(texts: &[&str], top_k: u32) -> SourceQuery {
+    SourceQuery { text: Line::default(), texts: texts.iter().map(|text| Line((*text).into())).collect(), top_k, repo: None, front: false }
+}
+
+fn paths(hits: &SourceHits) -> Vec<&str> {
+    hits.hits.iter().map(|hit| hit.path.as_str()).collect()
+}
+
+/// One text answers exactly as it did before texts existed, whether it came in
+/// `text` or in `texts`: the index's own cosine scores, unfused, top_k deep.
+#[test]
+fn one_text_answers_exactly_as_before() {
+    let (mut rig, mut embedder) = topic_rig(4);
+    let within = Duration::from_secs(5);
+    let old = search(&mut embedder, &mut rig.index, &query("topic-2", 3, None), within).unwrap();
+    let new = search(&mut embedder, &mut rig.index, &queries(&["topic-2"], 3), within).unwrap();
+    assert_eq!(old, new);
+    assert_eq!(old.hits.len(), 3);
+    assert_eq!((old.hits[0].path.as_str(), old.hits[0].score), ("chunk-2.rs", 1.0), "the cosine, not a reciprocal-rank sum");
+    assert!(old.hits[1..].iter().all(|hit| hit.score == 0.0));
+    let searches = rig.index.state.lock().unwrap().searches.clone();
+    assert_eq!(searches, [(None, 3), (None, 3)], "unfused, so asked for top_k and no more");
+}
+
+/// Two texts that are equal once trimmed are one text: one search, and the
+/// answer of the single text.
+#[test]
+fn a_duplicated_text_does_not_double_count() {
+    let (mut rig, mut embedder) = topic_rig(4);
+    let within = Duration::from_secs(5);
+    let single = search(&mut embedder, &mut rig.index, &queries(&["topic-1"], 4), within).unwrap();
+    rig.index.state.lock().unwrap().searches.clear();
+    let twice = search(&mut embedder, &mut rig.index, &queries(&["topic-1", "  topic-1\n"], 4), within).unwrap();
+    assert_eq!(twice, single);
+    assert_eq!(rig.index.state.lock().unwrap().searches.len(), 1);
+}
+
+/// n distinct texts are one embed call of n prompts and n searches, each
+/// `FUSE_DEPTH` deep; the answer is fused and cut to top_k.
+#[test]
+fn n_texts_make_n_searches_and_one_embed_call() {
+    let (mut rig, mut embedder) = topic_rig(4);
+    let within = Duration::from_secs(5);
+    let found = search(&mut embedder, &mut rig.index, &queries(&["topic-0", "topic-1", "topic-3"], 2), within).unwrap();
+    let embedded = rig.embedder.state.lock().unwrap().embedded.clone();
+    assert_eq!(embedded.len(), 1, "one embed call");
+    assert_eq!(embedded[0].len(), 3);
+    assert!(embedded[0].iter().all(|prompt| prompt.starts_with("Instruct: ")));
+    let searches = rig.index.state.lock().unwrap().searches.clone();
+    assert_eq!(searches, vec![(None, FUSE_DEPTH); 3]);
+    assert_eq!(found.hits.len(), 2);
+    assert!(found.hits.iter().all(|hit| hit.score >= 1.0 / 61.0), "a reciprocal-rank sum, at least the 1/(60+1) of a first place");
+    assert!(found.hits[0].score >= found.hits[1].score);
+}
+
+/// The reason the cut exists: a brief of eight 1,000-byte topics reaches the
+/// chunk that only its last topic is near.
+#[test]
+fn a_long_brief_of_eight_topics_reaches_the_chunk_only_the_last_is_near() {
+    let (mut rig, mut embedder) = topic_rig(8);
+    let within = Duration::from_secs(5);
+    let topics: Vec<String> = (0..8).map(|n| format!("topic-{n} {}", "x".repeat(1000 - 8))).collect();
+    assert!(topics.iter().all(|topic| topic.len() == 1000));
+    let refs: Vec<&str> = topics.iter().map(String::as_str).collect();
+    let found = search(&mut embedder, &mut rig.index, &queries(&refs, 8), within).unwrap();
+    assert!(paths(&found).contains(&"chunk-7.rs"), "{:?}", paths(&found));
+    assert_eq!(paths(&found).len(), 8);
+}

crates/huginn-daemon/tests/live_index_smoke.rs

@@ -179,6 +179,7 @@ fn a_small_repo_is_refreshed_into_a_scratch_source_collection_and_found() {
         .and_then(|_| {
             let query = huginn_mind::SourceQuery {
                 text: huginn_mind::eureka_pipeline::Line("what does this repository do".into()),
+                texts: vec![],
                 top_k: 3,
                 repo: None,
                 front: false,

crates/huginn-mind/src/lib.rs

@@ -72,7 +72,7 @@ pub use store::{MindStore, OwnedRedbMessagePackBackingStore};
 pub use wire::{
     DeferredAnswer, HuginnMindRequest, HuginnMindResponse, IndexStatus, MAX_DEFERRED_BODY_BYTES, MIND_REQUEST_SCHEMA, MIND_REQUEST_SCHEMA_JSON,
     MIND_RESPONSE_SCHEMA, MIND_RESPONSE_SCHEMA_JSON, MIND_SERVICE_ID, MindStatus, PersonaStatus,
-    ACTIVITY_REPOS_MAX, OpenPr, PrAction, RepoActivity, RepoChange, RepoEvent, RepoMoved, SOURCE_TOP_K_MAX, SourceHit, SourceHits, SourceModel, SourceQuery,
+    ACTIVITY_REPOS_MAX, OpenPr, PrAction, RepoActivity, RepoChange, RepoEvent, RepoMoved, SOURCE_TEXTS_MAX, SOURCE_TOP_K_MAX, SourceHit, SourceHits, SourceModel, SourceQuery,
 };
 
 /// The leaf, whole, through the one crate that pins its rev: the daemon and

crates/huginn-mind/src/wire.rs

@@ -181,13 +181,23 @@ pub struct RepoMoved {
 /// The most hits a source search answers.
 pub const SOURCE_TOP_K_MAX: u32 = 50;
 
-/// A source search: the text to find chunks near, how many, optionally the
+/// The most texts one source search takes.
-/// one forge repo (by name) to look in, and whether to look only in repos'
+pub const SOURCE_TEXTS_MAX: usize = 8;
-/// front-page files (`FRONT_PAGE_FILES` at the repo root). An old client's
+
-/// request has no `front` and means false.
+/// A source search: the texts to find chunks near (one to `SOURCE_TEXTS_MAX`,
+/// each a topic; several answer with their ranked lists fused), how many,
+/// optionally the one forge repo (by name) to look in, and whether to look
+/// only in repos' front-page files (`FRONT_PAGE_FILES` at the repo root). An
+/// old client's request has no `front` and means false.
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
 pub struct SourceQuery {
+    /// Transitional: an old client's single text, read as `texts` of one. Exactly
+    /// one of `text` and `texts` is given. To be deleted once no client sends it.
+    #[serde(default)]
     pub text: Line,
+    /// The topics to search by, one to `SOURCE_TEXTS_MAX`.
+    #[serde(default)]
+    pub texts: Vec<Line>,
     pub top_k: u32,
     pub repo: Option<Short>,
     #[serde(default)]
@@ -195,7 +205,8 @@ pub struct SourceQuery {
 }
 
 impl SourceQuery {
-    /// The door: text that says something, and between one and
+    /// The door: exactly one of `text` and `texts`, each text saying something,
+    /// at most `SOURCE_TEXTS_MAX` texts, and between one and
     /// `SOURCE_TOP_K_MAX` hits. A refusal names the field.
     pub fn check(&self) -> Result<(), MindRefusal> {
         let refused = |field: &str, value: Option<String>, message: &str| MindRefusal::SelectionInvalid {
@@ -203,9 +214,21 @@ impl SourceQuery {
             value,
             message: message.into(),
         };
-        if self.text.0.trim().is_empty() {
+        if !self.text.0.is_empty() && !self.texts.is_empty() {
+            return Err(refused("texts", Some(self.texts.len().to_string()), "a source search takes `text` or `texts`, not both"));
+        }
+        if self.texts.is_empty() && self.text.0.is_empty() {
+            return Err(refused("texts", None, "a source search needs texts to find"));
+        }
+        if self.texts.is_empty() && self.text.0.trim().is_empty() {
             return Err(refused("text", None, "a source search needs text to find"));
         }
+        if self.texts.len() > SOURCE_TEXTS_MAX {
+            return Err(refused("texts", Some(self.texts.len().to_string()), "a source search takes 1 to 8 texts"));
+        }
+        if self.texts.iter().any(|text| text.0.trim().is_empty()) {
+            return Err(refused("texts", None, "every text of a source search says something"));
+        }
         if !(1..=SOURCE_TOP_K_MAX).contains(&self.top_k) {
             return Err(refused("top_k", Some(self.top_k.to_string()), "a source search asks for 1 to 50 hits"));
         }
@@ -234,6 +257,8 @@ pub struct SourceHit {
     pub commit: String,
     pub line_start: u32,
     pub line_end: u32,
+    /// Cosine similarity for a search of one text; the reciprocal-rank sum
+    /// (k = 60) over the texts' lists for a search of several.
     pub score: f32,
     pub text: String,
 }
@@ -419,7 +444,11 @@ mod tests {
             },
             HuginnMindRequest::SearchSources {
                 instance: slug(INSTANCE),
-                query: SourceQuery { text: Line("reconcile".into()), top_k: 5, repo: Some(Short("Huginn".into())), front: true },
+                query: SourceQuery { text: Line("reconcile".into()), texts: vec![], top_k: 5, repo: Some(Short("Huginn".into())), front: true },
+            },
+            HuginnMindRequest::SearchSources {
+                instance: slug(INSTANCE),
+                query: SourceQuery { text: Line::default(), texts: vec![Line("reconcile".into()), Line("fuse".into())], top_k: 5, repo: None, front: false },
             },
             HuginnMindRequest::RootsTouched { instance: slug(INSTANCE) },
         ]
@@ -486,7 +515,7 @@ mod tests {
     #[test]
     fn the_wire_vocabulary_is_the_minds_methods_and_status_is_derived_from_the_response() {
         let names: Vec<&str> = requests().iter().map(|request| request.operation()).collect();
-        assert_eq!(names, ["whoami", "admit", "view", "query", "persona_get", "persona_put", "search_sources", "roots_touched"]);
+        assert_eq!(names, ["whoami", "admit", "view", "query", "persona_get", "persona_put", "search_sources", "search_sources", "roots_touched"]);
         for request in requests() {
             let declared = request.instance().is_none();
             assert_eq!(declared, matches!(request, HuginnMindRequest::Whoami), "{request:?}");
@@ -518,6 +547,7 @@ mod tests {
     fn a_source_query_without_text_or_within_one_to_fifty_hits_is_refused_by_field() {
         let query = |text: &str, top_k: u32, repo: Option<&str>| SourceQuery {
             text: Line(text.into()),
+            texts: vec![],
             top_k,
             repo: repo.map(|repo| Short(repo.into())),
             front: false,
@@ -529,6 +559,17 @@ mod tests {
         assert_eq!(query("x", 1, None).check(), Ok(()));
         assert_eq!(query("x", SOURCE_TOP_K_MAX, Some("Huginn")).check(), Ok(()));
         assert_eq!(field(query("  ", 5, None)), "text");
+        let many = |texts: &[&str]| SourceQuery { text: Line::default(), texts: texts.iter().map(|text| Line((*text).into())).collect(), ..query("", 5, None) };
+        assert_eq!(many(&["a", "b"]).check(), Ok(()));
+        assert_eq!(many(&["a"; SOURCE_TEXTS_MAX]).check(), Ok(()));
+        assert_eq!(field(query("", 5, None)), "texts", "neither text nor texts");
+        assert_eq!(field(many(&[])), "texts");
+        assert_eq!(field(SourceQuery { text: Line("x".into()), ..many(&["a"]) }), "texts", "both");
+        assert_eq!(field(many(&["a", " "])), "texts", "a blank entry");
+        match many(&["a"; SOURCE_TEXTS_MAX + 1]).check() {
+            Err(MindRefusal::SelectionInvalid { field, value, .. }) => assert_eq!((field.as_str(), value.as_deref()), ("texts", Some("9"))),
+            other => panic!("expected a refusal, got {other:?}"),
+        }
         assert_eq!(field(query("x", 0, None)), "top_k");
         assert_eq!(field(query("x", SOURCE_TOP_K_MAX + 1, None)), "top_k");
         assert_eq!(field(query("x", 5, Some(" "))), "repo");
@@ -549,6 +590,23 @@ mod tests {
         assert!(!rmp_serde::from_slice::<SourceQuery>(&bytes).unwrap().front);
     }
 
+    /// An old client's search has `text` and no `texts`: it decodes as the one
+    /// text it sent, and passes the door.
+    #[test]
+    fn a_source_query_encoded_without_texts_decodes_as_its_one_text() {
+        #[derive(Serialize)]
+        struct OldQuery {
+            text: Line,
+            top_k: u32,
+            repo: Option<Short>,
+            front: bool,
+        }
+        let old = OldQuery { text: Line("x".into()), top_k: 3, repo: None, front: false };
+        let bytes = rmp_serde::to_vec_named(&old).unwrap();
+        let query = rmp_serde::from_slice::<SourceQuery>(&bytes).unwrap();
+        assert_eq!((query.text.0.as_str(), query.texts.len(), query.check()), ("x", 0, Ok(())));
+    }
+
     /// A new client reads an old daemon's page: the five fields it had, no
     /// `scores`. This is what keeps rollback by flip safe.
     #[test]

schemas/cultnet/huginn.mind_request.v1.schema.json

@@ -3034,7 +3034,7 @@
       "maxLength": 64
     },
     "SourceQuery": {
-      "description": "A source search: the text to find chunks near, how many, optionally the\none forge repo (by name) to look in, and whether to look only in repos'\nfront-page files (`FRONT_PAGE_FILES` at the repo root). An old client's\nrequest has no `front` and means false.",
+      "description": "A source search: the texts to find chunks near (one to `SOURCE_TEXTS_MAX`,\neach a topic; several answer with their ranked lists fused), how many,\noptionally the one forge repo (by name) to look in, and whether to look\nonly in repos' front-page files (`FRONT_PAGE_FILES` at the repo root). An\nold client's request has no `front` and means false.",
       "type": "object",
       "properties": {
         "front": {
@@ -3052,7 +3052,17 @@
           ]
         },
         "text": {
-          "$ref": "#/$defs/Line"
+          "description": "Transitional: an old client's single text, read as `texts` of one. Exactly\none of `text` and `texts` is given. To be deleted once no client sends it.",
+          "$ref": "#/$defs/Line",
+          "default": ""
+        },
+        "texts": {
+          "description": "The topics to search by, one to `SOURCE_TEXTS_MAX`.",
+          "type": "array",
+          "default": [],
+          "items": {
+            "$ref": "#/$defs/Line"
+          }
         },
         "top_k": {
           "type": "integer",
@@ -3061,7 +3071,6 @@
         }
       },
       "required": [
-        "text",
         "top_k"
       ]
     },

schemas/cultnet/huginn.mind_response.v1.schema.json

@@ -6005,6 +6005,7 @@
           "type": "string"
         },
         "score": {
+          "description": "Cosine similarity for a search of one text; the reciprocal-rank sum\n(k = 60) over the texts' lists for a search of several.",
           "type": "number",
           "format": "float"
         },