Add reciprocal rank fusion to the source organ

fuse is a pure generic function over ranked lists and a key closure, with
FUSE_K = 60 (Cormack, Clarke, Buttcher, SIGIR 2009) and FUSE_DEPTH = 50 named
beside CORPUS. Nothing calls it yet. Cut
gamecult-chat:cut_spec:cut-source-query-texts.r1, commit 2.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

crates/huginn-daemon/src/sources/mod.rs

@@ -24,12 +24,13 @@ pub mod git;
 #[cfg(test)]
 mod tests;
 
-use std::collections::{BTreeMap, BTreeSet};
+use std::collections::hash_map::Entry;
+use std::collections::{BTreeMap, BTreeSet, HashMap};
 use std::path::Path;
 use std::time::{Duration, Instant};
 
 use anyhow::{Context, Result, bail, ensure};
-use huginn_mind::{SourceHit, SourceHits, SourceModel, SourceQuery};
+use huginn_mind::{SOURCE_TOP_K_MAX, SourceHit, SourceHits, SourceModel, SourceQuery};
 
 use crate::index::{BATCH, BULK_EMBED_TIMEOUT, Budget, Embedder, INDEX_CALL_TIMEOUT, MANAGED_BY, ModelIdentity, point_id};
 use chunk::{CHUNKER_VERSION, embedded_text};
@@ -49,6 +50,14 @@ pub const CORPUS: &str = "forge:GameCult";
 pub const SOURCE_QUERY_INSTRUCTION: &str =
     "Given a question about a codebase, retrieve the source code and documentation passages that answer it";
 
+/// The constant of reciprocal rank fusion, score = sum over lists of
+/// 1 / (FUSE_K + rank), rank 1-based: Cormack, Clarke and Buttcher, SIGIR 2009,
+/// whose value this is. The one calibration constant of a multi-text search.
+pub const FUSE_K: u32 = 60;
+
+/// How many chunks each text's list holds before the lists are fused.
+pub const FUSE_DEPTH: u32 = SOURCE_TOP_K_MAX;
+
 /// Paths named in one delete call.
 const DELETE_PATHS_PER_CALL: usize = 256;
 
@@ -438,6 +447,31 @@ impl Batch {
     }
 }
 
+/// Reciprocal rank fusion of ranked lists (best first): an item's score is the
+/// sum over the lists that hold it of `1 / (k + rank)`, rank 1-based, and items
+/// with the same `key` are one item (the first list's copy is kept). Best first,
+/// the first-seen item ahead on a tie, cut to `top_k`. Pure: the one fuse, which
+/// every search that ranks across lists calls and none copies.
+pub fn fuse<T>(lists: Vec<Vec<T>>, key: impl Fn(&T) -> String, k: u32, top_k: usize) -> Vec<(f32, T)> {
+    let mut seen: HashMap<String, usize> = HashMap::new();
+    let mut fused: Vec<(f32, T)> = Vec::new();
+    for list in lists {
+        for (rank, item) in list.into_iter().enumerate() {
+            let share = 1.0 / (k as f32 + rank as f32 + 1.0);
+            match seen.entry(key(&item)) {
+                Entry::Occupied(at) => fused[*at.get()].0 += share,
+                Entry::Vacant(at) => {
+                    at.insert(fused.len());
+                    fused.push((share, item));
+                }
+            }
+        }
+    }
+    fused.sort_by(|a, b| b.0.total_cmp(&a.0));
+    fused.truncate(top_k);
+    fused
+}
+
 /// A source search, which a daemon answers from the collection as it finds it:
 /// it embeds the query with its own embedder, and refuses if the collection
 /// was built under another model, because vectors of two models do not compare.

crates/huginn-daemon/src/sources/tests.rs

@@ -606,3 +606,28 @@ fn chunks_are_embedded_in_bounded_batches() {
     assert_eq!(sizes, [BATCH, BATCH, 6]);
     assert_eq!(rig.index.held().len(), 70);
 }
+
+fn fused(lists: &[&[&str]], top_k: usize) -> Vec<(f32, String)> {
+    fuse(lists.iter().map(|list| list.iter().map(|item| (*item).to_owned()).collect()).collect(), |item| item.clone(), FUSE_K, top_k)
+}
+
+/// A chunk that two lists put first outranks one that a single list puts first:
+/// 1/61 + 1/61 against 1/61, with k = 60 and ranks counted from 1.
+#[test]
+fn a_chunk_first_in_two_lists_outranks_a_chunk_first_in_one() {
+    let found = fused(&[&["both", "x"], &["both", "y"]], 10);
+    let names: Vec<&str> = found.iter().map(|(_, item)| item.as_str()).collect();
+    assert_eq!(names[0], "both");
+    assert!((found[0].0 - 2.0 / 61.0).abs() < 1e-6, "{}", found[0].0);
+    assert!((found[1].0 - 1.0 / 62.0).abs() < 1e-6, "second place in one list is 1/(60+2)");
+    assert_eq!(names, ["both", "x", "y"], "a tie keeps the earlier list's item ahead");
+    let deeper = fused(&[&["a", "b", "c"], &["c"]], 10);
+    assert_eq!(deeper[0].1, "c", "1/63 + 1/61 beats 1/61");
+}
+
+/// The fused list is cut to `top_k`, and an empty input fuses to nothing.
+#[test]
+fn a_fused_list_is_cut_to_top_k() {
+    assert_eq!(fused(&[&["a", "b", "c"]], 2).len(), 2);
+    assert!(fused(&[], 5).is_empty());
+}