From 869c82b686794bd06e679a9fb0d1a50ae0c91c95 Mon Sep 17 00:00:00 2001 From: Eureka Date: Thu, 29 Oct 2026 17:17:46 +0100 Subject: [PATCH] scorer: chrF and meaning cosine of whole output against whole reference, no mask; Embedder port with Ollama adapter; pooled novel-n-gram leak diagnostics; aggregation by derived split (squash of weksa-lowering/scorer onto 20865ca, head 5689e62; spec r3, Soul s1-s3) Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 76 +++ README.md | 12 +- crates/weksa/Cargo.toml | 2 + crates/weksa/src/lib.rs | 5 +- crates/weksa/src/score/chrf.rs | 60 +++ crates/weksa/src/score/diagnostics.rs | 169 +++++++ crates/weksa/src/score/embed.rs | 113 +++++ crates/weksa/src/score/mod.rs | 144 ++++++ crates/weksa/src/score/words.rs | 44 ++ crates/weksa/tests/score.rs | 692 ++++++++++++++++++++++++++ 10 files changed, 1311 insertions(+), 6 deletions(-) create mode 100644 crates/weksa/src/score/chrf.rs create mode 100644 crates/weksa/src/score/diagnostics.rs create mode 100644 crates/weksa/src/score/embed.rs create mode 100644 crates/weksa/src/score/mod.rs create mode 100644 crates/weksa/src/score/words.rs create mode 100644 crates/weksa/tests/score.rs diff --git a/Cargo.lock b/Cargo.lock index 3aa8a3e..38e73c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -82,6 +82,12 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "bitflags" version = "2.13.2" @@ -94,6 +100,21 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "caseless" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6fd507454086c8edfd769ca6ada439193cdb209c7681712ef6275cccbfe5d8" +dependencies = [ + "unicode-normalization", +] + [[package]] name = "cc" version = "1.10.0" @@ -285,6 +306,22 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -386,6 +423,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -686,6 +729,37 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" +[[package]] +name = "ureq" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cde93ff05758a36a851a398a772b0b79a82fbe3c186a46f2696baaeaf84c966b" +dependencies = [ + "base64", + "log", + "percent-encoding", + "ureq-proto", + "utf8-zero", +] + +[[package]] +name = "ureq-proto" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841583fc36089d7d59419568ae157f43a5a03a2e5a81ba87c77d1596ef46e922" +dependencies = [ + "base64", + "http", + "httparse", + "log", +] + +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "utf8parse" version = "0.2.2" @@ -753,6 +827,7 @@ name = "weksa" version = "0.1.0" dependencies = [ "anyhow", + "caseless", "clap", "cultcache-rs", "schemars", @@ -761,6 +836,7 @@ dependencies = [ "tempfile", "unicode-normalization", "unicode-segmentation", + "ureq", ] [[package]] diff --git a/README.md b/README.md index b599515..d327e48 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,10 @@ measured. - [Interlingua Standard](./docs/interlingua-standard.md): the packet contract. - `crates/weksa`: the packet, inventory and corpus types, `validate`, the - derived split and the `weksa` CLI (`weksa validate`, `corpus`, `entity`, `inventory`, - `split`, `schema`); published schemas in `schemas/cultnet/`. + derived split, the scorer (`score`: chrF, meaning cosine through an Ollama + embedder, whole output against whole reference with no mask, leak diagnostics) and the `weksa` CLI + (`weksa validate`, `corpus`, `entity`, `inventory`, `split`, `schema`); + published schemas in `schemas/cultnet/`. - `data/corpus/corpus.cc`, `data/entity/entity.cc` and `data/interlingua/inventory.cc`: the English corpus, its entities and the inventory, read with `weksa corpus dump`, `weksa entity dump` and `weksa inventory dump`. @@ -30,8 +32,8 @@ measured. - `settings/`: setting-specific proving grounds for language projects. **Live versus planned.** Live: docs, fixtures, examples, ontology data and the -packet crate `crates/weksa`. Planned: scoring, lowering, and -the golf loop that scores lowered English against an expressive corpus. No +packet crate `crates/weksa`, including the scoring library. Planned: lowering, +the `weksa score` subcommand, and the golf loop that scores lowered English against an expressive corpus. No daemon runs; the earlier Node daemon is parked at tag `parked/weksa-node-body`. ## Ownership Model @@ -92,7 +94,7 @@ AquaSynth or expanding pronunciation coverage. The persistent state spine is live: map, scratch, schemas, docs, examples and data. Packet types, validation and the corpus and inventory stores are live in -`crates/weksa`. Scoring, lowering and the service are planned (see the cut +`crates/weksa`, and so is the scoring library. Lowering and the service are planned (see the cut sequence in [the lowering map](./docs/targets/interlingua-lowering-map.md)). Writing and agent state will enter Weksa as intent pressure, and typed utterance documents will leave for AquaSynth. diff --git a/crates/weksa/Cargo.toml b/crates/weksa/Cargo.toml index 75b5ac6..5b9c407 100644 --- a/crates/weksa/Cargo.toml +++ b/crates/weksa/Cargo.toml @@ -14,6 +14,7 @@ path = "src/main.rs" [dependencies] anyhow = "1" +caseless = "0.2" clap = { version = "4", features = ["derive"] } cultcache-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "64b86e0a273a3936939357e9deb186b7bf3bc417", package = "cultcache-rs" } schemars = "1" @@ -21,6 +22,7 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" unicode-normalization = "0.1" unicode-segmentation = "1" +ureq = { version = "3", default-features = false } [dev-dependencies] tempfile = "3" diff --git a/crates/weksa/src/lib.rs b/crates/weksa/src/lib.rs index 8aa7384..ddd36c6 100644 --- a/crates/weksa/src/lib.rs +++ b/crates/weksa/src/lib.rs @@ -1,6 +1,7 @@ //! Weksa's typed interlingua state: packet, inventory and corpus documents in //! CultCache, the validator that decides whether a packet exists, the derived -//! train/validation/test split, and the stores that admit them. +//! train/validation/test split, the stores that admit them, and the scorer +//! (chrF, meaning cosine, leak diagnostics). pub mod corpus; pub mod entity; @@ -8,6 +9,7 @@ pub mod form; pub mod inventory; pub mod packet; pub mod schema; +pub mod score; pub mod split; pub mod store; pub mod validate; @@ -22,6 +24,7 @@ pub use inventory::{ pub use packet::{ EncoderId, EntityId, Grade, NodeId, Packet, PacketId, PacketKind, SCHEMA_VERSION, }; +pub use score::{Embedder, ItemScore, OllamaEmbedder, Scorer, chrf}; pub use split::{Split, SplitScheme}; pub use store::{ AddOutcome, CorpusStore, EntityBound, EntityStore, InventoryStore, PacketStore, StoreError, diff --git a/crates/weksa/src/score/chrf.rs b/crates/weksa/src/score/chrf.rs new file mode 100644 index 0000000..7650518 --- /dev/null +++ b/crates/weksa/src/score/chrf.rs @@ -0,0 +1,60 @@ +//! chrF as sacrebleu 2.4.3 computes it by default: character n-gram orders 1 to +//! 6, whitespace removed, case kept, beta 2. + +use std::collections::HashMap; + +/// The metric this module implements; a score stored beside it names its definition. +pub const CHRF_VERSION: &str = "chrF2|nc:6|nw:0|space:no|sacrebleu:2.4.3"; + +const ORDER: usize = 6; +const BETA_SQ: f64 = 4.0; + +fn grams(chars: &[char], n: usize) -> HashMap<&[char], u32> { + let mut counts = HashMap::new(); + for window in chars.windows(n) { + *counts.entry(window).or_insert(0) += 1; + } + counts +} + +/// Python `str.split`'s whitespace, which sacrebleu strips before gramming: +/// Unicode White_Space plus U+001C..=U+001F. +pub(super) fn is_split_space(c: char) -> bool { + c.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&c) +} + +/// chrF of `hypothesis` against `reference`, 0 to 100. Beta 2 weights recall, +/// so the arguments are not interchangeable. 0 when no order has n-grams on +/// both sides. +pub fn chrf(hypothesis: &str, reference: &str) -> f64 { + let strip = |s: &str| { + s.chars() + .filter(|c| !is_split_space(*c)) + .collect::>() + }; + let (hyp, reference) = (strip(hypothesis), strip(reference)); + let (mut precision, mut recall, mut orders) = (0.0, 0.0, 0u32); + for n in 1..=ORDER { + let (hyp_grams, ref_grams) = (grams(&hyp, n), grams(&reference, n)); + let (hyp_total, ref_total): (u32, u32) = + (hyp_grams.values().sum(), ref_grams.values().sum()); + if hyp_total == 0 || ref_total == 0 { + continue; + } + let matched: u32 = hyp_grams + .iter() + .map(|(gram, count)| (*count).min(ref_grams.get(gram).copied().unwrap_or(0))) + .sum(); + precision += f64::from(matched) / f64::from(hyp_total); + recall += f64::from(matched) / f64::from(ref_total); + orders += 1; + } + if orders == 0 { + return 0.0; + } + let (p, r) = (precision / f64::from(orders), recall / f64::from(orders)); + if p + r == 0.0 { + return 0.0; + } + 100.0 * (1.0 + BETA_SQ) * p * r / (BETA_SQ * p + r) +} diff --git a/crates/weksa/src/score/diagnostics.rs b/crates/weksa/src/score/diagnostics.rs new file mode 100644 index 0000000..9d91cf2 --- /dev/null +++ b/crates/weksa/src/score/diagnostics.rs @@ -0,0 +1,169 @@ +//! Leak diagnostics: values reported beside scores, never refusals. Pooled +//! coverage at n >= 2 above zero beside single-word glosses, or a high pairwise +//! chrF, means a flow bug or a corpus duplicate across splits. + +use std::collections::HashSet; + +use super::chrf::chrf; +use super::words::words; +use crate::corpus::CorpusItem; +use crate::split::{Split, SplitScheme}; + +/// Word n-gram orders read, 1 to this. +const ORDERS: usize = 3; + +/// The pair that reached the highest chrF. +#[derive(Clone, Debug, PartialEq)] +pub struct PairChrf { + pub chrf: f64, + pub minted: String, + pub item: String, +} + +/// One word-order's pooled novel coverage over a split's items. +#[derive(Clone, Debug, PartialEq)] +pub struct OrderCoverage { + pub n: usize, + /// Distinct n-grams of the split's items absent from Train, summed over items. + pub novel: usize, + /// Of those, the ones that occur in the minted text. + pub covered: usize, + /// `covered / novel`; `None` when `novel` is 0, never NaN. + pub share: Option, +} + +/// The item that read most at `n` >= 2. +#[derive(Clone, Debug, PartialEq)] +pub struct WorstItem { + pub item: String, + pub n: usize, + pub covered: usize, + pub novel: usize, +} + +/// One held-out split's readings. +#[derive(Clone, Debug, PartialEq, Default)] +pub struct SplitLeak { + /// Items of the split. + pub items: usize, + /// Max chrF of any minted string (hypothesis) against any item (reference). + pub max_chrf: Option, + /// Pooled novel coverage for n in 1..=3, in order. + pub coverage_by_n: Vec, + /// The item and order (n >= 2) with the largest covered count, then the + /// larger share; the first item, then the smaller n, on ties. `None` when + /// no item has a covered n-gram at n >= 2. + pub worst: Option, +} + +#[derive(Clone, Debug, PartialEq, Default)] +pub struct LeakDiagnostic { + pub validation: SplitLeak, + pub test: SplitLeak, +} + +/// The distinct word n-grams of a text for each order 1..=ORDERS, over its +/// normalised word segments. +fn grams(text: &str) -> Vec> { + let keys = words(text); + (1..=ORDERS) + .map(|n| keys.windows(n).map(|w| w.join(" ")).collect()) + .collect() +} + +fn union<'a>(texts: impl Iterator) -> Vec> { + let mut all = vec![HashSet::new(); ORDERS]; + for text in texts { + for (into, from) in all.iter_mut().zip(grams(text)) { + into.extend(from); + } + } + all +} + +fn split_leak( + minted: &[&str], + train: &[HashSet], + minted_grams: &[HashSet], + items: &[&CorpusItem], +) -> SplitLeak { + let mut leak = SplitLeak { + items: items.len(), + ..SplitLeak::default() + }; + let mut totals = [(0usize, 0usize); ORDERS]; + for item in items { + for m in minted { + let score = chrf(m, &item.text); + if leak.max_chrf.as_ref().is_none_or(|best| score > best.chrf) { + leak.max_chrf = Some(PairChrf { + chrf: score, + minted: (*m).to_string(), + item: item.id.clone(), + }); + } + } + for (i, item_grams) in grams(&item.text).iter().enumerate() { + let novel: Vec<&String> = item_grams + .iter() + .filter(|g| !train[i].contains(*g)) + .collect(); + let covered = novel + .iter() + .filter(|g| minted_grams[i].contains(**g)) + .count(); + totals[i].0 += novel.len(); + totals[i].1 += covered; + if i == 0 || covered == 0 { + continue; + } + let share = covered as f64 / novel.len() as f64; + let better = leak.worst.as_ref().is_none_or(|w| { + covered > w.covered + || (covered == w.covered && share > w.covered as f64 / w.novel as f64) + }); + if better { + leak.worst = Some(WorstItem { + item: item.id.clone(), + n: i + 1, + covered, + novel: novel.len(), + }); + } + } + } + leak.coverage_by_n = totals + .iter() + .enumerate() + .map(|(i, (novel, covered))| OrderCoverage { + n: i + 1, + novel: *novel, + covered: *covered, + share: (*novel > 0).then(|| *covered as f64 / *novel as f64), + }) + .collect(); + leak +} + +/// Reads minted text (inventory glosses, option labels, later ontology text) +/// against the held-out items. Per held-out item and for n in 1..=3, its word +/// n-grams that occur in no Train item are its novel n-grams; coverage at each +/// n is the covered over the novel n-grams summed across the split's items +/// (unigrams over all minted strings, longer grams within one string). Honest +/// single-word glosses saturate n = 1, so the leak signature is pooled n >= 2 +/// and the worst item at n >= 2. Train items are not measured; items in no +/// split are dropped. +pub fn leak_diagnostic(minted: &[&str], items: &[CorpusItem]) -> LeakDiagnostic { + let of = |want: Split| -> Vec<&CorpusItem> { + items + .iter() + .filter(|i| SplitScheme::V1.split(i) == Some(want)) + .collect() + }; + let train = union(of(Split::Train).iter().map(|i| i.text.as_str())); + let minted_grams = union(minted.iter().copied()); + LeakDiagnostic { + validation: split_leak(minted, &train, &minted_grams, &of(Split::Validation)), + test: split_leak(minted, &train, &minted_grams, &of(Split::Test)), + } +} diff --git a/crates/weksa/src/score/embed.rs b/crates/weksa/src/score/embed.rs new file mode 100644 index 0000000..45d1bcf --- /dev/null +++ b/crates/weksa/src/score/embed.rs @@ -0,0 +1,113 @@ +//! The meaning metric's embedding port and its Ollama adapter. + +use std::time::Duration; + +use anyhow::{Context, Result, bail, ensure}; +use serde::Deserialize; +use serde_json::json; + +/// Sentence embeddings. Every embedder (Ollama now, a style embedder later) +/// enters the scorer through this port. +pub trait Embedder { + fn model(&self) -> &str; + /// One vector per text, in order, all of one dimension. + fn embed(&self, texts: &[&str]) -> Result>>; +} + +/// Cosine similarity. Precondition: `a` and `b` have one equal nonzero length +/// and nonzero norm; `Scorer::score` refuses embedder output that does not. +/// Violating it panics rather than reading as a score. +pub fn cosine(a: &[f32], b: &[f32]) -> f64 { + assert!( + !a.is_empty() && a.len() == b.len(), + "cosine needs vectors of one nonzero length, got {} and {}", + a.len(), + b.len() + ); + let dot: f64 = a + .iter() + .zip(b) + .map(|(x, y)| f64::from(*x) * f64::from(*y)) + .sum(); + let norm = |v: &[f32]| v.iter().map(|x| f64::from(*x).powi(2)).sum::().sqrt(); + let denom = norm(a) * norm(b); + assert!(denom > 0.0, "cosine of a zero vector"); + dot / denom +} + +/// Ollama's `/api/embed` over plain HTTP. One server serves Huginn and the +/// loop, so batches stay small and `num_thread` is sent (without it a batch of +/// 32 timed out at 120 s under host load). +#[derive(Clone, Debug)] +pub struct OllamaEmbedder { + pub url: String, + pub model: String, + /// Sent as `options.num_thread`; omitted when `None`. + pub num_thread: Option, + /// Texts per request, clamped to 1..=32. + pub batch: usize, +} + +impl OllamaEmbedder { + pub const MAX_BATCH: usize = 32; + + pub fn new(url: impl Into, model: impl Into) -> Self { + Self { + url: url.into(), + model: model.into(), + num_thread: Some(8), + batch: Self::MAX_BATCH, + } + } + + fn request(&self, agent: &ureq::Agent, batch: &[&str]) -> Result>> { + let mut body = json!({ "model": self.model, "input": batch }); + if let Some(threads) = self.num_thread { + body["options"] = json!({ "num_thread": threads }); + } + let endpoint = format!("{}/api/embed", self.url.trim_end_matches('/')); + let mut response = agent + .post(&endpoint) + .header("Content-Type", "application/json") + .send(body.to_string()) + .with_context(|| format!("POST {endpoint}"))?; + #[derive(Deserialize)] + struct Reply { + embeddings: Vec>, + } + let reply: Reply = serde_json::from_str(&response.body_mut().read_to_string()?) + .context("ollama /api/embed reply")?; + ensure!( + reply.embeddings.len() == batch.len(), + "asked for {} embeddings, got {}", + batch.len(), + reply.embeddings.len() + ); + Ok(reply.embeddings) + } +} + +impl Embedder for OllamaEmbedder { + fn model(&self) -> &str { + &self.model + } + + fn embed(&self, texts: &[&str]) -> Result>> { + if texts.is_empty() { + return Ok(Vec::new()); + } + let agent: ureq::Agent = ureq::Agent::config_builder() + .timeout_global(Some(Duration::from_secs(120))) + .build() + .into(); + let mut out: Vec> = Vec::with_capacity(texts.len()); + for batch in texts.chunks(self.batch.clamp(1, Self::MAX_BATCH)) { + out.extend(self.request(&agent, batch)?); + } + let dim = out[0].len(); + if dim == 0 || out.iter().any(|v| v.len() != dim) { + bail!("embeddings disagree on dimension"); + } + Ok(out) + } +} diff --git a/crates/weksa/src/score/mod.rs b/crates/weksa/src/score/mod.rs new file mode 100644 index 0000000..713dede --- /dev/null +++ b/crates/weksa/src/score/mod.rs @@ -0,0 +1,144 @@ +//! What a score counts: chrF and the meaning cosine of the whole lowered +//! output against the whole reference. Nothing is deleted, hidden or replaced +//! before either is read, and the scorer takes no input from a packet, so no +//! text the output carries can raise a score except by resembling the +//! reference. No door judges text; the diagnostics report. + +mod chrf; +mod diagnostics; +mod embed; +mod words; + +use anyhow::{Result, ensure}; + +pub use chrf::{CHRF_VERSION, chrf}; +pub use diagnostics::{ + LeakDiagnostic, OrderCoverage, PairChrf, SplitLeak, WorstItem, leak_diagnostic, +}; +pub use embed::{Embedder, OllamaEmbedder, cosine}; + +use crate::corpus::CorpusItem; +use crate::split::{Split, SplitScheme}; + +/// One reference/output pair; both are scored whole. +pub struct ScoreInput<'a> { + pub reference: &'a str, + pub output: &'a str, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ItemScore { + /// chrF of the output against the reference. + pub chrf: f64, + /// Embedding cosine of the output and the reference; `None` when either + /// is empty after trimming. + pub meaning: Option, +} + +pub struct Scorer { + embedder: E, +} + +/// Every vector has one nonzero length and a nonzero norm. +fn check_vectors(vectors: &[Vec]) -> Result<()> { + let dim = vectors.first().map_or(0, Vec::len); + for vector in vectors { + ensure!( + dim > 0 && vector.len() == dim, + "embedder returned vectors of length {} and {}", + dim, + vector.len() + ); + let norm: f64 = vector.iter().map(|x| f64::from(*x).powi(2)).sum(); + ensure!( + norm > 0.0 && norm.is_finite(), + "embedder returned a zero or non-finite vector" + ); + } + Ok(()) +} + +impl Scorer { + pub fn new(embedder: E) -> Self { + Self { embedder } + } + + pub fn embedder(&self) -> &E { + &self.embedder + } + + /// Scores every input with one embed call; the caller chunks `inputs`. + /// Errs when the embedder's vectors disagree in length or are empty or zero. + pub fn score(&self, inputs: &[ScoreInput<'_>]) -> Result> { + let mut texts: Vec<&str> = Vec::new(); + let slots: Vec> = inputs + .iter() + .map(|input| { + if input.output.trim().is_empty() || input.reference.trim().is_empty() { + return None; + } + texts.push(input.reference); + texts.push(input.output); + Some(texts.len() - 2) + }) + .collect(); + let vectors = self.embedder.embed(&texts)?; + ensure!( + vectors.len() == texts.len(), + "embedder returned {} vectors for {} texts", + vectors.len(), + texts.len() + ); + check_vectors(&vectors)?; + Ok(inputs + .iter() + .zip(slots) + .map(|(input, slot)| ItemScore { + chrf: chrf(input.output, input.reference), + meaning: slot.map(|i| cosine(&vectors[i], &vectors[i + 1])), + }) + .collect()) + } +} + +/// Mean scores of one derived split. +#[derive(Clone, Debug, PartialEq, Default)] +pub struct SplitMean { + pub count: usize, + /// `None` when the split has no items. + pub chrf: Option, + /// Mean over the items that have a meaning score; `None` when none do. + pub meaning: Option, +} + +#[derive(Clone, Debug, PartialEq, Default)] +pub struct Aggregate { + pub train: SplitMean, + pub validation: SplitMean, + pub test: SplitMean, +} + +/// Groups scores by the item's derived split. Items in no split are counted +/// nowhere. The scorer only groups: the loop decides when Test is read. +pub fn aggregate(scored: &[(&CorpusItem, ItemScore)]) -> Aggregate { + let mean = |want: Split| { + let of: Vec<&ItemScore> = scored + .iter() + .filter(|(item, _)| SplitScheme::V1.split(item) == Some(want)) + .map(|(_, score)| score) + .collect(); + let average = |values: Vec| { + (!values.is_empty()).then(|| values.iter().sum::() / values.len() as f64) + }; + SplitMean { + count: of.len(), + chrf: average(of.iter().map(|s| s.chrf).collect()), + meaning: average(of.iter().filter_map(|s| s.meaning).collect()), + } + }; + Aggregate { + train: mean(Split::Train), + validation: mean(Split::Validation), + test: mean(Split::Test), + } +} diff --git a/crates/weksa/src/score/words.rs b/crates/weksa/src/score/words.rs new file mode 100644 index 0000000..58e7c1f --- /dev/null +++ b/crates/weksa/src/score/words.rs @@ -0,0 +1,44 @@ +//! The one way the scorer reads words: UAX #29 word segmentation, then NFC and +//! full Unicode case folding. The novel-coverage diagnostic reads text through +//! here, so every script is treated by one rule. + +use caseless::default_case_fold_str; +use unicode_normalization::UnicodeNormalization; +use unicode_segmentation::UnicodeSegmentation; + +/// NFC, then full default case folding. +pub(super) fn normalise(s: &str) -> String { + default_case_fold_str(&s.nfc().collect::()) +} + +/// The pieces of one segment: split at `_` and `.` (which are dropped) and +/// at every lowercase-to-uppercase transition, so `kick_the_bucket`, +/// `kick.the.bucket` and `kickTheBucket` read as `kick the bucket`. +fn pieces(segment: &str) -> Vec<&str> { + let mut out = Vec::new(); + let mut start = 0; + let mut previous: Option = None; + for (at, c) in segment.char_indices() { + let separator = c == '_' || c == '.'; + let camel = previous.is_some_and(char::is_lowercase) && c.is_uppercase(); + if separator || camel { + out.push(&segment[start..at]); + start = if separator { at + c.len_utf8() } else { at }; + } + previous = Some(c); + } + out.push(&segment[start..]); + out +} + +/// The normalised words of `text` in order: UAX #29 segments containing an +/// alphanumeric char (digits kept), each split by `pieces`, empty or +/// non-alphanumeric pieces dropped. +pub(super) fn words(text: &str) -> Vec { + text.split_word_bounds() + .filter(|segment| segment.chars().any(char::is_alphanumeric)) + .flat_map(pieces) + .filter(|piece| piece.chars().any(char::is_alphanumeric)) + .map(normalise) + .collect() +} diff --git a/crates/weksa/tests/score.rs b/crates/weksa/tests/score.rs new file mode 100644 index 0000000..6920b93 --- /dev/null +++ b/crates/weksa/tests/score.rs @@ -0,0 +1,692 @@ +mod common; + +use std::cell::RefCell; +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::TcpListener; +use std::sync::{Arc, Mutex}; + +use anyhow::Result; +use common::{cast, group_in_bucket, item_in, model}; +use serde_json::Value; +use weksa::CorpusItem; +use weksa::score::{ + Embedder, ItemScore, OllamaEmbedder, ScoreInput, Scorer, aggregate, chrf, cosine, + leak_diagnostic, +}; + +/// (reference, hypothesis, sacrebleu 2.4.3 sentence_chrf), map 491536c. +const VECTORS: [(&str, &str, f64); 6] = [ + ( + "Take your hand off that control unless you want your nervous system to become my least useful circuit diagram.", + "Take your hand off that control, unless you'd like your nervous system to be my least useful wiring diagram.", + 74.4498, + ), + ( + "The command has no receipt. It is not truth.", + "The order carries no receipt, so it is not true.", + 45.7164, + ), + ("I am not angry.", "I am not angry.", 100.0), + ("Forgive me, sir.", "Get out.", 3.9683), + ("その制御に触るな。", "その操作盤に触らないで。", 14.2931), + ("Well, aren't you a sight for sore eyes, love.", "", 0.0), +]; + +#[test] +fn chrf_matches_sacrebleu_vectors() { + for (reference, hypothesis, want) in VECTORS { + let got = chrf(hypothesis, reference); + assert!( + (got - want).abs() < 1e-4, + "{hypothesis:?} against {reference:?}: {got} != {want}" + ); + } +} + +#[test] +fn chrf_argument_order() { + let short = "mind the hatch"; + let long = "mind the hatch before the storm arrives tonight"; + let (as_hypothesis, as_reference) = (chrf(short, long), chrf(long, short)); + assert!( + as_hypothesis < as_reference - 1.0, + "recall is weighted: a short hypothesis of a long reference ({as_hypothesis}) must score below the reverse ({as_reference})" + ); +} + +#[derive(Default)] +struct Recorder { + seen: RefCell>, +} + +impl Embedder for Recorder { + fn model(&self) -> &str { + "fake" + } + fn embed(&self, texts: &[&str]) -> Result>> { + self.seen + .borrow_mut() + .extend(texts.iter().map(|t| t.to_string())); + Ok(texts.iter().map(|t| vec![t.len() as f32, 1.0]).collect()) + } +} + +/// A deterministic bag-of-letters embedder, so cosine differs with the text. +struct Bag; + +impl Embedder for Bag { + fn model(&self) -> &str { + "bag" + } + fn embed(&self, texts: &[&str]) -> Result>> { + Ok(texts + .iter() + .map(|t| { + let mut v = vec![0.0; 27]; + v[26] = 1.0; + for c in t.chars().filter(char::is_ascii_alphabetic) { + v[(c.to_ascii_lowercase() as u8 - b'a') as usize] += 1.0; + } + v + }) + .collect()) + } +} + +fn score_one(scorer: &Scorer, reference: &str, output: &str) -> ItemScore { + scorer + .score(&[ScoreInput { reference, output }]) + .unwrap() + .remove(0) +} + +fn close(got: f64, want: f64) { + assert!((got - want).abs() < 1e-4, "{got} != {want}"); +} + +#[test] +fn chrf_whitespace_parity() { + // sacrebleu 2.4.3 strips str.split's whitespace; every vector scores 100 + let vectors = [ + ("a\tb c", "ab c"), + ("a\u{a0}b", "ab"), + ("a\u{1c}b", "ab"), + ("a\u{85}b", "ab"), + ("a\u{2028}b c", "ab c"), + ("line one\nline two", "line one line two"), + ("その\u{3000}制御", "その制御"), + ]; + for (hypothesis, reference) in vectors { + assert_eq!(chrf(hypothesis, reference), 100.0, "{hypothesis:?}"); + assert_eq!(chrf(reference, hypothesis), 100.0, "{hypothesis:?}"); + } + // a zero-width space is not whitespace + close(chrf("a\u{200b}b", "ab"), 45.4545); +} + +#[test] +fn scorer_argument_order() { + let short = "mind the hatch"; + let long = "mind the hatch before the storm arrives tonight"; + let scorer = Scorer::new(Bag); + let got = score_one(&scorer, long, short); + assert_eq!(got.chrf, chrf(short, long)); + assert_ne!(got.chrf, chrf(long, short)); +} + +#[test] +fn meaning_none_on_empty_text() { + let scorer = Scorer::new(Recorder::default()); + let inputs = [ + ScoreInput { + reference: "Nibu says hi", + output: " ", + }, + ScoreInput { + reference: " ", + output: "says hi", + }, + ScoreInput { + reference: "Nibu says hi", + output: "says hi", + }, + ]; + let scores = scorer.score(&inputs).unwrap(); + assert_eq!(scores[0].meaning, None); + assert_eq!(scores[1].meaning, None); + assert!(scores[2].meaning.is_some()); + assert_eq!( + *scorer.embedder().seen.borrow(), + ["Nibu says hi", "says hi"] + ); + assert_eq!(scores[0].chrf, 0.0); +} + +/// Answers every embed call with the same vectors. +struct Fixed(Vec>); + +impl Embedder for Fixed { + fn model(&self) -> &str { + "fixed" + } + fn embed(&self, _texts: &[&str]) -> Result>> { + Ok(self.0.clone()) + } +} + +#[test] +fn scorer_refuses_bad_vectors() { + let run = |vectors: Vec>| { + Scorer::new(Fixed(vectors)).score(&[ScoreInput { + reference: "mind the hatch", + output: "mind the door", + }]) + }; + assert!(run(vec![vec![1.0, 2.0], vec![2.0, 1.0]]).is_ok()); + for bad in [ + vec![vec![1.0, 2.0], vec![1.0]], + vec![vec![1.0], vec![1.0, 2.0]], + vec![vec![], vec![]], + vec![vec![0.0, 0.0], vec![0.0, 0.0]], + vec![vec![1.0, 2.0], vec![0.0, 0.0]], + vec![vec![0.0, 0.0], vec![1.0, 2.0]], + vec![vec![1.0, f32::NAN], vec![1.0, 2.0]], + vec![vec![1.0, 2.0]], + ] { + assert!(run(bad.clone()).is_err(), "{bad:?}"); + } +} + +#[test] +fn cosine_precondition() { + assert!((cosine(&[1.0, 2.0], &[2.0, 4.0]) - 1.0).abs() < 1e-9); + assert!(cosine(&[1.0, 0.0], &[0.0, 1.0]).abs() < 1e-9); + for (a, b) in [ + (vec![1.0], vec![1.0, 1.0]), + (vec![], vec![]), + (vec![0.0, 0.0], vec![1.0, 1.0]), + ] { + let refused = std::panic::catch_unwind(|| cosine(&a, &b)); + assert!(refused.is_err(), "{a:?} {b:?}"); + } +} + +/// A plain-HTTP fake of `/api/embed`. Records each request body; answers with +/// `dim` floats per input, one fewer vector than asked when `drop_one`, and a +/// short last vector when `short_last`. +fn fake_ollama(drop_one: bool, short_last: bool) -> (String, Arc>>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let bodies = Arc::new(Mutex::new(Vec::new())); + let seen = Arc::clone(&bodies); + std::thread::spawn(move || { + for stream in listener.incoming() { + let mut stream = stream.unwrap(); + let mut reader = BufReader::new(stream.try_clone().unwrap()); + 'conn: loop { + let mut length = 0; + let mut line = String::new(); + let mut first = true; + loop { + line.clear(); + if reader.read_line(&mut line).unwrap_or(0) == 0 { + break 'conn; + } + if first { + assert!(line.starts_with("POST /api/embed "), "{line}"); + first = false; + } + let lower = line.to_ascii_lowercase(); + if let Some(v) = lower.strip_prefix("content-length:") { + length = v.trim().parse().unwrap(); + } + if line == "\r\n" { + break; + } + } + let mut body = vec![0u8; length]; + reader.read_exact(&mut body).unwrap(); + let body: Value = serde_json::from_slice(&body).unwrap(); + let mut count = body["input"].as_array().unwrap().len(); + seen.lock().unwrap().push(body); + if drop_one { + count -= 1; + } + let vectors: Vec> = (0..count) + .map(|i| vec![1.0; if short_last && i + 1 == count { 2 } else { 3 }]) + .collect(); + let reply = serde_json::json!({ "embeddings": vectors }).to_string(); + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{reply}", + reply.len() + ) + .unwrap(); + } + } + }); + (url, bodies) +} + +fn lengths(bodies: &Mutex>) -> Vec { + let bodies = bodies.lock().unwrap(); + bodies + .iter() + .map(|b| b["input"].as_array().unwrap().len()) + .collect() +} + +#[test] +fn ollama_request_shape_and_batching() { + let (url, bodies) = fake_ollama(false, false); + let embedder = OllamaEmbedder::new(&url, "qwen3-embedding:0.6b"); + let texts: Vec = (0..70).map(|i| format!("text {i}")).collect(); + let refs: Vec<&str> = texts.iter().map(String::as_str).collect(); + let vectors = embedder.embed(&refs).unwrap(); + assert_eq!(vectors.len(), 70); + assert_eq!(lengths(&bodies), [32, 32, 6]); + for body in bodies.lock().unwrap().iter() { + assert_eq!(body["model"], "qwen3-embedding:0.6b"); + assert_eq!(body["options"]["num_thread"], 8); + } + assert_eq!(bodies.lock().unwrap()[2]["input"][5], "text 69"); + + let (url, _) = fake_ollama(true, false); + let err = OllamaEmbedder::new(&url, "m") + .embed(&["a", "b"]) + .unwrap_err(); + assert!(err.to_string().contains("got 1"), "{err}"); + + let (url, _) = fake_ollama(false, true); + let err = OllamaEmbedder::new(&url, "m") + .embed(&["a", "b"]) + .unwrap_err(); + assert!(err.to_string().contains("dimension"), "{err}"); +} + +#[test] +fn ollama_clamps_batch_and_omits_unset_num_thread() { + let (url, bodies) = fake_ollama(false, false); + let mut embedder = OllamaEmbedder::new(&url, "m"); + embedder.batch = 500; + embedder.num_thread = None; + let texts: Vec = (0..40).map(|i| i.to_string()).collect(); + let refs: Vec<&str> = texts.iter().map(String::as_str).collect(); + embedder.embed(&refs).unwrap(); + assert_eq!(lengths(&bodies), [32, 8]); + assert!(bodies.lock().unwrap()[0].get("options").is_none()); + assert!(embedder.embed(&[]).unwrap().is_empty()); + assert_eq!(lengths(&bodies).len(), 2, "no request for no texts"); +} + +#[test] +#[ignore = "needs WEKSA_OLLAMA_URL, Yggdrasil's ollama-cpu"] +fn ollama_on_yggdrasil() { + let url = std::env::var("WEKSA_OLLAMA_URL").unwrap(); + let embedder = OllamaEmbedder::new(url, "qwen3-embedding:0.6b"); + let text = "Take your hand off that control."; + let vectors = embedder.embed(&[text, text]).unwrap(); + assert_eq!(vectors.len(), 2); + assert!(vectors.iter().all(|v| v.len() == 1024)); + assert!((cosine(&vectors[0], &vectors[1]) - 1.0).abs() < 1e-4); +} + +fn score(chrf: f64, meaning: Option) -> ItemScore { + ItemScore { chrf, meaning } +} + +#[test] +fn aggregate_by_derived_split() { + let train = item_in("item:a:1", "t", &group_in_bucket(10), cast()); + let train2 = item_in("item:a:2", "t", &group_in_bucket(10), cast()); + let validation = item_in("item:b:1", "v", &group_in_bucket(50), cast()); + let test = item_in("item:c:1", "x", &group_in_bucket(90), cast()); + let test_model = item_in("item:c:2", "x", &group_in_bucket(90), model()); + let agg = aggregate(&[ + (&train, score(10.0, Some(0.5))), + (&train2, score(20.0, None)), + (&validation, score(40.0, Some(0.8))), + (&test, score(60.0, Some(0.6))), + (&test_model, score(100.0, Some(1.0))), + ]); + assert_eq!(agg.train.count, 2); + assert_eq!(agg.train.chrf, Some(15.0)); + assert_eq!(agg.train.meaning, Some(0.5), "None is not averaged as zero"); + assert_eq!((agg.validation.count, agg.validation.chrf), (1, Some(40.0))); + assert_eq!((agg.test.count, agg.test.chrf), (1, Some(60.0))); + assert_eq!( + agg.test.meaning, + Some(0.6), + "the model-written test item is in no split" + ); + assert_eq!(aggregate(&[]).validation.chrf, None); +} + +const HELD_OUT: &str = "extraordinary lighthouse keepers whispered unforgettable stories throughout midnight thunderstorms before surrendering peacefully"; +const UNRELATED: &str = "harbor ledgers record ordinary tides"; + +fn chunks(size: usize, joiner: &str) -> Vec { + HELD_OUT + .split(' ') + .collect::>() + .chunks(size) + .map(|c| c.join(joiner)) + .collect() +} + +fn validation(key: &str, text: &str) -> CorpusItem { + item_in(key, text, &group_in_bucket(50), cast()) +} + +fn train(key: &str, text: &str) -> CorpusItem { + item_in(key, text, &group_in_bucket(10), cast()) +} + +/// (novel, covered) per order for the Validation split. +fn read(minted: &[&str], items: &[CorpusItem]) -> Vec<(usize, usize)> { + leak_diagnostic(minted, items) + .validation + .coverage_by_n + .iter() + .map(|c| (c.novel, c.covered)) + .collect() +} + +fn refs(strings: &[String]) -> Vec<&str> { + strings.iter().map(String::as_str).collect() +} + +#[test] +fn leak_diagnostic_catches_chunked_glosses() { + let minted = chunks(3, " "); + let minted = refs(&minted); + let held = validation("item:v:1", HELD_OUT); + let unrelated = validation("item:v:2", "gardening is quiet work"); + let train_item = train("item:t:1", UNRELATED); + let test_model = item_in("item:x:1", HELD_OUT, &group_in_bucket(90), model()); + + let d = leak_diagnostic( + &minted, + &[ + held.clone(), + unrelated.clone(), + train_item.clone(), + test_model, + ], + ); + let pair = d.validation.max_chrf.as_ref().unwrap(); + assert!( + pair.chrf < 50.0, + "each chunk alone scores low: {}", + pair.chrf + ); + assert_eq!(pair.item, "item:v:1"); + // pooled over both items: 12+4 words, 11+3 bigrams, 10+2 trigrams + assert_eq!( + read(&minted, &[held.clone(), unrelated, train_item.clone()]), + [(16, 12), (14, 8), (12, 4)] + ); + let worst = d.validation.worst.as_ref().unwrap(); + assert_eq!( + (worst.item.as_str(), worst.n, worst.covered, worst.novel), + ("item:v:1", 2, 8, 11) + ); + assert_eq!(d.validation.items, 2); + assert_eq!(d.test.items, 0, "a model-written test item is in no split"); + assert_eq!(d.test.max_chrf, None); + assert_eq!(d.test.worst, None); + + let only_train = leak_diagnostic(&minted, &[train_item]); + assert_eq!(only_train.validation.items, 0); + assert_eq!(only_train.validation.max_chrf, None); + assert_eq!(only_train.validation.worst, None); + + let full = leak_diagnostic(&[HELD_OUT], &[held]); + assert_eq!(full.validation.coverage_by_n[2].share, Some(1.0)); + assert!(full.validation.max_chrf.unwrap().chrf > 99.0); +} + +#[test] +fn coverage_reads_joined_labels() { + let items = [ + validation("item:v:1", HELD_OUT), + train("item:t:1", UNRELATED), + ]; + for size in [1, 2, 3] { + let plain = chunks(size, " "); + let want = read(&refs(&plain), &items); + for joiner in ["_", "."] { + let joined = chunks(size, joiner); + assert_eq!(read(&refs(&joined), &items), want, "{size} {joiner:?}"); + } + // camelCase: each word after the first capitalised within a chunk + let camel: Vec = chunks(size, " ") + .iter() + .map(|c| { + c.split(' ') + .enumerate() + .map(|(i, w)| match i { + 0 => w.to_string(), + _ => w[..1].to_uppercase() + &w[1..], + }) + .collect::() + }) + .collect(); + assert_eq!(read(&refs(&camel), &items), want, "{size} camel"); + } + let items = [ + validation("item:v:1", "He will kick the bucket"), + train("item:t:1", "he will go"), + ]; + for label in [ + "kick_the_bucket", + "kick.the.bucket", + "kickTheBucket", + "kick the bucket", + ] { + let d = leak_diagnostic(&[label], &items); + let by_n = &d.validation.coverage_by_n; + assert_eq!((by_n[2].novel, by_n[2].covered), (3, 1), "{label}"); + let worst = d.validation.worst.unwrap(); + assert_eq!((worst.n, worst.covered), (2, 2), "{label}"); + } +} + +#[test] +fn coverage_is_case_insensitive() { + let items = [ + validation("item:v:1", HELD_OUT), + train("item:t:1", UNRELATED), + ]; + let minted = chunks(2, " "); + let shouted: Vec = minted.iter().map(|m| m.to_uppercase()).collect(); + assert_eq!(read(&refs(&shouted), &items), read(&refs(&minted), &items)); + assert_eq!(read(&refs(&minted), &items)[0], (12, 12)); +} + +#[test] +fn coverage_pooled_per_order() { + // honest single-word glosses saturate n=1 and read nothing above it + let items = [ + train( + "item:t:1", + "They left before the storm and found the harbor.", + ), + validation("item:v:1", "They left the group before the storm."), + validation("item:v:2", "They found the place after the storm."), + validation("item:v:3", "Warn them before they leave the harbor."), + ]; + let glosses = [ + "left", "group", "found", "place", "storm", "after", "warn", "leave", "harbor", + ]; + let d = leak_diagnostic(&glosses, &items); + let by_n = &d.validation.coverage_by_n; + assert_eq!(by_n.iter().map(|c| c.n).collect::>(), [1, 2, 3]); + assert_eq!((by_n[0].novel, by_n[0].covered), (6, 5)); + assert!(by_n[0].share.unwrap() < 1.0); + for c in &by_n[1..] { + assert!(c.novel > 0); + assert_eq!((c.covered, c.share), (0, Some(0.0))); + } + assert_eq!(d.validation.worst, None); + + // no novel n-gram reads None, never NaN; an empty split reads the same + let items = [ + train("item:t:1", "tides harbors guard the quiet shore"), + validation("item:v:1", "harbors guard the quiet"), + ]; + let d = leak_diagnostic(&["anything at all"], &items); + for split in [&d.validation, &d.test] { + assert_eq!(split.coverage_by_n.len(), 3); + assert!( + split + .coverage_by_n + .iter() + .all(|c| (c.novel, c.share) == (0, None)) + ); + assert_eq!(split.worst, None); + } +} + +#[test] +fn coverage_keeps_digits_and_strings() { + let items = [ + validation("item:v:1", "code 0400 open"), + train("item:t:1", "code open"), + ]; + assert_eq!(read(&["0400"], &items)[0], (1, 1)); + // a bigram spanning two minted strings is not covered + let items = [validation("item:v:1", "zeta eta")]; + assert_eq!(read(&["zeta", "eta"], &items), [(2, 2), (1, 0), (0, 0)]); + assert_eq!(read(&["zeta eta"], &items), [(2, 2), (1, 1), (0, 0)]); +} + +#[test] +fn worst_item_rules() { + // equal covered counts: the larger share wins over an earlier, smaller one + let items = [ + validation("item:v:1", "alpha beta gamma"), + validation("item:v:2", "delta epsilon"), + ]; + let w = leak_diagnostic(&["alpha beta", "delta epsilon"], &items) + .validation + .worst + .unwrap(); + assert_eq!( + (w.item.as_str(), w.n, w.covered, w.novel), + ("item:v:2", 2, 1, 1) + ); + // a later, smaller share does not displace an earlier larger one + let items = [ + validation("item:v:1", "delta epsilon"), + validation("item:v:2", "alpha beta gamma"), + ]; + let w = leak_diagnostic(&["alpha beta", "delta epsilon"], &items) + .validation + .worst + .unwrap(); + assert_eq!(w.item, "item:v:1"); + // a larger covered count beats a larger share + let items = [ + validation("item:v:1", "delta epsilon"), + validation("item:v:2", "alpha beta gamma delta"), + ]; + let w = leak_diagnostic(&["alpha beta gamma", "delta epsilon"], &items) + .validation + .worst + .unwrap(); + assert_eq!((w.item.as_str(), w.covered, w.novel), ("item:v:2", 2, 3)); + // full ties go to the first item, then the smaller n + let items = [ + validation("item:v:1", "delta epsilon"), + validation("item:v:2", "delta epsilon"), + ]; + let w = leak_diagnostic(&["delta epsilon"], &items) + .validation + .worst + .unwrap(); + assert_eq!(w.item, "item:v:1"); + let items = [ + validation("item:v:1", "alpha beta gamma"), + train("item:t:1", "alpha beta"), + ]; + let w = leak_diagnostic(&["alpha beta gamma"], &items) + .validation + .worst + .unwrap(); + assert_eq!((w.n, w.covered, w.novel), (2, 1, 1)); +} + +#[test] +fn ollama_reports_its_model() { + assert_eq!( + OllamaEmbedder::new("http://x", "qwen3-embedding:0.6b").model(), + "qwen3-embedding:0.6b" + ); +} + +#[test] +fn leak_diagnostic_reports_the_best_pair_and_the_first_of_ties() { + let a = validation("item:v:1", HELD_OUT); + let b = validation("item:v:2", HELD_OUT); + // the better minted string comes second; equal items tie, the first is kept + let d = leak_diagnostic(&["zzzzzz qqqqqq", HELD_OUT], &[a, b]); + let pair = d.validation.max_chrf.unwrap(); + assert_eq!( + (pair.minted.as_str(), pair.item.as_str()), + (HELD_OUT, "item:v:1") + ); +} + +#[test] +fn scores_whole_texts() { + let reference = "He is going to kick the bucket tomorrow."; + let output = "He is going to die tomorrow."; + // exhaustive: a scorer input that grows a field breaks this line + let ScoreInput { reference, output } = ScoreInput { reference, output }; + let scorer = Scorer::new(Recorder::default()); + let got = score_one(&scorer, reference, output); + assert_eq!(*scorer.embedder().seen.borrow(), [reference, output]); + assert_eq!(got.chrf, chrf(output, reference)); +} + +#[test] +fn filler_costs() { + // naming filler cannot delete it: the scorer has no input but the two texts + let reference = "I am not angry."; + let filler = "Darling, I am truly not angry, darling."; + let scorer = Scorer::new(Bag); + close(score_one(&scorer, reference, filler).chrf, 46.8378); + let copy_reference = "He is going to kick the bucket tomorrow."; + let padded = "Well, he is going to kick the bucket tomorrow, honestly."; + let unnamed = score_one(&scorer, copy_reference, padded); + close(unnamed.chrf, chrf(padded, copy_reference)); + assert!(unnamed.chrf < 100.0); + let honest = score_one(&scorer, "Never again.", "Never again."); + assert_eq!(honest.chrf, 100.0); +} + +#[test] +fn honest_name_earns_credit() { + let scorer = Scorer::new(Bag); + let reference = "Brokkr came back."; + assert_eq!(score_one(&scorer, reference, reference).chrf, 100.0); + let wrong = score_one(&scorer, reference, "Kestrel came back."); + assert!(wrong.chrf < 100.0); + assert!(wrong.meaning.unwrap() < 1.0); +} + +#[test] +fn every_script_alike() { + let scorer = Scorer::new(Bag); + for text in [ + "민수가 왔다", + "Brokkr'ın gemisi geldi", + "Brokkrin laiva tuli", + "สมชายมาแล้ว", + "ブロッカーが来た。", + ] { + let got = score_one(&scorer, text, text); + assert_eq!(got.chrf, 100.0, "{text}"); + } +}