scorer: chrF and meaning cosine of whole output against whole reference, no mask; Embedder port with Ollama adapter; pooled novel-n-gram leak diagnostics; aggregation by derived split (squash of weksa-lowering/scorer onto 20865ca, head 5689e62; spec r3, Soul s1-s3)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
20865cac59
commit
869c82b686
10 changed files with 1311 additions and 6 deletions
76
Cargo.lock
generated
76
Cargo.lock
generated
|
|
@ -82,6 +82,12 @@ version = "1.5.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.23.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.13.2"
|
||||
|
|
@ -94,6 +100,21 @@ version = "3.20.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
|
||||
|
||||
[[package]]
|
||||
name = "caseless"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b6fd507454086c8edfd769ca6ada439193cdb209c7681712ef6275cccbfe5d8"
|
||||
dependencies = [
|
||||
"unicode-normalization",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.10.0"
|
||||
|
|
@ -285,6 +306,22 @@ version = "0.5.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||
|
||||
[[package]]
|
||||
name = "http"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"itoa",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "httparse"
|
||||
version = "1.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone"
|
||||
version = "0.1.65"
|
||||
|
|
@ -386,6 +423,12 @@ version = "1.70.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||
|
||||
[[package]]
|
||||
name = "percent-encoding"
|
||||
version = "2.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.17"
|
||||
|
|
@ -686,6 +729,37 @@ version = "1.13.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
|
||||
|
||||
[[package]]
|
||||
name = "ureq"
|
||||
version = "3.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cde93ff05758a36a851a398a772b0b79a82fbe3c186a46f2696baaeaf84c966b"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"log",
|
||||
"percent-encoding",
|
||||
"ureq-proto",
|
||||
"utf8-zero",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ureq-proto"
|
||||
version = "0.6.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "841583fc36089d7d59419568ae157f43a5a03a2e5a81ba87c77d1596ef46e922"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"http",
|
||||
"httparse",
|
||||
"log",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "utf8-zero"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e"
|
||||
|
||||
[[package]]
|
||||
name = "utf8parse"
|
||||
version = "0.2.2"
|
||||
|
|
@ -753,6 +827,7 @@ name = "weksa"
|
|||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"caseless",
|
||||
"clap",
|
||||
"cultcache-rs",
|
||||
"schemars",
|
||||
|
|
@ -761,6 +836,7 @@ dependencies = [
|
|||
"tempfile",
|
||||
"unicode-normalization",
|
||||
"unicode-segmentation",
|
||||
"ureq",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
|
|||
12
README.md
12
README.md
|
|
@ -18,8 +18,10 @@ measured.
|
|||
|
||||
- [Interlingua Standard](./docs/interlingua-standard.md): the packet contract.
|
||||
- `crates/weksa`: the packet, inventory and corpus types, `validate`, the
|
||||
derived split and the `weksa` CLI (`weksa validate`, `corpus`, `entity`, `inventory`,
|
||||
`split`, `schema`); published schemas in `schemas/cultnet/`.
|
||||
derived split, the scorer (`score`: chrF, meaning cosine through an Ollama
|
||||
embedder, whole output against whole reference with no mask, leak diagnostics) and the `weksa` CLI
|
||||
(`weksa validate`, `corpus`, `entity`, `inventory`, `split`, `schema`);
|
||||
published schemas in `schemas/cultnet/`.
|
||||
- `data/corpus/corpus.cc`, `data/entity/entity.cc` and `data/interlingua/inventory.cc`: the English
|
||||
corpus, its entities and the inventory, read with `weksa corpus dump`,
|
||||
`weksa entity dump` and `weksa inventory dump`.
|
||||
|
|
@ -30,8 +32,8 @@ measured.
|
|||
- `settings/`: setting-specific proving grounds for language projects.
|
||||
|
||||
**Live versus planned.** Live: docs, fixtures, examples, ontology data and the
|
||||
packet crate `crates/weksa`. Planned: scoring, lowering, and
|
||||
the golf loop that scores lowered English against an expressive corpus. No
|
||||
packet crate `crates/weksa`, including the scoring library. Planned: lowering,
|
||||
the `weksa score` subcommand, and the golf loop that scores lowered English against an expressive corpus. No
|
||||
daemon runs; the earlier Node daemon is parked at tag `parked/weksa-node-body`.
|
||||
|
||||
## Ownership Model
|
||||
|
|
@ -92,7 +94,7 @@ AquaSynth or expanding pronunciation coverage.
|
|||
|
||||
The persistent state spine is live: map, scratch, schemas, docs, examples and
|
||||
data. Packet types, validation and the corpus and inventory stores are live in
|
||||
`crates/weksa`. Scoring, lowering and the service are planned (see the cut
|
||||
`crates/weksa`, and so is the scoring library. Lowering and the service are planned (see the cut
|
||||
sequence in [the lowering map](./docs/targets/interlingua-lowering-map.md)).
|
||||
Writing and agent state will enter Weksa as intent pressure, and typed utterance
|
||||
documents will leave for AquaSynth.
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ path = "src/main.rs"
|
|||
|
||||
[dependencies]
|
||||
anyhow = "1"
|
||||
caseless = "0.2"
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
cultcache-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "64b86e0a273a3936939357e9deb186b7bf3bc417", package = "cultcache-rs" }
|
||||
schemars = "1"
|
||||
|
|
@ -21,6 +22,7 @@ serde = { version = "1", features = ["derive"] }
|
|||
serde_json = "1"
|
||||
unicode-normalization = "0.1"
|
||||
unicode-segmentation = "1"
|
||||
ureq = { version = "3", default-features = false }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
//! Weksa's typed interlingua state: packet, inventory and corpus documents in
|
||||
//! CultCache, the validator that decides whether a packet exists, the derived
|
||||
//! train/validation/test split, and the stores that admit them.
|
||||
//! train/validation/test split, the stores that admit them, and the scorer
|
||||
//! (chrF, meaning cosine, leak diagnostics).
|
||||
|
||||
pub mod corpus;
|
||||
pub mod entity;
|
||||
|
|
@ -8,6 +9,7 @@ pub mod form;
|
|||
pub mod inventory;
|
||||
pub mod packet;
|
||||
pub mod schema;
|
||||
pub mod score;
|
||||
pub mod split;
|
||||
pub mod store;
|
||||
pub mod validate;
|
||||
|
|
@ -22,6 +24,7 @@ pub use inventory::{
|
|||
pub use packet::{
|
||||
EncoderId, EntityId, Grade, NodeId, Packet, PacketId, PacketKind, SCHEMA_VERSION,
|
||||
};
|
||||
pub use score::{Embedder, ItemScore, OllamaEmbedder, Scorer, chrf};
|
||||
pub use split::{Split, SplitScheme};
|
||||
pub use store::{
|
||||
AddOutcome, CorpusStore, EntityBound, EntityStore, InventoryStore, PacketStore, StoreError,
|
||||
|
|
|
|||
60
crates/weksa/src/score/chrf.rs
Normal file
60
crates/weksa/src/score/chrf.rs
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
//! chrF as sacrebleu 2.4.3 computes it by default: character n-gram orders 1 to
|
||||
//! 6, whitespace removed, case kept, beta 2.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// The metric this module implements; a score stored beside it names its definition.
|
||||
pub const CHRF_VERSION: &str = "chrF2|nc:6|nw:0|space:no|sacrebleu:2.4.3";
|
||||
|
||||
const ORDER: usize = 6;
|
||||
const BETA_SQ: f64 = 4.0;
|
||||
|
||||
fn grams(chars: &[char], n: usize) -> HashMap<&[char], u32> {
|
||||
let mut counts = HashMap::new();
|
||||
for window in chars.windows(n) {
|
||||
*counts.entry(window).or_insert(0) += 1;
|
||||
}
|
||||
counts
|
||||
}
|
||||
|
||||
/// Python `str.split`'s whitespace, which sacrebleu strips before gramming:
|
||||
/// Unicode White_Space plus U+001C..=U+001F.
|
||||
pub(super) fn is_split_space(c: char) -> bool {
|
||||
c.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&c)
|
||||
}
|
||||
|
||||
/// chrF of `hypothesis` against `reference`, 0 to 100. Beta 2 weights recall,
|
||||
/// so the arguments are not interchangeable. 0 when no order has n-grams on
|
||||
/// both sides.
|
||||
pub fn chrf(hypothesis: &str, reference: &str) -> f64 {
|
||||
let strip = |s: &str| {
|
||||
s.chars()
|
||||
.filter(|c| !is_split_space(*c))
|
||||
.collect::<Vec<char>>()
|
||||
};
|
||||
let (hyp, reference) = (strip(hypothesis), strip(reference));
|
||||
let (mut precision, mut recall, mut orders) = (0.0, 0.0, 0u32);
|
||||
for n in 1..=ORDER {
|
||||
let (hyp_grams, ref_grams) = (grams(&hyp, n), grams(&reference, n));
|
||||
let (hyp_total, ref_total): (u32, u32) =
|
||||
(hyp_grams.values().sum(), ref_grams.values().sum());
|
||||
if hyp_total == 0 || ref_total == 0 {
|
||||
continue;
|
||||
}
|
||||
let matched: u32 = hyp_grams
|
||||
.iter()
|
||||
.map(|(gram, count)| (*count).min(ref_grams.get(gram).copied().unwrap_or(0)))
|
||||
.sum();
|
||||
precision += f64::from(matched) / f64::from(hyp_total);
|
||||
recall += f64::from(matched) / f64::from(ref_total);
|
||||
orders += 1;
|
||||
}
|
||||
if orders == 0 {
|
||||
return 0.0;
|
||||
}
|
||||
let (p, r) = (precision / f64::from(orders), recall / f64::from(orders));
|
||||
if p + r == 0.0 {
|
||||
return 0.0;
|
||||
}
|
||||
100.0 * (1.0 + BETA_SQ) * p * r / (BETA_SQ * p + r)
|
||||
}
|
||||
169
crates/weksa/src/score/diagnostics.rs
Normal file
169
crates/weksa/src/score/diagnostics.rs
Normal file
|
|
@ -0,0 +1,169 @@
|
|||
//! Leak diagnostics: values reported beside scores, never refusals. Pooled
|
||||
//! coverage at n >= 2 above zero beside single-word glosses, or a high pairwise
|
||||
//! chrF, means a flow bug or a corpus duplicate across splits.
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
use super::chrf::chrf;
|
||||
use super::words::words;
|
||||
use crate::corpus::CorpusItem;
|
||||
use crate::split::{Split, SplitScheme};
|
||||
|
||||
/// Word n-gram orders read, 1 to this.
|
||||
const ORDERS: usize = 3;
|
||||
|
||||
/// The pair that reached the highest chrF.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct PairChrf {
|
||||
pub chrf: f64,
|
||||
pub minted: String,
|
||||
pub item: String,
|
||||
}
|
||||
|
||||
/// One word-order's pooled novel coverage over a split's items.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct OrderCoverage {
|
||||
pub n: usize,
|
||||
/// Distinct n-grams of the split's items absent from Train, summed over items.
|
||||
pub novel: usize,
|
||||
/// Of those, the ones that occur in the minted text.
|
||||
pub covered: usize,
|
||||
/// `covered / novel`; `None` when `novel` is 0, never NaN.
|
||||
pub share: Option<f64>,
|
||||
}
|
||||
|
||||
/// The item that read most at `n` >= 2.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct WorstItem {
|
||||
pub item: String,
|
||||
pub n: usize,
|
||||
pub covered: usize,
|
||||
pub novel: usize,
|
||||
}
|
||||
|
||||
/// One held-out split's readings.
|
||||
#[derive(Clone, Debug, PartialEq, Default)]
|
||||
pub struct SplitLeak {
|
||||
/// Items of the split.
|
||||
pub items: usize,
|
||||
/// Max chrF of any minted string (hypothesis) against any item (reference).
|
||||
pub max_chrf: Option<PairChrf>,
|
||||
/// Pooled novel coverage for n in 1..=3, in order.
|
||||
pub coverage_by_n: Vec<OrderCoverage>,
|
||||
/// The item and order (n >= 2) with the largest covered count, then the
|
||||
/// larger share; the first item, then the smaller n, on ties. `None` when
|
||||
/// no item has a covered n-gram at n >= 2.
|
||||
pub worst: Option<WorstItem>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Default)]
|
||||
pub struct LeakDiagnostic {
|
||||
pub validation: SplitLeak,
|
||||
pub test: SplitLeak,
|
||||
}
|
||||
|
||||
/// The distinct word n-grams of a text for each order 1..=ORDERS, over its
|
||||
/// normalised word segments.
|
||||
fn grams(text: &str) -> Vec<HashSet<String>> {
|
||||
let keys = words(text);
|
||||
(1..=ORDERS)
|
||||
.map(|n| keys.windows(n).map(|w| w.join(" ")).collect())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn union<'a>(texts: impl Iterator<Item = &'a str>) -> Vec<HashSet<String>> {
|
||||
let mut all = vec![HashSet::new(); ORDERS];
|
||||
for text in texts {
|
||||
for (into, from) in all.iter_mut().zip(grams(text)) {
|
||||
into.extend(from);
|
||||
}
|
||||
}
|
||||
all
|
||||
}
|
||||
|
||||
fn split_leak(
|
||||
minted: &[&str],
|
||||
train: &[HashSet<String>],
|
||||
minted_grams: &[HashSet<String>],
|
||||
items: &[&CorpusItem],
|
||||
) -> SplitLeak {
|
||||
let mut leak = SplitLeak {
|
||||
items: items.len(),
|
||||
..SplitLeak::default()
|
||||
};
|
||||
let mut totals = [(0usize, 0usize); ORDERS];
|
||||
for item in items {
|
||||
for m in minted {
|
||||
let score = chrf(m, &item.text);
|
||||
if leak.max_chrf.as_ref().is_none_or(|best| score > best.chrf) {
|
||||
leak.max_chrf = Some(PairChrf {
|
||||
chrf: score,
|
||||
minted: (*m).to_string(),
|
||||
item: item.id.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
for (i, item_grams) in grams(&item.text).iter().enumerate() {
|
||||
let novel: Vec<&String> = item_grams
|
||||
.iter()
|
||||
.filter(|g| !train[i].contains(*g))
|
||||
.collect();
|
||||
let covered = novel
|
||||
.iter()
|
||||
.filter(|g| minted_grams[i].contains(**g))
|
||||
.count();
|
||||
totals[i].0 += novel.len();
|
||||
totals[i].1 += covered;
|
||||
if i == 0 || covered == 0 {
|
||||
continue;
|
||||
}
|
||||
let share = covered as f64 / novel.len() as f64;
|
||||
let better = leak.worst.as_ref().is_none_or(|w| {
|
||||
covered > w.covered
|
||||
|| (covered == w.covered && share > w.covered as f64 / w.novel as f64)
|
||||
});
|
||||
if better {
|
||||
leak.worst = Some(WorstItem {
|
||||
item: item.id.clone(),
|
||||
n: i + 1,
|
||||
covered,
|
||||
novel: novel.len(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
leak.coverage_by_n = totals
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (novel, covered))| OrderCoverage {
|
||||
n: i + 1,
|
||||
novel: *novel,
|
||||
covered: *covered,
|
||||
share: (*novel > 0).then(|| *covered as f64 / *novel as f64),
|
||||
})
|
||||
.collect();
|
||||
leak
|
||||
}
|
||||
|
||||
/// Reads minted text (inventory glosses, option labels, later ontology text)
|
||||
/// against the held-out items. Per held-out item and for n in 1..=3, its word
|
||||
/// n-grams that occur in no Train item are its novel n-grams; coverage at each
|
||||
/// n is the covered over the novel n-grams summed across the split's items
|
||||
/// (unigrams over all minted strings, longer grams within one string). Honest
|
||||
/// single-word glosses saturate n = 1, so the leak signature is pooled n >= 2
|
||||
/// and the worst item at n >= 2. Train items are not measured; items in no
|
||||
/// split are dropped.
|
||||
pub fn leak_diagnostic(minted: &[&str], items: &[CorpusItem]) -> LeakDiagnostic {
|
||||
let of = |want: Split| -> Vec<&CorpusItem> {
|
||||
items
|
||||
.iter()
|
||||
.filter(|i| SplitScheme::V1.split(i) == Some(want))
|
||||
.collect()
|
||||
};
|
||||
let train = union(of(Split::Train).iter().map(|i| i.text.as_str()));
|
||||
let minted_grams = union(minted.iter().copied());
|
||||
LeakDiagnostic {
|
||||
validation: split_leak(minted, &train, &minted_grams, &of(Split::Validation)),
|
||||
test: split_leak(minted, &train, &minted_grams, &of(Split::Test)),
|
||||
}
|
||||
}
|
||||
113
crates/weksa/src/score/embed.rs
Normal file
113
crates/weksa/src/score/embed.rs
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
//! The meaning metric's embedding port and its Ollama adapter.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result, bail, ensure};
|
||||
use serde::Deserialize;
|
||||
use serde_json::json;
|
||||
|
||||
/// Sentence embeddings. Every embedder (Ollama now, a style embedder later)
|
||||
/// enters the scorer through this port.
|
||||
pub trait Embedder {
|
||||
fn model(&self) -> &str;
|
||||
/// One vector per text, in order, all of one dimension.
|
||||
fn embed(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>>;
|
||||
}
|
||||
|
||||
/// Cosine similarity. Precondition: `a` and `b` have one equal nonzero length
|
||||
/// and nonzero norm; `Scorer::score` refuses embedder output that does not.
|
||||
/// Violating it panics rather than reading as a score.
|
||||
pub fn cosine(a: &[f32], b: &[f32]) -> f64 {
|
||||
assert!(
|
||||
!a.is_empty() && a.len() == b.len(),
|
||||
"cosine needs vectors of one nonzero length, got {} and {}",
|
||||
a.len(),
|
||||
b.len()
|
||||
);
|
||||
let dot: f64 = a
|
||||
.iter()
|
||||
.zip(b)
|
||||
.map(|(x, y)| f64::from(*x) * f64::from(*y))
|
||||
.sum();
|
||||
let norm = |v: &[f32]| v.iter().map(|x| f64::from(*x).powi(2)).sum::<f64>().sqrt();
|
||||
let denom = norm(a) * norm(b);
|
||||
assert!(denom > 0.0, "cosine of a zero vector");
|
||||
dot / denom
|
||||
}
|
||||
|
||||
/// Ollama's `/api/embed` over plain HTTP. One server serves Huginn and the
|
||||
/// loop, so batches stay small and `num_thread` is sent (without it a batch of
|
||||
/// 32 timed out at 120 s under host load).
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct OllamaEmbedder {
|
||||
pub url: String,
|
||||
pub model: String,
|
||||
/// Sent as `options.num_thread`; omitted when `None`.
|
||||
pub num_thread: Option<u32>,
|
||||
/// Texts per request, clamped to 1..=32.
|
||||
pub batch: usize,
|
||||
}
|
||||
|
||||
impl OllamaEmbedder {
|
||||
pub const MAX_BATCH: usize = 32;
|
||||
|
||||
pub fn new(url: impl Into<String>, model: impl Into<String>) -> Self {
|
||||
Self {
|
||||
url: url.into(),
|
||||
model: model.into(),
|
||||
num_thread: Some(8),
|
||||
batch: Self::MAX_BATCH,
|
||||
}
|
||||
}
|
||||
|
||||
fn request(&self, agent: &ureq::Agent, batch: &[&str]) -> Result<Vec<Vec<f32>>> {
|
||||
let mut body = json!({ "model": self.model, "input": batch });
|
||||
if let Some(threads) = self.num_thread {
|
||||
body["options"] = json!({ "num_thread": threads });
|
||||
}
|
||||
let endpoint = format!("{}/api/embed", self.url.trim_end_matches('/'));
|
||||
let mut response = agent
|
||||
.post(&endpoint)
|
||||
.header("Content-Type", "application/json")
|
||||
.send(body.to_string())
|
||||
.with_context(|| format!("POST {endpoint}"))?;
|
||||
#[derive(Deserialize)]
|
||||
struct Reply {
|
||||
embeddings: Vec<Vec<f32>>,
|
||||
}
|
||||
let reply: Reply = serde_json::from_str(&response.body_mut().read_to_string()?)
|
||||
.context("ollama /api/embed reply")?;
|
||||
ensure!(
|
||||
reply.embeddings.len() == batch.len(),
|
||||
"asked for {} embeddings, got {}",
|
||||
batch.len(),
|
||||
reply.embeddings.len()
|
||||
);
|
||||
Ok(reply.embeddings)
|
||||
}
|
||||
}
|
||||
|
||||
impl Embedder for OllamaEmbedder {
|
||||
fn model(&self) -> &str {
|
||||
&self.model
|
||||
}
|
||||
|
||||
fn embed(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
|
||||
if texts.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let agent: ureq::Agent = ureq::Agent::config_builder()
|
||||
.timeout_global(Some(Duration::from_secs(120)))
|
||||
.build()
|
||||
.into();
|
||||
let mut out: Vec<Vec<f32>> = Vec::with_capacity(texts.len());
|
||||
for batch in texts.chunks(self.batch.clamp(1, Self::MAX_BATCH)) {
|
||||
out.extend(self.request(&agent, batch)?);
|
||||
}
|
||||
let dim = out[0].len();
|
||||
if dim == 0 || out.iter().any(|v| v.len() != dim) {
|
||||
bail!("embeddings disagree on dimension");
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
144
crates/weksa/src/score/mod.rs
Normal file
144
crates/weksa/src/score/mod.rs
Normal file
|
|
@ -0,0 +1,144 @@
|
|||
//! What a score counts: chrF and the meaning cosine of the whole lowered
|
||||
//! output against the whole reference. Nothing is deleted, hidden or replaced
|
||||
//! before either is read, and the scorer takes no input from a packet, so no
|
||||
//! text the output carries can raise a score except by resembling the
|
||||
//! reference. No door judges text; the diagnostics report.
|
||||
|
||||
mod chrf;
|
||||
mod diagnostics;
|
||||
mod embed;
|
||||
mod words;
|
||||
|
||||
use anyhow::{Result, ensure};
|
||||
|
||||
pub use chrf::{CHRF_VERSION, chrf};
|
||||
pub use diagnostics::{
|
||||
LeakDiagnostic, OrderCoverage, PairChrf, SplitLeak, WorstItem, leak_diagnostic,
|
||||
};
|
||||
pub use embed::{Embedder, OllamaEmbedder, cosine};
|
||||
|
||||
use crate::corpus::CorpusItem;
|
||||
use crate::split::{Split, SplitScheme};
|
||||
|
||||
/// One reference/output pair; both are scored whole.
|
||||
pub struct ScoreInput<'a> {
|
||||
pub reference: &'a str,
|
||||
pub output: &'a str,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct ItemScore {
|
||||
/// chrF of the output against the reference.
|
||||
pub chrf: f64,
|
||||
/// Embedding cosine of the output and the reference; `None` when either
|
||||
/// is empty after trimming.
|
||||
pub meaning: Option<f64>,
|
||||
}
|
||||
|
||||
pub struct Scorer<E: Embedder> {
|
||||
embedder: E,
|
||||
}
|
||||
|
||||
/// Every vector has one nonzero length and a nonzero norm.
|
||||
fn check_vectors(vectors: &[Vec<f32>]) -> Result<()> {
|
||||
let dim = vectors.first().map_or(0, Vec::len);
|
||||
for vector in vectors {
|
||||
ensure!(
|
||||
dim > 0 && vector.len() == dim,
|
||||
"embedder returned vectors of length {} and {}",
|
||||
dim,
|
||||
vector.len()
|
||||
);
|
||||
let norm: f64 = vector.iter().map(|x| f64::from(*x).powi(2)).sum();
|
||||
ensure!(
|
||||
norm > 0.0 && norm.is_finite(),
|
||||
"embedder returned a zero or non-finite vector"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl<E: Embedder> Scorer<E> {
|
||||
pub fn new(embedder: E) -> Self {
|
||||
Self { embedder }
|
||||
}
|
||||
|
||||
pub fn embedder(&self) -> &E {
|
||||
&self.embedder
|
||||
}
|
||||
|
||||
/// Scores every input with one embed call; the caller chunks `inputs`.
|
||||
/// Errs when the embedder's vectors disagree in length or are empty or zero.
|
||||
pub fn score(&self, inputs: &[ScoreInput<'_>]) -> Result<Vec<ItemScore>> {
|
||||
let mut texts: Vec<&str> = Vec::new();
|
||||
let slots: Vec<Option<usize>> = inputs
|
||||
.iter()
|
||||
.map(|input| {
|
||||
if input.output.trim().is_empty() || input.reference.trim().is_empty() {
|
||||
return None;
|
||||
}
|
||||
texts.push(input.reference);
|
||||
texts.push(input.output);
|
||||
Some(texts.len() - 2)
|
||||
})
|
||||
.collect();
|
||||
let vectors = self.embedder.embed(&texts)?;
|
||||
ensure!(
|
||||
vectors.len() == texts.len(),
|
||||
"embedder returned {} vectors for {} texts",
|
||||
vectors.len(),
|
||||
texts.len()
|
||||
);
|
||||
check_vectors(&vectors)?;
|
||||
Ok(inputs
|
||||
.iter()
|
||||
.zip(slots)
|
||||
.map(|(input, slot)| ItemScore {
|
||||
chrf: chrf(input.output, input.reference),
|
||||
meaning: slot.map(|i| cosine(&vectors[i], &vectors[i + 1])),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
|
||||
/// Mean scores of one derived split.
|
||||
#[derive(Clone, Debug, PartialEq, Default)]
|
||||
pub struct SplitMean {
|
||||
pub count: usize,
|
||||
/// `None` when the split has no items.
|
||||
pub chrf: Option<f64>,
|
||||
/// Mean over the items that have a meaning score; `None` when none do.
|
||||
pub meaning: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Default)]
|
||||
pub struct Aggregate {
|
||||
pub train: SplitMean,
|
||||
pub validation: SplitMean,
|
||||
pub test: SplitMean,
|
||||
}
|
||||
|
||||
/// Groups scores by the item's derived split. Items in no split are counted
|
||||
/// nowhere. The scorer only groups: the loop decides when Test is read.
|
||||
pub fn aggregate(scored: &[(&CorpusItem, ItemScore)]) -> Aggregate {
|
||||
let mean = |want: Split| {
|
||||
let of: Vec<&ItemScore> = scored
|
||||
.iter()
|
||||
.filter(|(item, _)| SplitScheme::V1.split(item) == Some(want))
|
||||
.map(|(_, score)| score)
|
||||
.collect();
|
||||
let average = |values: Vec<f64>| {
|
||||
(!values.is_empty()).then(|| values.iter().sum::<f64>() / values.len() as f64)
|
||||
};
|
||||
SplitMean {
|
||||
count: of.len(),
|
||||
chrf: average(of.iter().map(|s| s.chrf).collect()),
|
||||
meaning: average(of.iter().filter_map(|s| s.meaning).collect()),
|
||||
}
|
||||
};
|
||||
Aggregate {
|
||||
train: mean(Split::Train),
|
||||
validation: mean(Split::Validation),
|
||||
test: mean(Split::Test),
|
||||
}
|
||||
}
|
||||
44
crates/weksa/src/score/words.rs
Normal file
44
crates/weksa/src/score/words.rs
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
//! The one way the scorer reads words: UAX #29 word segmentation, then NFC and
|
||||
//! full Unicode case folding. The novel-coverage diagnostic reads text through
|
||||
//! here, so every script is treated by one rule.
|
||||
|
||||
use caseless::default_case_fold_str;
|
||||
use unicode_normalization::UnicodeNormalization;
|
||||
use unicode_segmentation::UnicodeSegmentation;
|
||||
|
||||
/// NFC, then full default case folding.
|
||||
pub(super) fn normalise(s: &str) -> String {
|
||||
default_case_fold_str(&s.nfc().collect::<String>())
|
||||
}
|
||||
|
||||
/// The pieces of one segment: split at `_` and `.` (which are dropped) and
|
||||
/// at every lowercase-to-uppercase transition, so `kick_the_bucket`,
|
||||
/// `kick.the.bucket` and `kickTheBucket` read as `kick the bucket`.
|
||||
fn pieces(segment: &str) -> Vec<&str> {
|
||||
let mut out = Vec::new();
|
||||
let mut start = 0;
|
||||
let mut previous: Option<char> = None;
|
||||
for (at, c) in segment.char_indices() {
|
||||
let separator = c == '_' || c == '.';
|
||||
let camel = previous.is_some_and(char::is_lowercase) && c.is_uppercase();
|
||||
if separator || camel {
|
||||
out.push(&segment[start..at]);
|
||||
start = if separator { at + c.len_utf8() } else { at };
|
||||
}
|
||||
previous = Some(c);
|
||||
}
|
||||
out.push(&segment[start..]);
|
||||
out
|
||||
}
|
||||
|
||||
/// The normalised words of `text` in order: UAX #29 segments containing an
|
||||
/// alphanumeric char (digits kept), each split by `pieces`, empty or
|
||||
/// non-alphanumeric pieces dropped.
|
||||
pub(super) fn words(text: &str) -> Vec<String> {
|
||||
text.split_word_bounds()
|
||||
.filter(|segment| segment.chars().any(char::is_alphanumeric))
|
||||
.flat_map(pieces)
|
||||
.filter(|piece| piece.chars().any(char::is_alphanumeric))
|
||||
.map(normalise)
|
||||
.collect()
|
||||
}
|
||||
692
crates/weksa/tests/score.rs
Normal file
692
crates/weksa/tests/score.rs
Normal file
|
|
@ -0,0 +1,692 @@
|
|||
mod common;
|
||||
|
||||
use std::cell::RefCell;
|
||||
use std::io::{BufRead, BufReader, Read, Write};
|
||||
use std::net::TcpListener;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use anyhow::Result;
|
||||
use common::{cast, group_in_bucket, item_in, model};
|
||||
use serde_json::Value;
|
||||
use weksa::CorpusItem;
|
||||
use weksa::score::{
|
||||
Embedder, ItemScore, OllamaEmbedder, ScoreInput, Scorer, aggregate, chrf, cosine,
|
||||
leak_diagnostic,
|
||||
};
|
||||
|
||||
/// (reference, hypothesis, sacrebleu 2.4.3 sentence_chrf), map 491536c.
|
||||
const VECTORS: [(&str, &str, f64); 6] = [
|
||||
(
|
||||
"Take your hand off that control unless you want your nervous system to become my least useful circuit diagram.",
|
||||
"Take your hand off that control, unless you'd like your nervous system to be my least useful wiring diagram.",
|
||||
74.4498,
|
||||
),
|
||||
(
|
||||
"The command has no receipt. It is not truth.",
|
||||
"The order carries no receipt, so it is not true.",
|
||||
45.7164,
|
||||
),
|
||||
("I am not angry.", "I am not angry.", 100.0),
|
||||
("Forgive me, sir.", "Get out.", 3.9683),
|
||||
("その制御に触るな。", "その操作盤に触らないで。", 14.2931),
|
||||
("Well, aren't you a sight for sore eyes, love.", "", 0.0),
|
||||
];
|
||||
|
||||
#[test]
|
||||
fn chrf_matches_sacrebleu_vectors() {
|
||||
for (reference, hypothesis, want) in VECTORS {
|
||||
let got = chrf(hypothesis, reference);
|
||||
assert!(
|
||||
(got - want).abs() < 1e-4,
|
||||
"{hypothesis:?} against {reference:?}: {got} != {want}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chrf_argument_order() {
|
||||
let short = "mind the hatch";
|
||||
let long = "mind the hatch before the storm arrives tonight";
|
||||
let (as_hypothesis, as_reference) = (chrf(short, long), chrf(long, short));
|
||||
assert!(
|
||||
as_hypothesis < as_reference - 1.0,
|
||||
"recall is weighted: a short hypothesis of a long reference ({as_hypothesis}) must score below the reverse ({as_reference})"
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Recorder {
|
||||
seen: RefCell<Vec<String>>,
|
||||
}
|
||||
|
||||
impl Embedder for Recorder {
|
||||
fn model(&self) -> &str {
|
||||
"fake"
|
||||
}
|
||||
fn embed(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
|
||||
self.seen
|
||||
.borrow_mut()
|
||||
.extend(texts.iter().map(|t| t.to_string()));
|
||||
Ok(texts.iter().map(|t| vec![t.len() as f32, 1.0]).collect())
|
||||
}
|
||||
}
|
||||
|
||||
/// A deterministic bag-of-letters embedder, so cosine differs with the text.
|
||||
struct Bag;
|
||||
|
||||
impl Embedder for Bag {
|
||||
fn model(&self) -> &str {
|
||||
"bag"
|
||||
}
|
||||
fn embed(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
|
||||
Ok(texts
|
||||
.iter()
|
||||
.map(|t| {
|
||||
let mut v = vec![0.0; 27];
|
||||
v[26] = 1.0;
|
||||
for c in t.chars().filter(char::is_ascii_alphabetic) {
|
||||
v[(c.to_ascii_lowercase() as u8 - b'a') as usize] += 1.0;
|
||||
}
|
||||
v
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
|
||||
fn score_one<E: Embedder>(scorer: &Scorer<E>, reference: &str, output: &str) -> ItemScore {
|
||||
scorer
|
||||
.score(&[ScoreInput { reference, output }])
|
||||
.unwrap()
|
||||
.remove(0)
|
||||
}
|
||||
|
||||
fn close(got: f64, want: f64) {
|
||||
assert!((got - want).abs() < 1e-4, "{got} != {want}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chrf_whitespace_parity() {
|
||||
// sacrebleu 2.4.3 strips str.split's whitespace; every vector scores 100
|
||||
let vectors = [
|
||||
("a\tb c", "ab c"),
|
||||
("a\u{a0}b", "ab"),
|
||||
("a\u{1c}b", "ab"),
|
||||
("a\u{85}b", "ab"),
|
||||
("a\u{2028}b c", "ab c"),
|
||||
("line one\nline two", "line one line two"),
|
||||
("その\u{3000}制御", "その制御"),
|
||||
];
|
||||
for (hypothesis, reference) in vectors {
|
||||
assert_eq!(chrf(hypothesis, reference), 100.0, "{hypothesis:?}");
|
||||
assert_eq!(chrf(reference, hypothesis), 100.0, "{hypothesis:?}");
|
||||
}
|
||||
// a zero-width space is not whitespace
|
||||
close(chrf("a\u{200b}b", "ab"), 45.4545);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scorer_argument_order() {
|
||||
let short = "mind the hatch";
|
||||
let long = "mind the hatch before the storm arrives tonight";
|
||||
let scorer = Scorer::new(Bag);
|
||||
let got = score_one(&scorer, long, short);
|
||||
assert_eq!(got.chrf, chrf(short, long));
|
||||
assert_ne!(got.chrf, chrf(long, short));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meaning_none_on_empty_text() {
|
||||
let scorer = Scorer::new(Recorder::default());
|
||||
let inputs = [
|
||||
ScoreInput {
|
||||
reference: "Nibu says hi",
|
||||
output: " ",
|
||||
},
|
||||
ScoreInput {
|
||||
reference: " ",
|
||||
output: "says hi",
|
||||
},
|
||||
ScoreInput {
|
||||
reference: "Nibu says hi",
|
||||
output: "says hi",
|
||||
},
|
||||
];
|
||||
let scores = scorer.score(&inputs).unwrap();
|
||||
assert_eq!(scores[0].meaning, None);
|
||||
assert_eq!(scores[1].meaning, None);
|
||||
assert!(scores[2].meaning.is_some());
|
||||
assert_eq!(
|
||||
*scorer.embedder().seen.borrow(),
|
||||
["Nibu says hi", "says hi"]
|
||||
);
|
||||
assert_eq!(scores[0].chrf, 0.0);
|
||||
}
|
||||
|
||||
/// Answers every embed call with the same vectors.
|
||||
struct Fixed(Vec<Vec<f32>>);
|
||||
|
||||
impl Embedder for Fixed {
|
||||
fn model(&self) -> &str {
|
||||
"fixed"
|
||||
}
|
||||
fn embed(&self, _texts: &[&str]) -> Result<Vec<Vec<f32>>> {
|
||||
Ok(self.0.clone())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scorer_refuses_bad_vectors() {
|
||||
let run = |vectors: Vec<Vec<f32>>| {
|
||||
Scorer::new(Fixed(vectors)).score(&[ScoreInput {
|
||||
reference: "mind the hatch",
|
||||
output: "mind the door",
|
||||
}])
|
||||
};
|
||||
assert!(run(vec![vec![1.0, 2.0], vec![2.0, 1.0]]).is_ok());
|
||||
for bad in [
|
||||
vec![vec![1.0, 2.0], vec![1.0]],
|
||||
vec![vec![1.0], vec![1.0, 2.0]],
|
||||
vec![vec![], vec![]],
|
||||
vec![vec![0.0, 0.0], vec![0.0, 0.0]],
|
||||
vec![vec![1.0, 2.0], vec![0.0, 0.0]],
|
||||
vec![vec![0.0, 0.0], vec![1.0, 2.0]],
|
||||
vec![vec![1.0, f32::NAN], vec![1.0, 2.0]],
|
||||
vec![vec![1.0, 2.0]],
|
||||
] {
|
||||
assert!(run(bad.clone()).is_err(), "{bad:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cosine_precondition() {
|
||||
assert!((cosine(&[1.0, 2.0], &[2.0, 4.0]) - 1.0).abs() < 1e-9);
|
||||
assert!(cosine(&[1.0, 0.0], &[0.0, 1.0]).abs() < 1e-9);
|
||||
for (a, b) in [
|
||||
(vec![1.0], vec![1.0, 1.0]),
|
||||
(vec![], vec![]),
|
||||
(vec![0.0, 0.0], vec![1.0, 1.0]),
|
||||
] {
|
||||
let refused = std::panic::catch_unwind(|| cosine(&a, &b));
|
||||
assert!(refused.is_err(), "{a:?} {b:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// A plain-HTTP fake of `/api/embed`. Records each request body; answers with
|
||||
/// `dim` floats per input, one fewer vector than asked when `drop_one`, and a
|
||||
/// short last vector when `short_last`.
|
||||
fn fake_ollama(drop_one: bool, short_last: bool) -> (String, Arc<Mutex<Vec<Value>>>) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let url = format!("http://{}", listener.local_addr().unwrap());
|
||||
let bodies = Arc::new(Mutex::new(Vec::new()));
|
||||
let seen = Arc::clone(&bodies);
|
||||
std::thread::spawn(move || {
|
||||
for stream in listener.incoming() {
|
||||
let mut stream = stream.unwrap();
|
||||
let mut reader = BufReader::new(stream.try_clone().unwrap());
|
||||
'conn: loop {
|
||||
let mut length = 0;
|
||||
let mut line = String::new();
|
||||
let mut first = true;
|
||||
loop {
|
||||
line.clear();
|
||||
if reader.read_line(&mut line).unwrap_or(0) == 0 {
|
||||
break 'conn;
|
||||
}
|
||||
if first {
|
||||
assert!(line.starts_with("POST /api/embed "), "{line}");
|
||||
first = false;
|
||||
}
|
||||
let lower = line.to_ascii_lowercase();
|
||||
if let Some(v) = lower.strip_prefix("content-length:") {
|
||||
length = v.trim().parse().unwrap();
|
||||
}
|
||||
if line == "\r\n" {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let mut body = vec![0u8; length];
|
||||
reader.read_exact(&mut body).unwrap();
|
||||
let body: Value = serde_json::from_slice(&body).unwrap();
|
||||
let mut count = body["input"].as_array().unwrap().len();
|
||||
seen.lock().unwrap().push(body);
|
||||
if drop_one {
|
||||
count -= 1;
|
||||
}
|
||||
let vectors: Vec<Vec<f32>> = (0..count)
|
||||
.map(|i| vec![1.0; if short_last && i + 1 == count { 2 } else { 3 }])
|
||||
.collect();
|
||||
let reply = serde_json::json!({ "embeddings": vectors }).to_string();
|
||||
write!(
|
||||
stream,
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{reply}",
|
||||
reply.len()
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
});
|
||||
(url, bodies)
|
||||
}
|
||||
|
||||
fn lengths(bodies: &Mutex<Vec<Value>>) -> Vec<usize> {
|
||||
let bodies = bodies.lock().unwrap();
|
||||
bodies
|
||||
.iter()
|
||||
.map(|b| b["input"].as_array().unwrap().len())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ollama_request_shape_and_batching() {
|
||||
let (url, bodies) = fake_ollama(false, false);
|
||||
let embedder = OllamaEmbedder::new(&url, "qwen3-embedding:0.6b");
|
||||
let texts: Vec<String> = (0..70).map(|i| format!("text {i}")).collect();
|
||||
let refs: Vec<&str> = texts.iter().map(String::as_str).collect();
|
||||
let vectors = embedder.embed(&refs).unwrap();
|
||||
assert_eq!(vectors.len(), 70);
|
||||
assert_eq!(lengths(&bodies), [32, 32, 6]);
|
||||
for body in bodies.lock().unwrap().iter() {
|
||||
assert_eq!(body["model"], "qwen3-embedding:0.6b");
|
||||
assert_eq!(body["options"]["num_thread"], 8);
|
||||
}
|
||||
assert_eq!(bodies.lock().unwrap()[2]["input"][5], "text 69");
|
||||
|
||||
let (url, _) = fake_ollama(true, false);
|
||||
let err = OllamaEmbedder::new(&url, "m")
|
||||
.embed(&["a", "b"])
|
||||
.unwrap_err();
|
||||
assert!(err.to_string().contains("got 1"), "{err}");
|
||||
|
||||
let (url, _) = fake_ollama(false, true);
|
||||
let err = OllamaEmbedder::new(&url, "m")
|
||||
.embed(&["a", "b"])
|
||||
.unwrap_err();
|
||||
assert!(err.to_string().contains("dimension"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ollama_clamps_batch_and_omits_unset_num_thread() {
|
||||
let (url, bodies) = fake_ollama(false, false);
|
||||
let mut embedder = OllamaEmbedder::new(&url, "m");
|
||||
embedder.batch = 500;
|
||||
embedder.num_thread = None;
|
||||
let texts: Vec<String> = (0..40).map(|i| i.to_string()).collect();
|
||||
let refs: Vec<&str> = texts.iter().map(String::as_str).collect();
|
||||
embedder.embed(&refs).unwrap();
|
||||
assert_eq!(lengths(&bodies), [32, 8]);
|
||||
assert!(bodies.lock().unwrap()[0].get("options").is_none());
|
||||
assert!(embedder.embed(&[]).unwrap().is_empty());
|
||||
assert_eq!(lengths(&bodies).len(), 2, "no request for no texts");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "needs WEKSA_OLLAMA_URL, Yggdrasil's ollama-cpu"]
|
||||
fn ollama_on_yggdrasil() {
|
||||
let url = std::env::var("WEKSA_OLLAMA_URL").unwrap();
|
||||
let embedder = OllamaEmbedder::new(url, "qwen3-embedding:0.6b");
|
||||
let text = "Take your hand off that control.";
|
||||
let vectors = embedder.embed(&[text, text]).unwrap();
|
||||
assert_eq!(vectors.len(), 2);
|
||||
assert!(vectors.iter().all(|v| v.len() == 1024));
|
||||
assert!((cosine(&vectors[0], &vectors[1]) - 1.0).abs() < 1e-4);
|
||||
}
|
||||
|
||||
fn score(chrf: f64, meaning: Option<f64>) -> ItemScore {
|
||||
ItemScore { chrf, meaning }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aggregate_by_derived_split() {
|
||||
let train = item_in("item:a:1", "t", &group_in_bucket(10), cast());
|
||||
let train2 = item_in("item:a:2", "t", &group_in_bucket(10), cast());
|
||||
let validation = item_in("item:b:1", "v", &group_in_bucket(50), cast());
|
||||
let test = item_in("item:c:1", "x", &group_in_bucket(90), cast());
|
||||
let test_model = item_in("item:c:2", "x", &group_in_bucket(90), model());
|
||||
let agg = aggregate(&[
|
||||
(&train, score(10.0, Some(0.5))),
|
||||
(&train2, score(20.0, None)),
|
||||
(&validation, score(40.0, Some(0.8))),
|
||||
(&test, score(60.0, Some(0.6))),
|
||||
(&test_model, score(100.0, Some(1.0))),
|
||||
]);
|
||||
assert_eq!(agg.train.count, 2);
|
||||
assert_eq!(agg.train.chrf, Some(15.0));
|
||||
assert_eq!(agg.train.meaning, Some(0.5), "None is not averaged as zero");
|
||||
assert_eq!((agg.validation.count, agg.validation.chrf), (1, Some(40.0)));
|
||||
assert_eq!((agg.test.count, agg.test.chrf), (1, Some(60.0)));
|
||||
assert_eq!(
|
||||
agg.test.meaning,
|
||||
Some(0.6),
|
||||
"the model-written test item is in no split"
|
||||
);
|
||||
assert_eq!(aggregate(&[]).validation.chrf, None);
|
||||
}
|
||||
|
||||
const HELD_OUT: &str = "extraordinary lighthouse keepers whispered unforgettable stories throughout midnight thunderstorms before surrendering peacefully";
|
||||
const UNRELATED: &str = "harbor ledgers record ordinary tides";
|
||||
|
||||
fn chunks(size: usize, joiner: &str) -> Vec<String> {
|
||||
HELD_OUT
|
||||
.split(' ')
|
||||
.collect::<Vec<_>>()
|
||||
.chunks(size)
|
||||
.map(|c| c.join(joiner))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn validation(key: &str, text: &str) -> CorpusItem {
|
||||
item_in(key, text, &group_in_bucket(50), cast())
|
||||
}
|
||||
|
||||
fn train(key: &str, text: &str) -> CorpusItem {
|
||||
item_in(key, text, &group_in_bucket(10), cast())
|
||||
}
|
||||
|
||||
/// (novel, covered) per order for the Validation split.
|
||||
fn read(minted: &[&str], items: &[CorpusItem]) -> Vec<(usize, usize)> {
|
||||
leak_diagnostic(minted, items)
|
||||
.validation
|
||||
.coverage_by_n
|
||||
.iter()
|
||||
.map(|c| (c.novel, c.covered))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn refs(strings: &[String]) -> Vec<&str> {
|
||||
strings.iter().map(String::as_str).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn leak_diagnostic_catches_chunked_glosses() {
|
||||
let minted = chunks(3, " ");
|
||||
let minted = refs(&minted);
|
||||
let held = validation("item:v:1", HELD_OUT);
|
||||
let unrelated = validation("item:v:2", "gardening is quiet work");
|
||||
let train_item = train("item:t:1", UNRELATED);
|
||||
let test_model = item_in("item:x:1", HELD_OUT, &group_in_bucket(90), model());
|
||||
|
||||
let d = leak_diagnostic(
|
||||
&minted,
|
||||
&[
|
||||
held.clone(),
|
||||
unrelated.clone(),
|
||||
train_item.clone(),
|
||||
test_model,
|
||||
],
|
||||
);
|
||||
let pair = d.validation.max_chrf.as_ref().unwrap();
|
||||
assert!(
|
||||
pair.chrf < 50.0,
|
||||
"each chunk alone scores low: {}",
|
||||
pair.chrf
|
||||
);
|
||||
assert_eq!(pair.item, "item:v:1");
|
||||
// pooled over both items: 12+4 words, 11+3 bigrams, 10+2 trigrams
|
||||
assert_eq!(
|
||||
read(&minted, &[held.clone(), unrelated, train_item.clone()]),
|
||||
[(16, 12), (14, 8), (12, 4)]
|
||||
);
|
||||
let worst = d.validation.worst.as_ref().unwrap();
|
||||
assert_eq!(
|
||||
(worst.item.as_str(), worst.n, worst.covered, worst.novel),
|
||||
("item:v:1", 2, 8, 11)
|
||||
);
|
||||
assert_eq!(d.validation.items, 2);
|
||||
assert_eq!(d.test.items, 0, "a model-written test item is in no split");
|
||||
assert_eq!(d.test.max_chrf, None);
|
||||
assert_eq!(d.test.worst, None);
|
||||
|
||||
let only_train = leak_diagnostic(&minted, &[train_item]);
|
||||
assert_eq!(only_train.validation.items, 0);
|
||||
assert_eq!(only_train.validation.max_chrf, None);
|
||||
assert_eq!(only_train.validation.worst, None);
|
||||
|
||||
let full = leak_diagnostic(&[HELD_OUT], &[held]);
|
||||
assert_eq!(full.validation.coverage_by_n[2].share, Some(1.0));
|
||||
assert!(full.validation.max_chrf.unwrap().chrf > 99.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn coverage_reads_joined_labels() {
|
||||
let items = [
|
||||
validation("item:v:1", HELD_OUT),
|
||||
train("item:t:1", UNRELATED),
|
||||
];
|
||||
for size in [1, 2, 3] {
|
||||
let plain = chunks(size, " ");
|
||||
let want = read(&refs(&plain), &items);
|
||||
for joiner in ["_", "."] {
|
||||
let joined = chunks(size, joiner);
|
||||
assert_eq!(read(&refs(&joined), &items), want, "{size} {joiner:?}");
|
||||
}
|
||||
// camelCase: each word after the first capitalised within a chunk
|
||||
let camel: Vec<String> = chunks(size, " ")
|
||||
.iter()
|
||||
.map(|c| {
|
||||
c.split(' ')
|
||||
.enumerate()
|
||||
.map(|(i, w)| match i {
|
||||
0 => w.to_string(),
|
||||
_ => w[..1].to_uppercase() + &w[1..],
|
||||
})
|
||||
.collect::<String>()
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(read(&refs(&camel), &items), want, "{size} camel");
|
||||
}
|
||||
let items = [
|
||||
validation("item:v:1", "He will kick the bucket"),
|
||||
train("item:t:1", "he will go"),
|
||||
];
|
||||
for label in [
|
||||
"kick_the_bucket",
|
||||
"kick.the.bucket",
|
||||
"kickTheBucket",
|
||||
"kick the bucket",
|
||||
] {
|
||||
let d = leak_diagnostic(&[label], &items);
|
||||
let by_n = &d.validation.coverage_by_n;
|
||||
assert_eq!((by_n[2].novel, by_n[2].covered), (3, 1), "{label}");
|
||||
let worst = d.validation.worst.unwrap();
|
||||
assert_eq!((worst.n, worst.covered), (2, 2), "{label}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn coverage_is_case_insensitive() {
|
||||
let items = [
|
||||
validation("item:v:1", HELD_OUT),
|
||||
train("item:t:1", UNRELATED),
|
||||
];
|
||||
let minted = chunks(2, " ");
|
||||
let shouted: Vec<String> = minted.iter().map(|m| m.to_uppercase()).collect();
|
||||
assert_eq!(read(&refs(&shouted), &items), read(&refs(&minted), &items));
|
||||
assert_eq!(read(&refs(&minted), &items)[0], (12, 12));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn coverage_pooled_per_order() {
|
||||
// honest single-word glosses saturate n=1 and read nothing above it
|
||||
let items = [
|
||||
train(
|
||||
"item:t:1",
|
||||
"They left before the storm and found the harbor.",
|
||||
),
|
||||
validation("item:v:1", "They left the group before the storm."),
|
||||
validation("item:v:2", "They found the place after the storm."),
|
||||
validation("item:v:3", "Warn them before they leave the harbor."),
|
||||
];
|
||||
let glosses = [
|
||||
"left", "group", "found", "place", "storm", "after", "warn", "leave", "harbor",
|
||||
];
|
||||
let d = leak_diagnostic(&glosses, &items);
|
||||
let by_n = &d.validation.coverage_by_n;
|
||||
assert_eq!(by_n.iter().map(|c| c.n).collect::<Vec<_>>(), [1, 2, 3]);
|
||||
assert_eq!((by_n[0].novel, by_n[0].covered), (6, 5));
|
||||
assert!(by_n[0].share.unwrap() < 1.0);
|
||||
for c in &by_n[1..] {
|
||||
assert!(c.novel > 0);
|
||||
assert_eq!((c.covered, c.share), (0, Some(0.0)));
|
||||
}
|
||||
assert_eq!(d.validation.worst, None);
|
||||
|
||||
// no novel n-gram reads None, never NaN; an empty split reads the same
|
||||
let items = [
|
||||
train("item:t:1", "tides harbors guard the quiet shore"),
|
||||
validation("item:v:1", "harbors guard the quiet"),
|
||||
];
|
||||
let d = leak_diagnostic(&["anything at all"], &items);
|
||||
for split in [&d.validation, &d.test] {
|
||||
assert_eq!(split.coverage_by_n.len(), 3);
|
||||
assert!(
|
||||
split
|
||||
.coverage_by_n
|
||||
.iter()
|
||||
.all(|c| (c.novel, c.share) == (0, None))
|
||||
);
|
||||
assert_eq!(split.worst, None);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn coverage_keeps_digits_and_strings() {
|
||||
let items = [
|
||||
validation("item:v:1", "code 0400 open"),
|
||||
train("item:t:1", "code open"),
|
||||
];
|
||||
assert_eq!(read(&["0400"], &items)[0], (1, 1));
|
||||
// a bigram spanning two minted strings is not covered
|
||||
let items = [validation("item:v:1", "zeta eta")];
|
||||
assert_eq!(read(&["zeta", "eta"], &items), [(2, 2), (1, 0), (0, 0)]);
|
||||
assert_eq!(read(&["zeta eta"], &items), [(2, 2), (1, 1), (0, 0)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worst_item_rules() {
|
||||
// equal covered counts: the larger share wins over an earlier, smaller one
|
||||
let items = [
|
||||
validation("item:v:1", "alpha beta gamma"),
|
||||
validation("item:v:2", "delta epsilon"),
|
||||
];
|
||||
let w = leak_diagnostic(&["alpha beta", "delta epsilon"], &items)
|
||||
.validation
|
||||
.worst
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
(w.item.as_str(), w.n, w.covered, w.novel),
|
||||
("item:v:2", 2, 1, 1)
|
||||
);
|
||||
// a later, smaller share does not displace an earlier larger one
|
||||
let items = [
|
||||
validation("item:v:1", "delta epsilon"),
|
||||
validation("item:v:2", "alpha beta gamma"),
|
||||
];
|
||||
let w = leak_diagnostic(&["alpha beta", "delta epsilon"], &items)
|
||||
.validation
|
||||
.worst
|
||||
.unwrap();
|
||||
assert_eq!(w.item, "item:v:1");
|
||||
// a larger covered count beats a larger share
|
||||
let items = [
|
||||
validation("item:v:1", "delta epsilon"),
|
||||
validation("item:v:2", "alpha beta gamma delta"),
|
||||
];
|
||||
let w = leak_diagnostic(&["alpha beta gamma", "delta epsilon"], &items)
|
||||
.validation
|
||||
.worst
|
||||
.unwrap();
|
||||
assert_eq!((w.item.as_str(), w.covered, w.novel), ("item:v:2", 2, 3));
|
||||
// full ties go to the first item, then the smaller n
|
||||
let items = [
|
||||
validation("item:v:1", "delta epsilon"),
|
||||
validation("item:v:2", "delta epsilon"),
|
||||
];
|
||||
let w = leak_diagnostic(&["delta epsilon"], &items)
|
||||
.validation
|
||||
.worst
|
||||
.unwrap();
|
||||
assert_eq!(w.item, "item:v:1");
|
||||
let items = [
|
||||
validation("item:v:1", "alpha beta gamma"),
|
||||
train("item:t:1", "alpha beta"),
|
||||
];
|
||||
let w = leak_diagnostic(&["alpha beta gamma"], &items)
|
||||
.validation
|
||||
.worst
|
||||
.unwrap();
|
||||
assert_eq!((w.n, w.covered, w.novel), (2, 1, 1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ollama_reports_its_model() {
|
||||
assert_eq!(
|
||||
OllamaEmbedder::new("http://x", "qwen3-embedding:0.6b").model(),
|
||||
"qwen3-embedding:0.6b"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn leak_diagnostic_reports_the_best_pair_and_the_first_of_ties() {
|
||||
let a = validation("item:v:1", HELD_OUT);
|
||||
let b = validation("item:v:2", HELD_OUT);
|
||||
// the better minted string comes second; equal items tie, the first is kept
|
||||
let d = leak_diagnostic(&["zzzzzz qqqqqq", HELD_OUT], &[a, b]);
|
||||
let pair = d.validation.max_chrf.unwrap();
|
||||
assert_eq!(
|
||||
(pair.minted.as_str(), pair.item.as_str()),
|
||||
(HELD_OUT, "item:v:1")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scores_whole_texts() {
|
||||
let reference = "He is going to kick the bucket tomorrow.";
|
||||
let output = "He is going to die tomorrow.";
|
||||
// exhaustive: a scorer input that grows a field breaks this line
|
||||
let ScoreInput { reference, output } = ScoreInput { reference, output };
|
||||
let scorer = Scorer::new(Recorder::default());
|
||||
let got = score_one(&scorer, reference, output);
|
||||
assert_eq!(*scorer.embedder().seen.borrow(), [reference, output]);
|
||||
assert_eq!(got.chrf, chrf(output, reference));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filler_costs() {
|
||||
// naming filler cannot delete it: the scorer has no input but the two texts
|
||||
let reference = "I am not angry.";
|
||||
let filler = "Darling, I am truly not angry, darling.";
|
||||
let scorer = Scorer::new(Bag);
|
||||
close(score_one(&scorer, reference, filler).chrf, 46.8378);
|
||||
let copy_reference = "He is going to kick the bucket tomorrow.";
|
||||
let padded = "Well, he is going to kick the bucket tomorrow, honestly.";
|
||||
let unnamed = score_one(&scorer, copy_reference, padded);
|
||||
close(unnamed.chrf, chrf(padded, copy_reference));
|
||||
assert!(unnamed.chrf < 100.0);
|
||||
let honest = score_one(&scorer, "Never again.", "Never again.");
|
||||
assert_eq!(honest.chrf, 100.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn honest_name_earns_credit() {
|
||||
let scorer = Scorer::new(Bag);
|
||||
let reference = "Brokkr came back.";
|
||||
assert_eq!(score_one(&scorer, reference, reference).chrf, 100.0);
|
||||
let wrong = score_one(&scorer, reference, "Kestrel came back.");
|
||||
assert!(wrong.chrf < 100.0);
|
||||
assert!(wrong.meaning.unwrap() < 1.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_script_alike() {
|
||||
let scorer = Scorer::new(Bag);
|
||||
for text in [
|
||||
"민수가 왔다",
|
||||
"Brokkr'ın gemisi geldi",
|
||||
"Brokkrin laiva tuli",
|
||||
"สมชายมาแล้ว",
|
||||
"ブロッカーが来た。",
|
||||
] {
|
||||
let got = score_one(&scorer, text, text);
|
||||
assert_eq!(got.chrf, 100.0, "{text}");
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue