From 5e2a4a0ebbe67d7f29dea37abe0f2070f8e54bb7 Mon Sep 17 00:00:00 2001 From: Weksa Date: Mon, 9 Nov 2026 07:47:17 +0000 Subject: [PATCH] Scorer coverage: words break at any non-word character, unspaced scripts read by run; near-miss chrF pairs per script Soul scorer s3 Lows. words.rs no longer lists separators (_ and .) or defers to UAX #29, which splits Han, kana and Thai per character and made honest single-word glosses read as an n>=2 leak. A word is a run of letters, digits, marks and ZWNJ/ZWJ; a run of Han, kana, Thai, Lao, Khmer or Burmese is one word. Tests pin separators, marks, joiners, NFC, every range edge, and five sacrebleu 2.4.3 near-miss pairs (Ko, Tr, Fi, Th vowel-strip, Ja). 40 hand mutants killed. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/score/words.rs | 106 +++++++++++----- crates/weksa/tests/score.rs | 152 +++++++++++++++++++++++ docs/targets/interlingua-lowering-map.md | 9 +- 3 files changed, 236 insertions(+), 31 deletions(-) diff --git a/crates/weksa/src/score/words.rs b/crates/weksa/src/score/words.rs index 58e7c1f..4909ee9 100644 --- a/crates/weksa/src/score/words.rs +++ b/crates/weksa/src/score/words.rs @@ -1,44 +1,94 @@ -//! The one way the scorer reads words: UAX #29 word segmentation, then NFC and -//! full Unicode case folding. The novel-coverage diagnostic reads text through -//! here, so every script is treated by one rule. +//! The one way the scorer reads words, then NFC and full Unicode case folding. +//! The novel-coverage diagnostic reads text through here, so every script is +//! treated by one rule. +//! +//! A word is a run of word characters: alphanumerics, combining marks and the +//! joiners ZWNJ and ZWJ. Any other character ends it, so `kick:the:bucket`, +//! `kick·the·bucket`, `kick_the_bucket` and `kick'the'bucket` read as +//! `kick the bucket`, as does `kickTheBucket` (a lowercase-to-uppercase +//! transition also ends a word). +//! +//! Han, kana, Thai, Lao, Khmer and Burmese write no word boundaries, and the +//! scorer holds no dictionary to find them. A run of those characters is one +//! word, never one word per character: per-character words would make every +//! honest single-word gloss share bigrams with a held-out sentence, which is +//! the leak signature. A change between such a run and any other script also +//! ends a word. A leak in these scripts shows at n = 1 and in chrF; a +//! dictionary segmenter is the ja-JP panel's to add. use caseless::default_case_fold_str; use unicode_normalization::UnicodeNormalization; -use unicode_segmentation::UnicodeSegmentation; +use unicode_normalization::char::is_combining_mark; /// NFC, then full default case folding. pub(super) fn normalise(s: &str) -> String { default_case_fold_str(&s.nfc().collect::()) } -/// The pieces of one segment: split at `_` and `.` (which are dropped) and -/// at every lowercase-to-uppercase transition, so `kick_the_bucket`, -/// `kick.the.bucket` and `kickTheBucket` read as `kick the bucket`. -fn pieces(segment: &str) -> Vec<&str> { +fn is_joiner(c: char) -> bool { + matches!(c, '\u{200c}' | '\u{200d}') +} + +/// A character that belongs inside a word. Marks must stay: Thai vowels and +/// tone marks, Hindi vowel signs and Hangul jamo are not alphanumeric. +fn is_word_char(c: char) -> bool { + c.is_alphanumeric() || is_combining_mark(c) || is_joiner(c) +} + +/// Han, kana, Thai, Lao, Khmer and Burmese: scripts written without spaces. +fn is_unspaced(c: char) -> bool { + matches!( + c, + '\u{3005}' + | '\u{3007}' + | '\u{3021}'..='\u{3029}' + | '\u{3038}'..='\u{303b}' + | '\u{3041}'..='\u{30ff}' + | '\u{3400}'..='\u{4dbf}' + | '\u{4e00}'..='\u{9fff}' + | '\u{f900}'..='\u{fad9}' + | '\u{ff66}'..='\u{ff9f}' + | '\u{31f0}'..='\u{31ff}' + | '\u{20000}'..='\u{323af}' + | '\u{0e01}'..='\u{0e5b}' + | '\u{0e81}'..='\u{0edf}' + | '\u{1000}'..='\u{109f}' + | '\u{1780}'..='\u{17ff}' + ) +} + +/// The normalised words of `text` in order; runs with no alphanumeric char +/// (marks alone, say) are dropped. +pub(super) fn words(text: &str) -> Vec { let mut out = Vec::new(); let mut start = 0; let mut previous: Option = None; - for (at, c) in segment.char_indices() { - let separator = c == '_' || c == '.'; - let camel = previous.is_some_and(char::is_lowercase) && c.is_uppercase(); - if separator || camel { - out.push(&segment[start..at]); - start = if separator { at + c.len_utf8() } else { at }; + let end = |from: usize, to: usize, out: &mut Vec| { + let word = &text[from..to]; + if word.chars().any(char::is_alphanumeric) { + out.push(normalise(word)); + } + }; + for (at, c) in text.char_indices() { + if !is_word_char(c) { + end(start, at, &mut out); + start = at + c.len_utf8(); + continue; + } + // a mark or joiner belongs to the character before it + let attached = is_combining_mark(c) || is_joiner(c); + let breaks = !attached + && previous.is_some_and(|p| { + (p.is_lowercase() && c.is_uppercase()) || is_unspaced(p) != is_unspaced(c) + }); + if breaks { + end(start, at, &mut out); + start = at; + } + if !attached { + previous = Some(c); } - previous = Some(c); } - out.push(&segment[start..]); + end(start, text.len(), &mut out); out } - -/// The normalised words of `text` in order: UAX #29 segments containing an -/// alphanumeric char (digits kept), each split by `pieces`, empty or -/// non-alphanumeric pieces dropped. -pub(super) fn words(text: &str) -> Vec { - text.split_word_bounds() - .filter(|segment| segment.chars().any(char::is_alphanumeric)) - .flat_map(pieces) - .filter(|piece| piece.chars().any(char::is_alphanumeric)) - .map(normalise) - .collect() -} diff --git a/crates/weksa/tests/score.rs b/crates/weksa/tests/score.rs index 6920b93..e8c363a 100644 --- a/crates/weksa/tests/score.rs +++ b/crates/weksa/tests/score.rs @@ -690,3 +690,155 @@ fn every_script_alike() { assert_eq!(got.chrf, 100.0, "{text}"); } } + +/// (reference, hypothesis, sacrebleu 2.4.3 sentence_chrf), one near miss per +/// script: the Thai hypothesis is the reference with its vowels and tone mark +/// stripped. +const SCRIPT_PAIRS: [(&str, &str, f64); 5] = [ + ("민수가 왔다", "민수는 왔다", 26.0), + ("Brokkr'ın gemisi geldi", "Brokkr'ın gemisi gitti", 77.7548), + ("Brokkrin laiva tuli", "Brokkrin laiva meni", 73.002), + ("สมชายมาแล้ว", "สมชยมแลว", 21.8989), + ("ブロッカーが来た。", "ブロッカーが行った。", 54.4013), +]; + +#[test] +fn every_script_scores_a_near_miss_as_sacrebleu_does() { + let scorer = Scorer::new(Bag); + for (reference, hypothesis, want) in SCRIPT_PAIRS { + close(score_one(&scorer, reference, hypothesis).chrf, want); + } +} + +#[test] +fn coverage_splits_at_any_non_word_character() { + let items = [ + validation("item:v:1", "mind the hatch before the storm"), + train("item:t:1", UNRELATED), + ]; + let want = read(&["mind the hatch"], &items); + assert_eq!(want, [(5, 3), (5, 2), (4, 1)]); + for joiner in [ + ":", "\u{b7}", "'", "\u{2019}", "\u{ff3f}", "-", "/", "\u{2014}", "|", "+", "\u{ff1a}", + "\u{30fb}", + ] { + let glued = format!("mind{joiner}the{joiner}hatch"); + assert_eq!(read(&[&glued], &items), want, "{joiner:?}"); + } + // a mark, a joiner or a digit is part of its word, not a break + let items = [validation( + "item:v:1", + "\u{e2a}\u{e21}\u{e0a}\u{e32}\u{e22} \u{e21}\u{e32}\u{e41}\u{e25}\u{e49}\u{e27} 0400 \u{628}\u{200c}\u{628}", + )]; + assert_eq!( + read(&["\u{e21}\u{e32}\u{e41}\u{e25}\u{e49}\u{e27}"], &items)[0], + (4, 1) + ); + assert_eq!(read(&["\u{628}\u{200c}\u{628}", "0400"], &items)[0], (4, 2)); + // a joiner keeps its word whole: the minted half is not a held-out word + let items = [validation("item:v:1", "\u{628}\u{200c}\u{62a} \u{62c}")]; + assert_eq!(read(&["\u{628}\u{200c}\u{62a}"], &items)[0], (2, 1)); + // decomposed and composed spellings are one word + let items = [validation("item:v:1", "Cafe\u{301}")]; + assert_eq!(read(&["Caf\u{e9}"], &items)[0], (1, 1)); +} + +#[test] +fn coverage_reads_unspaced_scripts_by_run() { + // Han, kana and Thai write no word boundaries and the scorer holds no + // dictionary, so a run of them is one word, never one word per character. + // Honest single-word glosses then read nothing above n=1, as they do in a + // spaced script. + for (held, glosses) in [ + ( + "\u{5c71}\u{7530}\u{592a}\u{90ce}\u{304c}\u{6765}\u{305f}", + ["\u{5c71}\u{7530}", "\u{592a}\u{90ce}", "\u{6765}\u{305f}"], + ), + ( + "\u{30d6}\u{30ed}\u{30c3}\u{30ab}\u{30fc}\u{304c}\u{6765}\u{305f}\u{3002}", + ["\u{30d6}\u{30ed}", "\u{30ab}\u{30fc}", "\u{6765}"], + ), + ( + "\u{e2a}\u{e21}\u{e0a}\u{e32}\u{e22}\u{e21}\u{e32}\u{e41}\u{e25}\u{e49}\u{e27}", + [ + "\u{e2a}\u{e21}", + "\u{e0a}\u{e32}\u{e22}", + "\u{e41}\u{e25}\u{e49}", + ], + ), + ] { + let items = [validation("item:v:1", held), train("item:t:1", UNRELATED)]; + assert_eq!(read(&glosses, &items), [(1, 0), (0, 0), (0, 0)], "{held}"); + assert_eq!(read(&[held], &items), [(1, 1), (0, 0), (0, 0)], "{held}"); + } + // Delimiters still make words: spaces and punctuation between runs, and a + // change between an unspaced script and a spaced one. + let items = [ + validation( + "item:v:1", + "\u{72ac}\u{304c} \u{8d70}\u{308b}\u{3001}\u{732b}\u{3082} Brokkr\u{5c71}", + ), + train("item:t:1", UNRELATED), + ]; + assert_eq!( + read( + &["\u{72ac}\u{304c} \u{8d70}\u{308b} \u{732b}\u{3082} brokkr \u{5c71}"], + &items + ), + [(5, 5), (4, 4), (3, 3)] + ); +} + +#[test] +fn an_unspaced_script_ends_a_word_beside_any_other_script() { + // the first and last letters or digits of every range the rule names + let unspaced = [ + '\u{3005}', + '\u{3007}', + '\u{3021}', + '\u{3029}', + '\u{3038}', + '\u{303b}', + '\u{3041}', + '\u{30ff}', + '\u{31f0}', + '\u{31ff}', + '\u{3400}', + '\u{4dbf}', + '\u{4e00}', + '\u{9fff}', + '\u{f900}', + '\u{fad9}', + '\u{ff66}', + '\u{ff9f}', + '\u{20000}', + '\u{2a6df}', + '\u{31350}', + '\u{323af}', + '\u{e01}', + '\u{e59}', + '\u{e81}', + '\u{edf}', + '\u{1000}', + '\u{1049}', + '\u{1780}', + '\u{17e9}', + ]; + for c in unspaced { + assert!(c.is_alphanumeric(), "{c:?} is a letter or digit"); + let items = [validation("item:v:1", &format!("ab{c}"))]; + assert_eq!(read(&["ab", &c.to_string()], &items)[0], (2, 2), "{c:?}"); + } + // Hangul, Latin and Cyrillic are spaced scripts: no break between them + for c in ['\u{ac00}', '\u{d7a3}', '\u{e9}', '\u{44f}', '7'] { + let glued = format!("ab{c}"); + let items = [validation("item:v:1", &glued)]; + assert_eq!(read(&[&glued], &items)[0], (1, 1), "{c:?}"); + assert_eq!(read(&["ab"], &items)[0], (1, 0), "{c:?}"); + } + // a combining mark or a joiner after an unspaced character stays on it + let held = "\u{4e00}\u{301}\u{200d}ab"; + let items = [validation("item:v:1", held)]; + assert_eq!(read(&["\u{4e00}\u{301}\u{200d}", "ab"], &items)[0], (2, 2)); + assert_eq!(read(&["\u{4e00}"], &items)[0], (2, 0)); +} diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index c828266..4517320 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -202,9 +202,12 @@ operator-attested annotations alone. Who annotates is question text against held-out items; novel coverage pooled per order, for n = 1, 2 and 3 separately, the covered share of all novel n-grams (word n-grams of a held-out item that occur in no Train item) over a split's items, with the - worst item at n >= 2. Keys split on `_`, `.` and lower-to-upper case - transitions as well as UAX #29 boundaries, so a snake_case `OptionLabel` - reads as words. Honest glosses of single words saturate n = 1 (the 54 seed + worst item at n >= 2. A word is a run of letters, digits, marks and + joiners; any other character, and a lower-to-upper case transition, ends it, + so `kick:the:bucket` and a snake_case `OptionLabel` read as words. Han, kana, + Thai, Lao, Khmer and Burmese write no word boundaries and the scorer holds no + dictionary, so a run of them is one word (per-character words would give + every honest gloss shared bigrams); a leak there shows at n = 1 and in chrF. Honest glosses of single words saturate n = 1 (the 54 seed glosses read max 1.000, s2), so the leak signature is pooled n >= 2. Costs: a live persona mints before it names (one typed write, which also buys coreference and one rendering per language); public-domain intake