Merge remote-tracking branch 'origin/main' into weksa-page/show-her-working-wt
This commit is contained in:
weksa
commit
ec38d01f2f
3 changed files with 236 additions and 31 deletions
|
|
@ -1,44 +1,94 @@
|
|||
//! The one way the scorer reads words: UAX #29 word segmentation, then NFC and
|
||||
//! full Unicode case folding. The novel-coverage diagnostic reads text through
|
||||
//! here, so every script is treated by one rule.
|
||||
//! The one way the scorer reads words, then NFC and full Unicode case folding.
|
||||
//! The novel-coverage diagnostic reads text through here, so every script is
|
||||
//! treated by one rule.
|
||||
//!
|
||||
//! A word is a run of word characters: alphanumerics, combining marks and the
|
||||
//! joiners ZWNJ and ZWJ. Any other character ends it, so `kick:the:bucket`,
|
||||
//! `kick·the·bucket`, `kick_the_bucket` and `kick'the'bucket` read as
|
||||
//! `kick the bucket`, as does `kickTheBucket` (a lowercase-to-uppercase
|
||||
//! transition also ends a word).
|
||||
//!
|
||||
//! Han, kana, Thai, Lao, Khmer and Burmese write no word boundaries, and the
|
||||
//! scorer holds no dictionary to find them. A run of those characters is one
|
||||
//! word, never one word per character: per-character words would make every
|
||||
//! honest single-word gloss share bigrams with a held-out sentence, which is
|
||||
//! the leak signature. A change between such a run and any other script also
|
||||
//! ends a word. A leak in these scripts shows at n = 1 and in chrF; a
|
||||
//! dictionary segmenter is the ja-JP panel's to add.
|
||||
|
||||
use caseless::default_case_fold_str;
|
||||
use unicode_normalization::UnicodeNormalization;
|
||||
use unicode_segmentation::UnicodeSegmentation;
|
||||
use unicode_normalization::char::is_combining_mark;
|
||||
|
||||
/// NFC, then full default case folding.
|
||||
pub(super) fn normalise(s: &str) -> String {
|
||||
default_case_fold_str(&s.nfc().collect::<String>())
|
||||
}
|
||||
|
||||
/// The pieces of one segment: split at `_` and `.` (which are dropped) and
|
||||
/// at every lowercase-to-uppercase transition, so `kick_the_bucket`,
|
||||
/// `kick.the.bucket` and `kickTheBucket` read as `kick the bucket`.
|
||||
fn pieces(segment: &str) -> Vec<&str> {
|
||||
fn is_joiner(c: char) -> bool {
|
||||
matches!(c, '\u{200c}' | '\u{200d}')
|
||||
}
|
||||
|
||||
/// A character that belongs inside a word. Marks must stay: Thai vowels and
|
||||
/// tone marks, Hindi vowel signs and Hangul jamo are not alphanumeric.
|
||||
fn is_word_char(c: char) -> bool {
|
||||
c.is_alphanumeric() || is_combining_mark(c) || is_joiner(c)
|
||||
}
|
||||
|
||||
/// Han, kana, Thai, Lao, Khmer and Burmese: scripts written without spaces.
|
||||
fn is_unspaced(c: char) -> bool {
|
||||
matches!(
|
||||
c,
|
||||
'\u{3005}'
|
||||
| '\u{3007}'
|
||||
| '\u{3021}'..='\u{3029}'
|
||||
| '\u{3038}'..='\u{303b}'
|
||||
| '\u{3041}'..='\u{30ff}'
|
||||
| '\u{3400}'..='\u{4dbf}'
|
||||
| '\u{4e00}'..='\u{9fff}'
|
||||
| '\u{f900}'..='\u{fad9}'
|
||||
| '\u{ff66}'..='\u{ff9f}'
|
||||
| '\u{31f0}'..='\u{31ff}'
|
||||
| '\u{20000}'..='\u{323af}'
|
||||
| '\u{0e01}'..='\u{0e5b}'
|
||||
| '\u{0e81}'..='\u{0edf}'
|
||||
| '\u{1000}'..='\u{109f}'
|
||||
| '\u{1780}'..='\u{17ff}'
|
||||
)
|
||||
}
|
||||
|
||||
/// The normalised words of `text` in order; runs with no alphanumeric char
|
||||
/// (marks alone, say) are dropped.
|
||||
pub(super) fn words(text: &str) -> Vec<String> {
|
||||
let mut out = Vec::new();
|
||||
let mut start = 0;
|
||||
let mut previous: Option<char> = None;
|
||||
for (at, c) in segment.char_indices() {
|
||||
let separator = c == '_' || c == '.';
|
||||
let camel = previous.is_some_and(char::is_lowercase) && c.is_uppercase();
|
||||
if separator || camel {
|
||||
out.push(&segment[start..at]);
|
||||
start = if separator { at + c.len_utf8() } else { at };
|
||||
let end = |from: usize, to: usize, out: &mut Vec<String>| {
|
||||
let word = &text[from..to];
|
||||
if word.chars().any(char::is_alphanumeric) {
|
||||
out.push(normalise(word));
|
||||
}
|
||||
};
|
||||
for (at, c) in text.char_indices() {
|
||||
if !is_word_char(c) {
|
||||
end(start, at, &mut out);
|
||||
start = at + c.len_utf8();
|
||||
continue;
|
||||
}
|
||||
// a mark or joiner belongs to the character before it
|
||||
let attached = is_combining_mark(c) || is_joiner(c);
|
||||
let breaks = !attached
|
||||
&& previous.is_some_and(|p| {
|
||||
(p.is_lowercase() && c.is_uppercase()) || is_unspaced(p) != is_unspaced(c)
|
||||
});
|
||||
if breaks {
|
||||
end(start, at, &mut out);
|
||||
start = at;
|
||||
}
|
||||
if !attached {
|
||||
previous = Some(c);
|
||||
}
|
||||
previous = Some(c);
|
||||
}
|
||||
out.push(&segment[start..]);
|
||||
end(start, text.len(), &mut out);
|
||||
out
|
||||
}
|
||||
|
||||
/// The normalised words of `text` in order: UAX #29 segments containing an
|
||||
/// alphanumeric char (digits kept), each split by `pieces`, empty or
|
||||
/// non-alphanumeric pieces dropped.
|
||||
pub(super) fn words(text: &str) -> Vec<String> {
|
||||
text.split_word_bounds()
|
||||
.filter(|segment| segment.chars().any(char::is_alphanumeric))
|
||||
.flat_map(pieces)
|
||||
.filter(|piece| piece.chars().any(char::is_alphanumeric))
|
||||
.map(normalise)
|
||||
.collect()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -690,3 +690,155 @@ fn every_script_alike() {
|
|||
assert_eq!(got.chrf, 100.0, "{text}");
|
||||
}
|
||||
}
|
||||
|
||||
/// (reference, hypothesis, sacrebleu 2.4.3 sentence_chrf), one near miss per
|
||||
/// script: the Thai hypothesis is the reference with its vowels and tone mark
|
||||
/// stripped.
|
||||
const SCRIPT_PAIRS: [(&str, &str, f64); 5] = [
|
||||
("민수가 왔다", "민수는 왔다", 26.0),
|
||||
("Brokkr'ın gemisi geldi", "Brokkr'ın gemisi gitti", 77.7548),
|
||||
("Brokkrin laiva tuli", "Brokkrin laiva meni", 73.002),
|
||||
("สมชายมาแล้ว", "สมชยมแลว", 21.8989),
|
||||
("ブロッカーが来た。", "ブロッカーが行った。", 54.4013),
|
||||
];
|
||||
|
||||
#[test]
|
||||
fn every_script_scores_a_near_miss_as_sacrebleu_does() {
|
||||
let scorer = Scorer::new(Bag);
|
||||
for (reference, hypothesis, want) in SCRIPT_PAIRS {
|
||||
close(score_one(&scorer, reference, hypothesis).chrf, want);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn coverage_splits_at_any_non_word_character() {
|
||||
let items = [
|
||||
validation("item:v:1", "mind the hatch before the storm"),
|
||||
train("item:t:1", UNRELATED),
|
||||
];
|
||||
let want = read(&["mind the hatch"], &items);
|
||||
assert_eq!(want, [(5, 3), (5, 2), (4, 1)]);
|
||||
for joiner in [
|
||||
":", "\u{b7}", "'", "\u{2019}", "\u{ff3f}", "-", "/", "\u{2014}", "|", "+", "\u{ff1a}",
|
||||
"\u{30fb}",
|
||||
] {
|
||||
let glued = format!("mind{joiner}the{joiner}hatch");
|
||||
assert_eq!(read(&[&glued], &items), want, "{joiner:?}");
|
||||
}
|
||||
// a mark, a joiner or a digit is part of its word, not a break
|
||||
let items = [validation(
|
||||
"item:v:1",
|
||||
"\u{e2a}\u{e21}\u{e0a}\u{e32}\u{e22} \u{e21}\u{e32}\u{e41}\u{e25}\u{e49}\u{e27} 0400 \u{628}\u{200c}\u{628}",
|
||||
)];
|
||||
assert_eq!(
|
||||
read(&["\u{e21}\u{e32}\u{e41}\u{e25}\u{e49}\u{e27}"], &items)[0],
|
||||
(4, 1)
|
||||
);
|
||||
assert_eq!(read(&["\u{628}\u{200c}\u{628}", "0400"], &items)[0], (4, 2));
|
||||
// a joiner keeps its word whole: the minted half is not a held-out word
|
||||
let items = [validation("item:v:1", "\u{628}\u{200c}\u{62a} \u{62c}")];
|
||||
assert_eq!(read(&["\u{628}\u{200c}\u{62a}"], &items)[0], (2, 1));
|
||||
// decomposed and composed spellings are one word
|
||||
let items = [validation("item:v:1", "Cafe\u{301}")];
|
||||
assert_eq!(read(&["Caf\u{e9}"], &items)[0], (1, 1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn coverage_reads_unspaced_scripts_by_run() {
|
||||
// Han, kana and Thai write no word boundaries and the scorer holds no
|
||||
// dictionary, so a run of them is one word, never one word per character.
|
||||
// Honest single-word glosses then read nothing above n=1, as they do in a
|
||||
// spaced script.
|
||||
for (held, glosses) in [
|
||||
(
|
||||
"\u{5c71}\u{7530}\u{592a}\u{90ce}\u{304c}\u{6765}\u{305f}",
|
||||
["\u{5c71}\u{7530}", "\u{592a}\u{90ce}", "\u{6765}\u{305f}"],
|
||||
),
|
||||
(
|
||||
"\u{30d6}\u{30ed}\u{30c3}\u{30ab}\u{30fc}\u{304c}\u{6765}\u{305f}\u{3002}",
|
||||
["\u{30d6}\u{30ed}", "\u{30ab}\u{30fc}", "\u{6765}"],
|
||||
),
|
||||
(
|
||||
"\u{e2a}\u{e21}\u{e0a}\u{e32}\u{e22}\u{e21}\u{e32}\u{e41}\u{e25}\u{e49}\u{e27}",
|
||||
[
|
||||
"\u{e2a}\u{e21}",
|
||||
"\u{e0a}\u{e32}\u{e22}",
|
||||
"\u{e41}\u{e25}\u{e49}",
|
||||
],
|
||||
),
|
||||
] {
|
||||
let items = [validation("item:v:1", held), train("item:t:1", UNRELATED)];
|
||||
assert_eq!(read(&glosses, &items), [(1, 0), (0, 0), (0, 0)], "{held}");
|
||||
assert_eq!(read(&[held], &items), [(1, 1), (0, 0), (0, 0)], "{held}");
|
||||
}
|
||||
// Delimiters still make words: spaces and punctuation between runs, and a
|
||||
// change between an unspaced script and a spaced one.
|
||||
let items = [
|
||||
validation(
|
||||
"item:v:1",
|
||||
"\u{72ac}\u{304c} \u{8d70}\u{308b}\u{3001}\u{732b}\u{3082} Brokkr\u{5c71}",
|
||||
),
|
||||
train("item:t:1", UNRELATED),
|
||||
];
|
||||
assert_eq!(
|
||||
read(
|
||||
&["\u{72ac}\u{304c} \u{8d70}\u{308b} \u{732b}\u{3082} brokkr \u{5c71}"],
|
||||
&items
|
||||
),
|
||||
[(5, 5), (4, 4), (3, 3)]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unspaced_script_ends_a_word_beside_any_other_script() {
|
||||
// the first and last letters or digits of every range the rule names
|
||||
let unspaced = [
|
||||
'\u{3005}',
|
||||
'\u{3007}',
|
||||
'\u{3021}',
|
||||
'\u{3029}',
|
||||
'\u{3038}',
|
||||
'\u{303b}',
|
||||
'\u{3041}',
|
||||
'\u{30ff}',
|
||||
'\u{31f0}',
|
||||
'\u{31ff}',
|
||||
'\u{3400}',
|
||||
'\u{4dbf}',
|
||||
'\u{4e00}',
|
||||
'\u{9fff}',
|
||||
'\u{f900}',
|
||||
'\u{fad9}',
|
||||
'\u{ff66}',
|
||||
'\u{ff9f}',
|
||||
'\u{20000}',
|
||||
'\u{2a6df}',
|
||||
'\u{31350}',
|
||||
'\u{323af}',
|
||||
'\u{e01}',
|
||||
'\u{e59}',
|
||||
'\u{e81}',
|
||||
'\u{edf}',
|
||||
'\u{1000}',
|
||||
'\u{1049}',
|
||||
'\u{1780}',
|
||||
'\u{17e9}',
|
||||
];
|
||||
for c in unspaced {
|
||||
assert!(c.is_alphanumeric(), "{c:?} is a letter or digit");
|
||||
let items = [validation("item:v:1", &format!("ab{c}"))];
|
||||
assert_eq!(read(&["ab", &c.to_string()], &items)[0], (2, 2), "{c:?}");
|
||||
}
|
||||
// Hangul, Latin and Cyrillic are spaced scripts: no break between them
|
||||
for c in ['\u{ac00}', '\u{d7a3}', '\u{e9}', '\u{44f}', '7'] {
|
||||
let glued = format!("ab{c}");
|
||||
let items = [validation("item:v:1", &glued)];
|
||||
assert_eq!(read(&[&glued], &items)[0], (1, 1), "{c:?}");
|
||||
assert_eq!(read(&["ab"], &items)[0], (1, 0), "{c:?}");
|
||||
}
|
||||
// a combining mark or a joiner after an unspaced character stays on it
|
||||
let held = "\u{4e00}\u{301}\u{200d}ab";
|
||||
let items = [validation("item:v:1", held)];
|
||||
assert_eq!(read(&["\u{4e00}\u{301}\u{200d}", "ab"], &items)[0], (2, 2));
|
||||
assert_eq!(read(&["\u{4e00}"], &items)[0], (2, 0));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -202,9 +202,12 @@ operator-attested annotations alone. Who annotates is question
|
|||
text against held-out items; novel coverage pooled per order, for n = 1, 2
|
||||
and 3 separately, the covered share of all novel n-grams (word n-grams of a
|
||||
held-out item that occur in no Train item) over a split's items, with the
|
||||
worst item at n >= 2. Keys split on `_`, `.` and lower-to-upper case
|
||||
transitions as well as UAX #29 boundaries, so a snake_case `OptionLabel`
|
||||
reads as words. Honest glosses of single words saturate n = 1 (the 54 seed
|
||||
worst item at n >= 2. A word is a run of letters, digits, marks and
|
||||
joiners; any other character, and a lower-to-upper case transition, ends it,
|
||||
so `kick:the:bucket` and a snake_case `OptionLabel` read as words. Han, kana,
|
||||
Thai, Lao, Khmer and Burmese write no word boundaries and the scorer holds no
|
||||
dictionary, so a run of them is one word (per-character words would give
|
||||
every honest gloss shared bigrams); a leak there shows at n = 1 and in chrF. Honest glosses of single words saturate n = 1 (the 54 seed
|
||||
glosses read max 1.000, s2), so the leak signature is pooled n >= 2.
|
||||
Costs: a live persona mints before it names (one typed write, which also
|
||||
buys coreference and one rendering per language); public-domain intake
|
||||
|
|
|
|||
Loading…
Reference in a new issue