diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index f469589..6739db6 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -281,12 +281,23 @@ impl CorpusStore { return Err(StoreRefusal::NotNfc { key: item.id }.into()); } let text = normalized(&item.text); - if !text.chars().any(char::is_alphanumeric) { - return Err(malformed(&item.id, "text has no alphanumeric character")); - } // Read from the text as given: NFKC maps modifier letters (ʰ) to base letters. + // A text of marks, joiners and modifiers says so; any other text with + // no alphanumeric character (punctuation, space) gets the plainer reason. if !item.text.chars().any(is_base) { - return Err(malformed(&item.id, "text has no base letter or digit")); + let marks_only = item.text.chars().any(|c| { + is_combining_mark(c) + || matches!(c, '\u{200c}' | '\u{200d}') + || is_modifier_letter(c) + }); + return Err(malformed( + &item.id, + if marks_only { + "text has no base letter or digit" + } else { + "text has no alphanumeric character" + }, + )); } if !is_calendar_date(&item.admitted_on) { return Err(malformed(&item.id, "admitted_on is not a YYYY-MM-DD date")); @@ -415,31 +426,51 @@ fn check_entities(item: &CorpusItem, entities: &EntityStore) -> Result<(), Store /// Turkish dotless i and dotted capital I read as i, katakana reads as the /// hiragana of the same syllable, and Arabic-Indic and Extended Arabic-Indic /// digits read as 0-9. Homoglyphs across scripts (Cyrillic a for Latin a) stay -/// apart: no key here reads what a character looks like. +/// apart: no key here reads what a character looks like. A combining dot above +/// (U+0307) after i is dropped, so İx and i̇x collide as the old case fold made +/// them; so does plain ix, with both. Over-merging a rare sequence errs toward +/// held, a lost collision toward leak. U+0307 after any other letter stays. fn normalized(text: &str) -> String { - let mapped: String = text.nfkc().map(fold_variant).collect(); - default_case_fold_str(&mapped) + let mapped: String = text.nfkc().flat_map(fold_variant).collect(); + // The dot goes before the last NFKC so the marks left on the i compose with + // it (i, dot, acute reads as the i with an acute). + let mut dotless = String::new(); + for c in default_case_fold_str(&mapped).chars() { + if c == '\u{307}' && dotless.ends_with('i') { + continue; + } + dotless.push(c); + } + dotless .nfkc() - .filter(|c| c.is_alphanumeric() || is_combining_mark(*c)) + .filter(|&c| c.is_alphanumeric() || is_combining_mark(c)) .collect() } /// The one-to-one folds `normalized` adds to NFKC and case folding, applied /// before the case fold (U+0130 would otherwise fold to i plus a combining dot, /// which the key keeps as a mark). -fn fold_variant(c: char) -> char { - match c { - '\u{131}' => 'i', - '\u{130}' => 'I', +fn fold_variant(c: char) -> impl Iterator { + let one = |c: char| (c, None); + let (first, second) = match c { + '\u{131}' => one('i'), + '\u{130}' => one('I'), + // Voiced katakana wa, wi, we, wo: the hiragana (U+308F-3092) plus the + // combining voiced mark, which is how NFKC spells the hiragana forms. + '\u{30f7}'..='\u{30fa}' => ( + char::from_u32(c as u32 - 0x68).unwrap_or(c), + Some('\u{3099}'), + ), // Katakana letters and small letters to hiragana; the iteration marks // ヽヾ to ゝゞ. The prolonged sound mark U+30FC stays. '\u{30a1}'..='\u{30f6}' | '\u{30fd}'..='\u{30fe}' => { - char::from_u32(c as u32 - 0x60).unwrap_or(c) + one(char::from_u32(c as u32 - 0x60).unwrap_or(c)) } - '\u{660}'..='\u{669}' => char::from(b'0' + (c as u32 - 0x660) as u8), - '\u{6f0}'..='\u{6f9}' => char::from(b'0' + (c as u32 - 0x6f0) as u8), - _ => c, - } + '\u{660}'..='\u{669}' => one(char::from(b'0' + (c as u32 - 0x660) as u8)), + '\u{6f0}'..='\u{6f9}' => one(char::from(b'0' + (c as u32 - 0x6f0) as u8)), + _ => one(c), + }; + std::iter::once(first).chain(second) } /// A character that can carry a word: alphanumeric and neither a combining diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs index a8486e7..22ecb83 100644 --- a/crates/weksa/tests/stores.rs +++ b/crates/weksa/tests/stores.rs @@ -359,6 +359,15 @@ fn s6_folds_dotless_i_kana_and_arabic_indic_digits() { ), ("gate 45", "gate \u{6f4}\u{6f5}"), ("\u{3096}", "\u{30f6}"), + // Voiced wa, wi, we, wo: katakana U+30F7-30FA read as the hiragana + // plus the combining voiced mark. + ("\u{308f}\u{3099}", "\u{30f7}"), + ("\u{3090}\u{3099}", "\u{30f8}"), + ("\u{3091}\u{3099}", "\u{30f9}"), + ("\u{3092}\u{3099}", "\u{30fa}"), + // Dotted capital I and i plus a combining dot above + // read alike (plain ix collides with both, below). + ("\u{130}x", "i\u{307}x"), ]; for (n, (a, b)) in pairs.into_iter().enumerate() { let (first, second) = ( @@ -375,6 +384,15 @@ fn s6_folds_dotless_i_kana_and_arabic_indic_digits() { "{a:?} vs {b:?}" ); } + // Plain ix collides with the dotted spellings too: a rare sequence + // over-merges toward held. + assert_eq!( + refusal(store.add(item("item:fold:80", "ix"), &ents(&d))), + StoreRefusal::S6TextHeld { + key: "item:fold:80".into(), + held_by: "item:fold:27".into() + } + ); // Nothing wider: the prolonged sound mark, Devanagari digits and the // Cyrillic homoglyph of i stay apart from what they resemble. store @@ -390,9 +408,31 @@ fn s6_folds_dotless_i_kana_and_arabic_indic_digits() { .add(item("item:fold:93", "room \u{967}\u{968}"), &ents(&d)) .unwrap(); store.add(item("item:fold:94", "bit"), &ents(&d)).unwrap(); + // A combining dot above after any other letter is kept. + store.add(item("item:fold:96", "x"), &ents(&d)).unwrap(); + store + .add(item("item:fold:97", "\u{1e8b}"), &ents(&d)) + .unwrap(); store .add(item("item:fold:95", "b\u{456}t"), &ents(&d)) .unwrap(); + // j has no precomposed dot-above form, so this reaches the guard itself. + store.add(item("item:fold:98", "jx"), &ents(&d)).unwrap(); + store + .add(item("item:fold:99", "j\u{307}x"), &ents(&d)) + .unwrap(); + // Marks after the dotted i compose with it: i, dot, acute is an i with an + // acute, whatever order the writer used. + store + .add(item("item:fold:100", "\u{ed}x"), &ents(&d)) + .unwrap(); + assert_eq!( + refusal(store.add(item("item:fold:101", "i\u{307}\u{301}x"), &ents(&d))), + StoreRefusal::S6TextHeld { + key: "item:fold:101".into(), + held_by: "item:fold:100".into() + } + ); } /// `modifier_letters.txt` is the Unicode 17 Lm list, the oracle for the test @@ -520,11 +560,23 @@ fn s6_keeps_tone_marks_and_viramas_apart() { .unwrap_or_else(|e| panic!("{b:?} held by {a:?}: {e}")); } // A mark alone is still no text. - for (n, text) in ["\u{e49}", "\u{94d}"].into_iter().enumerate() { + for (n, text) in [ + "\u{e49}", + "\u{94d}", + "\u{301}", + "\u{200d}", + "\u{200c}", + "\u{fe0f}", + "\u{200d}\u{fe0f}\u{301}", + ] + .into_iter() + .enumerate() + { let key = format!("item:mk:{}", 90 + n); assert_eq!( refusal(store.add(item(&key, text), &ents(&d))), - malformed(&key, "text has no alphanumeric character") + malformed(&key, "text has no base letter or digit"), + "{text:?}" ); } }