crates/weksa/src/store.rs

@@ -281,12 +281,23 @@ impl CorpusStore {
             return Err(StoreRefusal::NotNfc { key: item.id }.into());
         }
         let text = normalized(&item.text);
-        if !text.chars().any(char::is_alphanumeric) {
-            return Err(malformed(&item.id, "text has no alphanumeric character"));
-        }
         // Read from the text as given: NFKC maps modifier letters (ʰ) to base letters.
+        // A text of marks, joiners and modifiers says so; any other text with
+        // no alphanumeric character (punctuation, space) gets the plainer reason.
         if !item.text.chars().any(is_base) {
-            return Err(malformed(&item.id, "text has no base letter or digit"));
+            let marks_only = item.text.chars().any(|c| {
+                is_combining_mark(c)
+                    || matches!(c, '\u{200c}' | '\u{200d}')
+                    || is_modifier_letter(c)
+            });
+            return Err(malformed(
+                &item.id,
+                if marks_only {
+                    "text has no base letter or digit"
+                } else {
+                    "text has no alphanumeric character"
+                },
+            ));
         }
         if !is_calendar_date(&item.admitted_on) {
             return Err(malformed(&item.id, "admitted_on is not a YYYY-MM-DD date"));
@@ -415,31 +426,51 @@ fn check_entities(item: &CorpusItem, entities: &EntityStore) -> Result<(), Store
 /// Turkish dotless i and dotted capital I read as i, katakana reads as the
 /// hiragana of the same syllable, and Arabic-Indic and Extended Arabic-Indic
 /// digits read as 0-9. Homoglyphs across scripts (Cyrillic a for Latin a) stay
-/// apart: no key here reads what a character looks like.
+/// apart: no key here reads what a character looks like. A combining dot above
+/// (U+0307) after i is dropped, so İx and i̇x collide as the old case fold made
+/// them; so does plain ix, with both. Over-merging a rare sequence errs toward
+/// held, a lost collision toward leak. U+0307 after any other letter stays.
 fn normalized(text: &str) -> String {
-    let mapped: String = text.nfkc().map(fold_variant).collect();
-    default_case_fold_str(&mapped)
+    let mapped: String = text.nfkc().flat_map(fold_variant).collect();
+    // The dot goes before the last NFKC so the marks left on the i compose with
+    // it (i, dot, acute reads as the i with an acute).
+    let mut dotless = String::new();
+    for c in default_case_fold_str(&mapped).chars() {
+        if c == '\u{307}' && dotless.ends_with('i') {
+            continue;
+        }
+        dotless.push(c);
+    }
+    dotless
         .nfkc()
-        .filter(|c| c.is_alphanumeric() || is_combining_mark(*c))
+        .filter(|&c| c.is_alphanumeric() || is_combining_mark(c))
         .collect()
 }
 
 /// The one-to-one folds `normalized` adds to NFKC and case folding, applied
 /// before the case fold (U+0130 would otherwise fold to i plus a combining dot,
 /// which the key keeps as a mark).
-fn fold_variant(c: char) -> char {
-    match c {
-        '\u{131}' => 'i',
-        '\u{130}' => 'I',
+fn fold_variant(c: char) -> impl Iterator<Item = char> {
+    let one = |c: char| (c, None);
+    let (first, second) = match c {
+        '\u{131}' => one('i'),
+        '\u{130}' => one('I'),
+        // Voiced katakana wa, wi, we, wo: the hiragana (U+308F-3092) plus the
+        // combining voiced mark, which is how NFKC spells the hiragana forms.
+        '\u{30f7}'..='\u{30fa}' => (
+            char::from_u32(c as u32 - 0x68).unwrap_or(c),
+            Some('\u{3099}'),
+        ),
         // Katakana letters and small letters to hiragana; the iteration marks
         // ヽヾ to ゝゞ. The prolonged sound mark U+30FC stays.
         '\u{30a1}'..='\u{30f6}' | '\u{30fd}'..='\u{30fe}' => {
-            char::from_u32(c as u32 - 0x60).unwrap_or(c)
+            one(char::from_u32(c as u32 - 0x60).unwrap_or(c))
         }
-        '\u{660}'..='\u{669}' => char::from(b'0' + (c as u32 - 0x660) as u8),
-        '\u{6f0}'..='\u{6f9}' => char::from(b'0' + (c as u32 - 0x6f0) as u8),
-        _ => c,
-    }
+        '\u{660}'..='\u{669}' => one(char::from(b'0' + (c as u32 - 0x660) as u8)),
+        '\u{6f0}'..='\u{6f9}' => one(char::from(b'0' + (c as u32 - 0x6f0) as u8)),
+        _ => one(c),
+    };
+    std::iter::once(first).chain(second)
 }
 
 /// A character that can carry a word: alphanumeric and neither a combining

crates/weksa/tests/stores.rs

@@ -359,6 +359,15 @@ fn s6_folds_dotless_i_kana_and_arabic_indic_digits() {
         ),
         ("gate 45", "gate \u{6f4}\u{6f5}"),
         ("\u{3096}", "\u{30f6}"),
+        // Voiced wa, wi, we, wo: katakana U+30F7-30FA read as the hiragana
+        // plus the combining voiced mark.
+        ("\u{308f}\u{3099}", "\u{30f7}"),
+        ("\u{3090}\u{3099}", "\u{30f8}"),
+        ("\u{3091}\u{3099}", "\u{30f9}"),
+        ("\u{3092}\u{3099}", "\u{30fa}"),
+        // Dotted capital I and i plus a combining dot above
+        // read alike (plain ix collides with both, below).
+        ("\u{130}x", "i\u{307}x"),
     ];
     for (n, (a, b)) in pairs.into_iter().enumerate() {
         let (first, second) = (
@@ -375,6 +384,15 @@ fn s6_folds_dotless_i_kana_and_arabic_indic_digits() {
             "{a:?} vs {b:?}"
         );
     }
+    // Plain ix collides with the dotted spellings too: a rare sequence
+    // over-merges toward held.
+    assert_eq!(
+        refusal(store.add(item("item:fold:80", "ix"), &ents(&d))),
+        StoreRefusal::S6TextHeld {
+            key: "item:fold:80".into(),
+            held_by: "item:fold:27".into()
+        }
+    );
     // Nothing wider: the prolonged sound mark, Devanagari digits and the
     // Cyrillic homoglyph of i stay apart from what they resemble.
     store
@@ -390,9 +408,31 @@ fn s6_folds_dotless_i_kana_and_arabic_indic_digits() {
         .add(item("item:fold:93", "room \u{967}\u{968}"), &ents(&d))
         .unwrap();
     store.add(item("item:fold:94", "bit"), &ents(&d)).unwrap();
+    // A combining dot above after any other letter is kept.
+    store.add(item("item:fold:96", "x"), &ents(&d)).unwrap();
+    store
+        .add(item("item:fold:97", "\u{1e8b}"), &ents(&d))
+        .unwrap();
     store
         .add(item("item:fold:95", "b\u{456}t"), &ents(&d))
         .unwrap();
+    // j has no precomposed dot-above form, so this reaches the guard itself.
+    store.add(item("item:fold:98", "jx"), &ents(&d)).unwrap();
+    store
+        .add(item("item:fold:99", "j\u{307}x"), &ents(&d))
+        .unwrap();
+    // Marks after the dotted i compose with it: i, dot, acute is an i with an
+    // acute, whatever order the writer used.
+    store
+        .add(item("item:fold:100", "\u{ed}x"), &ents(&d))
+        .unwrap();
+    assert_eq!(
+        refusal(store.add(item("item:fold:101", "i\u{307}\u{301}x"), &ents(&d))),
+        StoreRefusal::S6TextHeld {
+            key: "item:fold:101".into(),
+            held_by: "item:fold:100".into()
+        }
+    );
 }
 
 /// `modifier_letters.txt` is the Unicode 17 Lm list, the oracle for the test
@@ -520,11 +560,23 @@ fn s6_keeps_tone_marks_and_viramas_apart() {
             .unwrap_or_else(|e| panic!("{b:?} held by {a:?}: {e}"));
     }
     // A mark alone is still no text.
-    for (n, text) in ["\u{e49}", "\u{94d}"].into_iter().enumerate() {
+    for (n, text) in [
+        "\u{e49}",
+        "\u{94d}",
+        "\u{301}",
+        "\u{200d}",
+        "\u{200c}",
+        "\u{fe0f}",
+        "\u{200d}\u{fe0f}\u{301}",
+    ]
+    .into_iter()
+    .enumerate()
+    {
         let key = format!("item:mk:{}", 90 + n);
         assert_eq!(
             refusal(store.add(item(&key, text), &ents(&d))),
-            malformed(&key, "text has no alphanumeric character")
+            malformed(&key, "text has no base letter or digit"),
+            "{text:?}"
         );
     }
 }