From 8c94838c303d28969321f0726ee919d87313567d Mon Sep 17 00:00:00 2001 From: Weksa Date: Mon, 9 Nov 2026 11:10:23 +0000 Subject: [PATCH 1/2] =?UTF-8?q?S6=20key:=20voiced=20katakana=20=E3=83=B7-?= =?UTF-8?q?=E3=83=BA=20fold,=20dot=20above=20after=20i=20drops=20(=C4=B0x?= =?UTF-8?q?=20=3D=20i=CC=87x);=20marks-only=20text=20gets=20the=20base-let?= =?UTF-8?q?ter=20reason=20first?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/store.rs | 66 +++++++++++++++++++++++++----------- crates/weksa/tests/stores.rs | 39 +++++++++++++++++++-- 2 files changed, 84 insertions(+), 21 deletions(-) diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index f469589..752900e 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -281,12 +281,23 @@ impl CorpusStore { return Err(StoreRefusal::NotNfc { key: item.id }.into()); } let text = normalized(&item.text); - if !text.chars().any(char::is_alphanumeric) { - return Err(malformed(&item.id, "text has no alphanumeric character")); - } // Read from the text as given: NFKC maps modifier letters (ʰ) to base letters. + // A text of marks, joiners and modifiers says so; any other text with + // no alphanumeric character (punctuation, space) gets the plainer reason. if !item.text.chars().any(is_base) { - return Err(malformed(&item.id, "text has no base letter or digit")); + let marks_only = item.text.chars().any(|c| { + is_combining_mark(c) + || matches!(c, '\u{200c}' | '\u{200d}') + || is_modifier_letter(c) + }); + return Err(malformed( + &item.id, + if marks_only { + "text has no base letter or digit" + } else { + "text has no alphanumeric character" + }, + )); } if !is_calendar_date(&item.admitted_on) { return Err(malformed(&item.id, "admitted_on is not a YYYY-MM-DD date")); @@ -415,31 +426,48 @@ fn check_entities(item: &CorpusItem, entities: &EntityStore) -> Result<(), Store /// Turkish dotless i and dotted capital I read as i, katakana reads as the /// hiragana of the same syllable, and Arabic-Indic and Extended Arabic-Indic /// digits read as 0-9. Homoglyphs across scripts (Cyrillic a for Latin a) stay -/// apart: no key here reads what a character looks like. +/// apart: no key here reads what a character looks like. A combining dot above +/// (U+0307) after i is dropped, so İx and i̇x collide as the old case fold made +/// them; so does plain ix, with both. Over-merging a rare sequence errs toward +/// held, a lost collision toward leak. U+0307 after any other letter stays. fn normalized(text: &str) -> String { - let mapped: String = text.nfkc().map(fold_variant).collect(); - default_case_fold_str(&mapped) - .nfkc() - .filter(|c| c.is_alphanumeric() || is_combining_mark(*c)) - .collect() + let mapped: String = text.nfkc().flat_map(fold_variant).collect(); + let mut key = String::new(); + for c in default_case_fold_str(&mapped).nfkc() { + if c == '\u{307}' && key.ends_with('i') { + continue; + } + if c.is_alphanumeric() || is_combining_mark(c) { + key.push(c); + } + } + key } /// The one-to-one folds `normalized` adds to NFKC and case folding, applied /// before the case fold (U+0130 would otherwise fold to i plus a combining dot, /// which the key keeps as a mark). -fn fold_variant(c: char) -> char { - match c { - '\u{131}' => 'i', - '\u{130}' => 'I', +fn fold_variant(c: char) -> impl Iterator { + let one = |c: char| (c, None); + let (first, second) = match c { + '\u{131}' => one('i'), + '\u{130}' => one('I'), + // Voiced katakana wa, wi, we, wo: the hiragana (U+308F-3092) plus the + // combining voiced mark, which is how NFKC spells the hiragana forms. + '\u{30f7}'..='\u{30fa}' => ( + char::from_u32(c as u32 - 0x68).unwrap_or(c), + Some('\u{3099}'), + ), // Katakana letters and small letters to hiragana; the iteration marks // ヽヾ to ゝゞ. The prolonged sound mark U+30FC stays. '\u{30a1}'..='\u{30f6}' | '\u{30fd}'..='\u{30fe}' => { - char::from_u32(c as u32 - 0x60).unwrap_or(c) + one(char::from_u32(c as u32 - 0x60).unwrap_or(c)) } - '\u{660}'..='\u{669}' => char::from(b'0' + (c as u32 - 0x660) as u8), - '\u{6f0}'..='\u{6f9}' => char::from(b'0' + (c as u32 - 0x6f0) as u8), - _ => c, - } + '\u{660}'..='\u{669}' => one(char::from(b'0' + (c as u32 - 0x660) as u8)), + '\u{6f0}'..='\u{6f9}' => one(char::from(b'0' + (c as u32 - 0x6f0) as u8)), + _ => one(c), + }; + std::iter::once(first).chain(second) } /// A character that can carry a word: alphanumeric and neither a combining diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs index a8486e7..726f26e 100644 --- a/crates/weksa/tests/stores.rs +++ b/crates/weksa/tests/stores.rs @@ -359,6 +359,15 @@ fn s6_folds_dotless_i_kana_and_arabic_indic_digits() { ), ("gate 45", "gate \u{6f4}\u{6f5}"), ("\u{3096}", "\u{30f6}"), + // Voiced wa, wi, we, wo: katakana U+30F7-30FA read as the hiragana + // plus the combining voiced mark. + ("\u{308f}\u{3099}", "\u{30f7}"), + ("\u{3090}\u{3099}", "\u{30f8}"), + ("\u{3091}\u{3099}", "\u{30f9}"), + ("\u{3092}\u{3099}", "\u{30fa}"), + // Dotted capital I and i plus a combining dot above + // read alike (plain ix collides with both, below). + ("\u{130}x", "i\u{307}x"), ]; for (n, (a, b)) in pairs.into_iter().enumerate() { let (first, second) = ( @@ -375,6 +384,15 @@ fn s6_folds_dotless_i_kana_and_arabic_indic_digits() { "{a:?} vs {b:?}" ); } + // Plain ix collides with the dotted spellings too: a rare sequence + // over-merges toward held. + assert_eq!( + refusal(store.add(item("item:fold:80", "ix"), &ents(&d))), + StoreRefusal::S6TextHeld { + key: "item:fold:80".into(), + held_by: "item:fold:27".into() + } + ); // Nothing wider: the prolonged sound mark, Devanagari digits and the // Cyrillic homoglyph of i stay apart from what they resemble. store @@ -390,6 +408,11 @@ fn s6_folds_dotless_i_kana_and_arabic_indic_digits() { .add(item("item:fold:93", "room \u{967}\u{968}"), &ents(&d)) .unwrap(); store.add(item("item:fold:94", "bit"), &ents(&d)).unwrap(); + // A combining dot above after any other letter is kept. + store.add(item("item:fold:96", "x"), &ents(&d)).unwrap(); + store + .add(item("item:fold:97", "\u{1e8b}"), &ents(&d)) + .unwrap(); store .add(item("item:fold:95", "b\u{456}t"), &ents(&d)) .unwrap(); @@ -520,11 +543,23 @@ fn s6_keeps_tone_marks_and_viramas_apart() { .unwrap_or_else(|e| panic!("{b:?} held by {a:?}: {e}")); } // A mark alone is still no text. - for (n, text) in ["\u{e49}", "\u{94d}"].into_iter().enumerate() { + for (n, text) in [ + "\u{e49}", + "\u{94d}", + "\u{301}", + "\u{200d}", + "\u{200c}", + "\u{fe0f}", + "\u{200d}\u{fe0f}\u{301}", + ] + .into_iter() + .enumerate() + { let key = format!("item:mk:{}", 90 + n); assert_eq!( refusal(store.add(item(&key, text), &ents(&d))), - malformed(&key, "text has no alphanumeric character") + malformed(&key, "text has no base letter or digit"), + "{text:?}" ); } } -- 2.47.3 From a59d0a5e3c457dd673e4b639304b54145abe8acb Mon Sep 17 00:00:00 2001 From: Weksa Date: Mon, 9 Nov 2026 11:15:46 +0000 Subject: [PATCH 2/2] S6 key: drop the dot above i before the last NFKC so following marks compose; test the guard with j (Soul s22) Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/store.rs | 17 ++++++++++------- crates/weksa/tests/stores.rs | 17 +++++++++++++++++ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index 752900e..6739db6 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -432,16 +432,19 @@ fn check_entities(item: &CorpusItem, entities: &EntityStore) -> Result<(), Store /// held, a lost collision toward leak. U+0307 after any other letter stays. fn normalized(text: &str) -> String { let mapped: String = text.nfkc().flat_map(fold_variant).collect(); - let mut key = String::new(); - for c in default_case_fold_str(&mapped).nfkc() { - if c == '\u{307}' && key.ends_with('i') { + // The dot goes before the last NFKC so the marks left on the i compose with + // it (i, dot, acute reads as the i with an acute). + let mut dotless = String::new(); + for c in default_case_fold_str(&mapped).chars() { + if c == '\u{307}' && dotless.ends_with('i') { continue; } - if c.is_alphanumeric() || is_combining_mark(c) { - key.push(c); - } + dotless.push(c); } - key + dotless + .nfkc() + .filter(|&c| c.is_alphanumeric() || is_combining_mark(c)) + .collect() } /// The one-to-one folds `normalized` adds to NFKC and case folding, applied diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs index 726f26e..22ecb83 100644 --- a/crates/weksa/tests/stores.rs +++ b/crates/weksa/tests/stores.rs @@ -416,6 +416,23 @@ fn s6_folds_dotless_i_kana_and_arabic_indic_digits() { store .add(item("item:fold:95", "b\u{456}t"), &ents(&d)) .unwrap(); + // j has no precomposed dot-above form, so this reaches the guard itself. + store.add(item("item:fold:98", "jx"), &ents(&d)).unwrap(); + store + .add(item("item:fold:99", "j\u{307}x"), &ents(&d)) + .unwrap(); + // Marks after the dotted i compose with it: i, dot, acute is an i with an + // acute, whatever order the writer used. + store + .add(item("item:fold:100", "\u{ed}x"), &ents(&d)) + .unwrap(); + assert_eq!( + refusal(store.add(item("item:fold:101", "i\u{307}\u{301}x"), &ents(&d))), + StoreRefusal::S6TextHeld { + key: "item:fold:101".into(), + held_by: "item:fold:100".into() + } + ); } /// `modifier_letters.txt` is the Unicode 17 Lm list, the oracle for the test -- 2.47.3