From afea3e65e933dcd5cf80428c0d5c9cef00937c4c Mon Sep 17 00:00:00 2001 From: Weksa Date: Thu, 12 Nov 2026 17:44:28 +0000 Subject: [PATCH] ja-inventory: delete the keyword lexicon tripwire and the byte-identity test A substring table judging whether Japanese names a concept is a rules-based reader of meaning; byte identity of data/original and data/daodejing stops being true by design (rev 5 follows). Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/original_corpus.rs | 148 -------------------------- 1 file changed, 148 deletions(-) diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 6a651e7..33940a3 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -240,25 +240,6 @@ fn the_first_six_original_items_are_unchanged() { } } -#[test] -fn the_original_inventory_is_the_dao_inventory() { - let dao = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/daodejing/inventory.cc"); - let own = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/original/inventory.cc"); - assert_eq!(std::fs::read(dao).unwrap(), std::fs::read(&own).unwrap()); - let tmp = tempfile::tempdir().unwrap(); - let copy = tmp.path().join("inventory.cc"); - std::fs::copy(&own, ©).unwrap(); - let inventory = InventoryStore::open(©).unwrap().inventory().unwrap(); - // Revision 4 is the speaker-state vocabulary: the same six features the - // generator admits, ids and all. - assert_eq!(inventory.latest().unwrap().n, 4); - let (rev, entries) = weksa::inventory::speaker_scene_revision(4); - assert_eq!(inventory.latest().unwrap().changes, rev.changes); - for e in &entries { - assert_eq!(inventory.entry(&e.id), Some(e), "{}", e.gloss); - } -} - /// The committed seed ontology: every entry model-drafted, exactly the two /// panel languages, each with a Stance-conditioned entry, and every committed /// corpus opens a run over it (ruling `ja-ontology-owner-answer`: a model-drafted @@ -889,135 +870,6 @@ fn the_eight_ja_test_sentences_are_pinned_verbatim() { } } -/// The concepts the pinned inventory holds, as each language names them in the -/// passages: English by word stem, Japanese by a multi-character surface. This -/// is a tripwire, not the encoder: it fails a passage that names no live -/// concept, and `FALSE_FRIENDS` strips the idioms that hold a concept's -/// character without its meaning (天気, 心配, 腹が立つ ...) before matching. -const LEXICON: [(&str, &[&str], &[&str]); 21] = [ - ("endanger", &["endanger"], &["危う", "危ない"]), - ("prevent", &["prevent"], &["防い", "防げ", "妨げ"]), - ("control", &["control"], &["操ろ", "操る", "操れ", "支配"]), - ( - "request", - &["request"], - &["いただけませんか", "もらえますか", "くれませんか", "お願い"], - ), - ("sage", &["sage"], &["賢者"]), - ("mind", &["mind"], &["心が", "心は", "心を", "心の", "心も"]), - ( - "belly", - &["belly", "bellies"], - &["お腹", "空腹", "満腹", "腹の虫"], - ), - ("will", &["will"], &["意志"]), - ("bone", &["bone"], &["骨が", "骨を", "骨は", "骨の"]), - ("empty", &["empt"], &["空っぽ", "空にし", "空の器"]), - ("fill", &["fill"], &["満た", "いっぱい"]), - ("weaken", &["weaken"], &["弱ら", "弱め", "弱っ"]), - ("strengthen", &["strengthen"], &["強くな", "強め", "強くし"]), - ( - "heaven", - &["heaven"], - &["天が", "天を", "天は", "天上", "天の"], - ), - ("earth", &["earth"], &["大地"]), - ("water", &["water"], &["水が", "水を", "水は", "水面"]), - ("compare", &["compare"], &["比べ"]), - ("resemble", &["resemble"], &["似て", "似た", "似る"]), - ("continues_long", &[], &["長く続", "久しく続"]), - ("true_words", &[], &["誠の言葉", "真実の言葉"]), - ("seem_paradoxical", &[], &["逆さま"]), -]; - -/// Compounds and idioms that contain a surface above without naming its concept. -const FALSE_FRIENDS: [&str; 8] = [ - "天気", - "心配", - "腹が立", - "腹を立", - "水曜", - "骨が折れ", - "天の川", - "天井", -]; - -/// The live concepts a passage names, by `LEXICON`, in `lang_col` (0 en-US, 1 ja-JP). -fn concepts_in(text: &str, col: usize) -> Vec<&'static str> { - let mut text = text.to_lowercase(); - for f in FALSE_FRIENDS { - text = text.replace(f, ""); - } - let words: Vec<&str> = text.split(|c: char| !c.is_alphabetic()).collect(); - LEXICON - .iter() - .filter(|(_, en, ja)| { - if col == 0 { - en.iter().any(|s| words.iter().any(|w| w.starts_with(s))) - } else { - ja.iter().any(|s| text.contains(s)) - } - }) - .map(|(c, _, _)| *c) - .collect() -} - -#[test] -fn the_concept_lexicon_does_not_match_a_word_that_merely_holds_a_character() { - for ja in [ - "今日の天気はどうだろう。", - "試験が心配でならない。", - "腹が立って仕方がない。", - "水曜に会いましょう。", - "骨が折れる仕事だ。", - ] { - assert!(concepts_in(ja, 1).is_empty(), "{ja}"); - } - assert!(concepts_in("I attempt it, with a reminder.", 0).is_empty()); - assert_eq!(concepts_in("水が澄んでいる。", 1), ["water"]); -} - -#[test] -fn every_original_passage_rests_on_a_live_inventory_concept() { - use weksa::inventory::Family; - let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/original"); - let tmp = tempfile::tempdir().unwrap(); - std::fs::copy(dir.join("inventory.cc"), tmp.path().join("inventory.cc")).unwrap(); - let inventory = InventoryStore::open(&tmp.path().join("inventory.cc")) - .unwrap() - .inventory() - .unwrap(); - let rev = inventory.latest().unwrap().n; - let live: Vec<&str> = inventory - .entries() - .filter(|e| e.family() == Family::Concept && e.live_at(rev)) - .map(|e| e.gloss.as_str()) - .collect(); - for (concept, _, _) in LEXICON { - assert!(live.contains(&concept), "{concept} is not a live concept"); - } - let (corpus, _) = stores(); - for (col, lang) in PANEL.iter().enumerate() { - let mut hit = std::collections::BTreeSet::new(); - let mut passages = 0; - for item in corpus.all().unwrap() { - if item.language != *lang || !matches!(item.source, Source::ModelWritten { .. }) { - continue; - } - passages += 1; - let found = concepts_in(&item.text, col); - assert!(!found.is_empty(), "{} rests on no live concept", item.id); - hit.extend(found); - } - eprintln!( - "{lang}: {passages} passages on a live concept, {} distinct concepts", - hit.len() - ); - assert_eq!(passages, 28); - assert!(hit.len() >= 12, "{lang}: {} distinct concepts", hit.len()); - } -} - /// The committed scenes: one per live Train or Validation group of /// `data/original`, none for any other group, every one model-annotated and /// never Test evidence (a Test group gets no scene and reads `NoScene`).