ja-inventory: delete the keyword lexicon tripwire and the byte-identity test

A substring table judging whether Japanese names a concept is a rules-based reader of meaning; byte identity of data/original and data/daodejing stops being true by design (rev 5 follows).

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Weksa 2026-11-12 17:44:28 +00:00
commit afea3e65e9

View file

@ -240,25 +240,6 @@ fn the_first_six_original_items_are_unchanged() {
}
}
#[test]
fn the_original_inventory_is_the_dao_inventory() {
let dao = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/daodejing/inventory.cc");
let own = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/original/inventory.cc");
assert_eq!(std::fs::read(dao).unwrap(), std::fs::read(&own).unwrap());
let tmp = tempfile::tempdir().unwrap();
let copy = tmp.path().join("inventory.cc");
std::fs::copy(&own, &copy).unwrap();
let inventory = InventoryStore::open(&copy).unwrap().inventory().unwrap();
// Revision 4 is the speaker-state vocabulary: the same six features the
// generator admits, ids and all.
assert_eq!(inventory.latest().unwrap().n, 4);
let (rev, entries) = weksa::inventory::speaker_scene_revision(4);
assert_eq!(inventory.latest().unwrap().changes, rev.changes);
for e in &entries {
assert_eq!(inventory.entry(&e.id), Some(e), "{}", e.gloss);
}
}
/// The committed seed ontology: every entry model-drafted, exactly the two
/// panel languages, each with a Stance-conditioned entry, and every committed
/// corpus opens a run over it (ruling `ja-ontology-owner-answer`: a model-drafted
@ -889,135 +870,6 @@ fn the_eight_ja_test_sentences_are_pinned_verbatim() {
}
}
/// The concepts the pinned inventory holds, as each language names them in the
/// passages: English by word stem, Japanese by a multi-character surface. This
/// is a tripwire, not the encoder: it fails a passage that names no live
/// concept, and `FALSE_FRIENDS` strips the idioms that hold a concept's
/// character without its meaning (天気, 心配, 腹が立つ ...) before matching.
const LEXICON: [(&str, &[&str], &[&str]); 21] = [
("endanger", &["endanger"], &["危う", "危ない"]),
("prevent", &["prevent"], &["防い", "防げ", "妨げ"]),
("control", &["control"], &["操ろ", "操る", "操れ", "支配"]),
(
"request",
&["request"],
&["いただけませんか", "もらえますか", "くれませんか", "お願い"],
),
("sage", &["sage"], &["賢者"]),
("mind", &["mind"], &["心が", "心は", "心を", "心の", "心も"]),
(
"belly",
&["belly", "bellies"],
&["お腹", "空腹", "満腹", "腹の虫"],
),
("will", &["will"], &["意志"]),
("bone", &["bone"], &["骨が", "骨を", "骨は", "骨の"]),
("empty", &["empt"], &["空っぽ", "空にし", "空の器"]),
("fill", &["fill"], &["満た", "いっぱい"]),
("weaken", &["weaken"], &["弱ら", "弱め", "弱っ"]),
("strengthen", &["strengthen"], &["強くな", "強め", "強くし"]),
(
"heaven",
&["heaven"],
&["天が", "天を", "天は", "天上", "天の"],
),
("earth", &["earth"], &["大地"]),
("water", &["water"], &["水が", "水を", "水は", "水面"]),
("compare", &["compare"], &["比べ"]),
("resemble", &["resemble"], &["似て", "似た", "似る"]),
("continues_long", &[], &["長く続", "久しく続"]),
("true_words", &[], &["誠の言葉", "真実の言葉"]),
("seem_paradoxical", &[], &["逆さま"]),
];
/// Compounds and idioms that contain a surface above without naming its concept.
const FALSE_FRIENDS: [&str; 8] = [
"天気",
"心配",
"腹が立",
"腹を立",
"水曜",
"骨が折れ",
"天の川",
"天井",
];
/// The live concepts a passage names, by `LEXICON`, in `lang_col` (0 en-US, 1 ja-JP).
fn concepts_in(text: &str, col: usize) -> Vec<&'static str> {
let mut text = text.to_lowercase();
for f in FALSE_FRIENDS {
text = text.replace(f, "");
}
let words: Vec<&str> = text.split(|c: char| !c.is_alphabetic()).collect();
LEXICON
.iter()
.filter(|(_, en, ja)| {
if col == 0 {
en.iter().any(|s| words.iter().any(|w| w.starts_with(s)))
} else {
ja.iter().any(|s| text.contains(s))
}
})
.map(|(c, _, _)| *c)
.collect()
}
#[test]
fn the_concept_lexicon_does_not_match_a_word_that_merely_holds_a_character() {
for ja in [
"今日の天気はどうだろう。",
"試験が心配でならない。",
"腹が立って仕方がない。",
"水曜に会いましょう。",
"骨が折れる仕事だ。",
] {
assert!(concepts_in(ja, 1).is_empty(), "{ja}");
}
assert!(concepts_in("I attempt it, with a reminder.", 0).is_empty());
assert_eq!(concepts_in("水が澄んでいる。", 1), ["water"]);
}
#[test]
fn every_original_passage_rests_on_a_live_inventory_concept() {
use weksa::inventory::Family;
let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/original");
let tmp = tempfile::tempdir().unwrap();
std::fs::copy(dir.join("inventory.cc"), tmp.path().join("inventory.cc")).unwrap();
let inventory = InventoryStore::open(&tmp.path().join("inventory.cc"))
.unwrap()
.inventory()
.unwrap();
let rev = inventory.latest().unwrap().n;
let live: Vec<&str> = inventory
.entries()
.filter(|e| e.family() == Family::Concept && e.live_at(rev))
.map(|e| e.gloss.as_str())
.collect();
for (concept, _, _) in LEXICON {
assert!(live.contains(&concept), "{concept} is not a live concept");
}
let (corpus, _) = stores();
for (col, lang) in PANEL.iter().enumerate() {
let mut hit = std::collections::BTreeSet::new();
let mut passages = 0;
for item in corpus.all().unwrap() {
if item.language != *lang || !matches!(item.source, Source::ModelWritten { .. }) {
continue;
}
passages += 1;
let found = concepts_in(&item.text, col);
assert!(!found.is_empty(), "{} rests on no live concept", item.id);
hit.extend(found);
}
eprintln!(
"{lang}: {passages} passages on a live concept, {} distinct concepts",
hit.len()
);
assert_eq!(passages, 28);
assert!(hit.len() >= 12, "{lang}: {} distinct concepts", hit.len());
}
}
/// The committed scenes: one per live Train or Validation group of
/// `data/original`, none for any other group, every one model-annotated and
/// never Test evidence (a Test group gets no scene and reads `NoScene`).