data/original: ja passages replaced, none a rendering of an en-US original or a Dao line

All 28 Train/Validation ja passages are new Japanese sentences on the inventory's
concepts with their own imagery (paddies, charcoal, kites, the bath, the market); none
renders an en-US original, and none restates a data/daodejing line (ch3, 5, 7, 8, 78) or
other canon (the old ones on 'true words', 'long-enduring', 'bellows', 'water vs hard
things' are gone). corpus.cc restored from 21eaa07 and rebuilt through weksa corpus add:
numbering 1..28 unbroken, holdout-ja:1..8 and original:1..28 byte-identical.

The concept test now matches Japanese by multi-character surface and English by word
stem, strips false friends (天気, 心配, 腹が立つ, 水曜 ...), and has a negative test.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Weksa 2026-11-12 13:05:16 +00:00
commit 20863615b3
2 changed files with 77 additions and 26 deletions

View file

@ -750,37 +750,93 @@ fn the_eight_ja_test_sentences_are_pinned_verbatim() {
}
/// The concepts the pinned inventory holds, as each language names them in the
/// passages (a surface that appears in a passage). Every passage of both
/// languages rests on at least one live concept, so the two panel legs are
/// measured on the same inventory.
/// passages: English by word stem, Japanese by a multi-character surface. This
/// is a tripwire, not the encoder: it fails a passage that names no live
/// concept, and `FALSE_FRIENDS` strips the idioms that hold a concept's
/// character without its meaning (天気, 心配, 腹が立つ ...) before matching.
const LEXICON: [(&str, &[&str], &[&str]); 21] = [
("endanger", &["endanger"], &["危う", "危な"]),
("prevent", &["prevent"], &["防げ", "妨げ"]),
("control", &["control"], &["操", "支配"]),
("endanger", &["endanger"], &["危う", "危ない"]),
("prevent", &["prevent"], &["防い", "防げ", "妨げ"]),
("control", &["control"], &["操ろ", "操る", "操れ", "支配"]),
(
"request",
&["please ask", "request"],
&["くれませんか", "いただけませんか", "お願い"],
&["request"],
&["いただけませんか", "もらえますか", "くれませんか", "お願い"],
),
("sage", &["sage"], &["賢者"]),
("mind", &["mind"], &["心"]),
("belly", &["belly"], &["腹"]),
("mind", &["mind"], &["心が", "心は", "心を", "心の", "心も"]),
(
"belly",
&["belly", "bellies"],
&["お腹", "空腹", "満腹", "腹の虫"],
),
("will", &["will"], &["意志"]),
("bone", &["bone"], &["骨"]),
("empty", &["empt"], &["空にし", "空の器", "空なのに"]),
("bone", &["bone"], &["骨が", "骨を", "骨は", "骨の"]),
("empty", &["empt"], &["空っぽ", "空にし", "空の器"]),
("fill", &["fill"], &["満た", "いっぱい"]),
("weaken", &["weaken"], &["弱ら", "弱っ"]),
("strengthen", &["strengthen"], &["強く", "強い"]),
("heaven", &["heaven"], &["天"]),
("earth", &["earth"], &["大地", "地は", "天と地"]),
("water", &["water"], &["水"]),
("weaken", &["weaken"], &["弱ら", "弱め", "弱っ"]),
("strengthen", &["strengthen"], &["強くな", "強め", "強くし"]),
(
"heaven",
&["heaven"],
&["天が", "天を", "天は", "天上", "天の"],
),
("earth", &["earth"], &["大地"]),
("water", &["water"], &["水が", "水を", "水は", "水面"]),
("compare", &["compare"], &["比べ"]),
("resemble", &["resemble"], &["似"]),
("resemble", &["resemble"], &["似て", "似た", "似る"]),
("continues_long", &[], &["長く続", "久しく続"]),
("true_words", &[], &["誠の言葉", "真実の言葉"]),
("seem_paradoxical", &[], &["逆さま"]),
];
/// Compounds and idioms that contain a surface above without naming its concept.
const FALSE_FRIENDS: [&str; 8] = [
"天気",
"心配",
"腹が立",
"腹を立",
"水曜",
"骨が折れ",
"天の川",
"天井",
];
/// The live concepts a passage names, by `LEXICON`, in `lang_col` (0 en-US, 1 ja-JP).
fn concepts_in(text: &str, col: usize) -> Vec<&'static str> {
let mut text = text.to_lowercase();
for f in FALSE_FRIENDS {
text = text.replace(f, "");
}
let words: Vec<&str> = text.split(|c: char| !c.is_alphabetic()).collect();
LEXICON
.iter()
.filter(|(_, en, ja)| {
if col == 0 {
en.iter().any(|s| words.iter().any(|w| w.starts_with(s)))
} else {
ja.iter().any(|s| text.contains(s))
}
})
.map(|(c, _, _)| *c)
.collect()
}
#[test]
fn the_concept_lexicon_does_not_match_a_word_that_merely_holds_a_character() {
for ja in [
"今日の天気はどうだろう。",
"試験が心配でならない。",
"腹が立って仕方がない。",
"水曜に会いましょう。",
"骨が折れる仕事だ。",
] {
assert!(concepts_in(ja, 1).is_empty(), "{ja}");
}
assert!(concepts_in("I attempt it, with a reminder.", 0).is_empty());
assert_eq!(concepts_in("水が澄んでいる。", 1), ["water"]);
}
#[test]
fn every_original_passage_rests_on_a_live_inventory_concept() {
use weksa::inventory::Family;
@ -801,20 +857,15 @@ fn every_original_passage_rests_on_a_live_inventory_concept() {
assert!(live.contains(&concept), "{concept} is not a live concept");
}
let (corpus, _) = stores();
for (lang, col) in [("en-US", 0), ("ja-JP", 1)] {
for (col, lang) in PANEL.iter().enumerate() {
let mut hit = std::collections::BTreeSet::new();
let mut passages = 0;
for item in corpus.all().unwrap() {
if item.language != lang || !matches!(item.source, Source::ModelWritten { .. }) {
if item.language != *lang || !matches!(item.source, Source::ModelWritten { .. }) {
continue;
}
passages += 1;
let text = item.text.to_lowercase();
let found: Vec<&str> = LEXICON
.iter()
.filter(|(_, en, ja)| [en, ja][col].iter().any(|s| text.contains(s)))
.map(|(c, _, _)| *c)
.collect();
let found = concepts_in(&item.text, col);
assert!(!found.is_empty(), "{} rests on no live concept", item.id);
hit.extend(found);
}

Binary file not shown.