From 20863615b35c9fdbb7adf2b7e71eb7e091f220ec Mon Sep 17 00:00:00 2001 From: Weksa Date: Thu, 12 Nov 2026 13:05:16 +0000 Subject: [PATCH] data/original: ja passages replaced, none a rendering of an en-US original or a Dao line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All 28 Train/Validation ja passages are new Japanese sentences on the inventory's concepts with their own imagery (paddies, charcoal, kites, the bath, the market); none renders an en-US original, and none restates a data/daodejing line (ch3, 5, 7, 8, 78) or other canon (the old ones on 'true words', 'long-enduring', 'bellows', 'water vs hard things' are gone). corpus.cc restored from 21eaa07 and rebuilt through weksa corpus add: numbering 1..28 unbroken, holdout-ja:1..8 and original:1..28 byte-identical. The concept test now matches Japanese by multi-character surface and English by word stem, strips false friends (天気, 心配, 腹が立つ, 水曜 ...), and has a negative test. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/original_corpus.rs | 103 +++++++++++++++++++------- data/original/corpus.cc | Bin 28446 -> 28720 bytes 2 files changed, 77 insertions(+), 26 deletions(-) diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 47ed2d6..9213939 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -750,37 +750,93 @@ fn the_eight_ja_test_sentences_are_pinned_verbatim() { } /// The concepts the pinned inventory holds, as each language names them in the -/// passages (a surface that appears in a passage). Every passage of both -/// languages rests on at least one live concept, so the two panel legs are -/// measured on the same inventory. +/// passages: English by word stem, Japanese by a multi-character surface. This +/// is a tripwire, not the encoder: it fails a passage that names no live +/// concept, and `FALSE_FRIENDS` strips the idioms that hold a concept's +/// character without its meaning (天気, 心配, 腹が立つ ...) before matching. const LEXICON: [(&str, &[&str], &[&str]); 21] = [ - ("endanger", &["endanger"], &["危う", "危な"]), - ("prevent", &["prevent"], &["防げ", "妨げ"]), - ("control", &["control"], &["操", "支配"]), + ("endanger", &["endanger"], &["危う", "危ない"]), + ("prevent", &["prevent"], &["防い", "防げ", "妨げ"]), + ("control", &["control"], &["操ろ", "操る", "操れ", "支配"]), ( "request", - &["please ask", "request"], - &["くれませんか", "いただけませんか", "お願い"], + &["request"], + &["いただけませんか", "もらえますか", "くれませんか", "お願い"], ), ("sage", &["sage"], &["賢者"]), - ("mind", &["mind"], &["心"]), - ("belly", &["belly"], &["腹"]), + ("mind", &["mind"], &["心が", "心は", "心を", "心の", "心も"]), + ( + "belly", + &["belly", "bellies"], + &["お腹", "空腹", "満腹", "腹の虫"], + ), ("will", &["will"], &["意志"]), - ("bone", &["bone"], &["骨"]), - ("empty", &["empt"], &["空にし", "空の器", "空なのに"]), + ("bone", &["bone"], &["骨が", "骨を", "骨は", "骨の"]), + ("empty", &["empt"], &["空っぽ", "空にし", "空の器"]), ("fill", &["fill"], &["満た", "いっぱい"]), - ("weaken", &["weaken"], &["弱ら", "弱っ"]), - ("strengthen", &["strengthen"], &["強く", "強い"]), - ("heaven", &["heaven"], &["天"]), - ("earth", &["earth"], &["大地", "地は", "天と地"]), - ("water", &["water"], &["水"]), + ("weaken", &["weaken"], &["弱ら", "弱め", "弱っ"]), + ("strengthen", &["strengthen"], &["強くな", "強め", "強くし"]), + ( + "heaven", + &["heaven"], + &["天が", "天を", "天は", "天上", "天の"], + ), + ("earth", &["earth"], &["大地"]), + ("water", &["water"], &["水が", "水を", "水は", "水面"]), ("compare", &["compare"], &["比べ"]), - ("resemble", &["resemble"], &["似"]), + ("resemble", &["resemble"], &["似て", "似た", "似る"]), ("continues_long", &[], &["長く続", "久しく続"]), ("true_words", &[], &["誠の言葉", "真実の言葉"]), ("seem_paradoxical", &[], &["逆さま"]), ]; +/// Compounds and idioms that contain a surface above without naming its concept. +const FALSE_FRIENDS: [&str; 8] = [ + "天気", + "心配", + "腹が立", + "腹を立", + "水曜", + "骨が折れ", + "天の川", + "天井", +]; + +/// The live concepts a passage names, by `LEXICON`, in `lang_col` (0 en-US, 1 ja-JP). +fn concepts_in(text: &str, col: usize) -> Vec<&'static str> { + let mut text = text.to_lowercase(); + for f in FALSE_FRIENDS { + text = text.replace(f, ""); + } + let words: Vec<&str> = text.split(|c: char| !c.is_alphabetic()).collect(); + LEXICON + .iter() + .filter(|(_, en, ja)| { + if col == 0 { + en.iter().any(|s| words.iter().any(|w| w.starts_with(s))) + } else { + ja.iter().any(|s| text.contains(s)) + } + }) + .map(|(c, _, _)| *c) + .collect() +} + +#[test] +fn the_concept_lexicon_does_not_match_a_word_that_merely_holds_a_character() { + for ja in [ + "今日の天気はどうだろう。", + "試験が心配でならない。", + "腹が立って仕方がない。", + "水曜に会いましょう。", + "骨が折れる仕事だ。", + ] { + assert!(concepts_in(ja, 1).is_empty(), "{ja}"); + } + assert!(concepts_in("I attempt it, with a reminder.", 0).is_empty()); + assert_eq!(concepts_in("水が澄んでいる。", 1), ["water"]); +} + #[test] fn every_original_passage_rests_on_a_live_inventory_concept() { use weksa::inventory::Family; @@ -801,20 +857,15 @@ fn every_original_passage_rests_on_a_live_inventory_concept() { assert!(live.contains(&concept), "{concept} is not a live concept"); } let (corpus, _) = stores(); - for (lang, col) in [("en-US", 0), ("ja-JP", 1)] { + for (col, lang) in PANEL.iter().enumerate() { let mut hit = std::collections::BTreeSet::new(); let mut passages = 0; for item in corpus.all().unwrap() { - if item.language != lang || !matches!(item.source, Source::ModelWritten { .. }) { + if item.language != *lang || !matches!(item.source, Source::ModelWritten { .. }) { continue; } passages += 1; - let text = item.text.to_lowercase(); - let found: Vec<&str> = LEXICON - .iter() - .filter(|(_, en, ja)| [en, ja][col].iter().any(|s| text.contains(s))) - .map(|(c, _, _)| *c) - .collect(); + let found = concepts_in(&item.text, col); assert!(!found.is_empty(), "{} rests on no live concept", item.id); hit.extend(found); } diff --git a/data/original/corpus.cc b/data/original/corpus.cc index 4e28fcae918a2755c39ce726e937f36c62623870..9c53141206b52a394c56f05303a5e2b7230c6d54 100644 GIT binary patch delta 3391 zcmZWsTTEQn6&1#IoHmfec4XQrGFF{dDaB?6%)_C?!p5D(F$on?B}cAuoj5ign`u1e z!ylFAjv3zWV2-hQhSx9yGvUs_;CR4b6shT766K>(uas6*@0|PSN7cti)xWNN?hM?f zKWOgj?7i1sYwgWPe=11*xgfQx;F-#k6;&rItJBXGEbrRB__Xj}I~Py({&wf0>scj# zQIme{xxegO9DRQKfSUBzFRt!9d3f;G2fuB{sk-o4!4KJ|KRwXwQh%YZC9|16O&-vK z_^TKhnaiu19Mb|a-@1eQJatc#A8PWLCP&OCo0=TfwGPJH@3!ul01Xua1=- zNc(QKU1@8Vu9shv%5Qf_SEQS)#dDv%?mobpe^ly7;Sg(SIB?cgp*C6LaWjSk5lGQf zh(2IG8RDTHEjX0jnAGGhYbtH0WDLi%>+71F)JNv=)R>wVnM|<{%MY=y&lW$^;ca{0 z=e^91)X%fYvLlXgXDHk?mrK3LGeJI$D2g=}vX3>>vt|IllYDIj#p+XAI9!waHF-q9 zf_i~ZFEFtJW8w}T^=rX?N`^jE%umuh{D2+>hxFl;Cik!pUoBDV34$laV?YcjQ82!b@$ML%6~@gWMAGC1glGRObF))bMSE{aUSD~Y zIP&zdBRS5TU|$_C@l>iF0E&!_#k*QCm}?b&1Y}LdrNWo{_31TKYmG1H8(}+XYXp^} zoiXZYa6pr%m{fOk?`^65axTu|@qFqhK%nE4`a7;F^@1k%nIRm9Z$`xtQ-I2nr}Yqu z85hJ!mjw)&uPl%hUj4=z1Egp^g#0tDrW@-svz1Xid&) zazY=O2JSqvh}VPyQbYkcY)1QqVunPAzpO6WeM{=F8Q0==rgV%_`lYK{y~raI)*N6# zBRjRAtOb!$7j=|Ecj=vBzJ)ZBl-pWTPvN&>B+&9mHnV}RcrVIHh~@zq0M*A=k(Fo? zi2n`EN&c%oFDtE&vUsI4oia-2=dK#{qAm~OYaW7xfc;a^w*dq)GndIDEp$;3FHf?@L*kq+Gxx_(33z=5pwC&ACR0WL=1o` z`a=*Lt$8Z?n-GPBvl|=Rg!TI_5r9$VzxZ;oyVYgS1!~pv6rs_jHVX3>z))T!NRi;8 zCZqoekuQXrG6yjcW{gS58*>3gKr_gzD-YW)LB$vgyymRq2t_&PR$KGbSNO)VVD^x? z^080|FoS^ms53;pF?|BQsPN1kK0ZezrF@_$n~Ym*rlFLHF8K+!-{pLtD%wY}Dk{}F zzP3uJlS>06a#QLUTyPS$g*ypj6*iQcZtS+Jl!0u!Z~6RJa|3KYd!=E=YX4SW@^+Ql zVl1u;cmRW9_AMLilUyg_%zTPRph8mEOW62Lr}co0CEx5QagHnnZeWX${i?i#q856k zOYGeXFSEZlBpjt2rc#Yv$zDXqs&$DU9yV88y)$rU_U$7=oIl$s?k%Uwpgy4aepMk?vL?FoL%65Bcpt_k#)lXjQ8sznNk)Ncb1hIEevFBTGp(@+ zzBGssMw;e1ag*9K4$8n5^-&-UrX!^p>Lyl+;D9l%(3|UbXj)`*aLNMiP6rU{Q zuV?b{X%#z6t5|EL+M2UpLdv!#P#kVZ&S+44#k#YA*^LDAg`)ovJ%dpI9>K|D`Z{!o zuP`F=!+{lhzqJRe*iD}Z-E{W6 z83C9DcL4PgSPg0aBlhm`OKl%?Alzq;_3JpQeCeHcc_hXs@cEFLL77{^W})z?zJ$=Y zL$ajFjOcT?BhF9%rfF zA(R`d)pN!?#ty*3adB&d$cqs#20BeRzKln?JoHHvrkHW03q%I8TWI%)@K7r9mPAvC z+IjK~XP_4-P*aWClwT^b#s!U86qztrh?(2=lg;!B@h0s>TTjp0Ltl^$J3yx}(Omzo z*s&JQ?Mxr(jYF-q>U$6hzZcYJU_1$|55QJ9H7py#LwBGR97anSu+^s-UNC0iOu&|q z&w)HgO`+cy=%2fPKq}q+K}U`?h3B0W)>4Jn>(uK)B!YcF6aOqgt^v3-VxpN)4xmiv z_mfr#0?v)oZ5f-gVq!&xU8OY$ahfdYR}r~X zQ9ck*5V;5yW?QW~EMAjsf0nTLV?R!^m~E%$w9S4kOZIQcp7%Q~J!bqNd_Cv9@B2LO z^FHqv`pt0jwIMXkFw0`g&$s29!*dMxrcEARPX9Z77@zrl>R<`i&k3Xtg>V0nK4j0I z43HcCb-@Giq`;PA33n{|X3<+BPoDMg4~7+d)!M~dcB`QWiJpF$`4T-{GJ7U_o9Qnb zmOU&U?vHye$Sjx`$D;w64aLTo!XC(MREZ4Z7rx47T=K|W?WlCP>?ciM*-cfBbG1%; zrM-$9wh!^o*R9qP=I0>c0=~L%q0O8l?TBA^EVDqoYn0N45wAsNy)p|Uus&`4;)v2v zk2jU)=VfmzcjXxQi;~P4Rkf#2i1@(|_3;f9e~H;F<;1VmsiE%>E&8}a0P^#0lZD?} zn>n-gbZw=(T6}FW>A(JfzW&r~kq$*8KA8<7ZHxtxRt>hv4D}AlUN5SX*&Uhn$=*iP zu6AA);p=1u)`Dtx4`o;eT};H0Mg6 zDK`hyTSy#?`4BV^efChfa2o|BI`7M@mC{gFv<1}pP+6>fBpQj}eVO%81>RPr74@Ow z5$;)QdzO;in z%a8~R))T)hK=pf=>bWh@L3iU|Ji%@Wk&GogcpunBNvwsKC0sL{Zl{}nvcH`hO$+&i zMW1XrCEH>y0JaVU^{LkbZ(sm`J|DH3jjHz_Q^O-3!6{fknkE#-SL}bEl$T+59e2C< z@1`MMn3tw*W#Q6o=1o#@d<0bn;(ky3D%3$R1U6zS;en3Z#bk}f*-e>onKclnNzAuW zNHVJIPInD2+utUj+|JiS$sj0Z^Jb|a(GGdH#2#HIT6DaL!a*kSqYQ)qR|&Q7S4AtS z{OK-xW$X4~{#SusaC0?&tWr^&3dYC1O60CGMw(L_(M|#qeGE?!kGi3@q%71ii~?S$ z6U7(XkibzRT7Ko@CSJPPcsT7gb8p;t)Kpa2bW>onNJVN01B!kQ%Iqpt?O}=TZd4s_ zMyu*b{!sCgdt%&f2>PH1u+}J2z9^YD{e+_`$yOF0vg(0l1K6TNInp7u6Tt$YEO7%; z3&z?ZP+e4VysKhl?@;~-xC>~5C(t!$)~7rn#>H@pffnNgLSc0@PDi&k^M7nY*OE{XF2WX5dR#1RYH=> z;saas;f)l&ELSR12GQ(bVziT3@Gzwt;tDF40iX`XkTxC=eGO2Dp*x}Ykr*rybO(|= z()&qlewBdKC` z@AiX2K2c;8ZeLU3KJHF8Xzw=tG%X@%R+vz9{F%!kpj;USR1E zNxoj6IonxVZ9i6l{bH~VasPIGigyKj1yY&J9*Xj@N&?ZbyQD%=2B07&Ej^8;_(O;c zXu1;pP>e8gI#S%j7(&A0Oas>@Q&$UXi^f9j&#pk5Y?Ad7FS+^ru!qJaXJv8%`h0Cm) zrD731s01B+^4oE0DQII}Hu1V))!~C9U-OLkiSA{7TZp-=TF3U;nhC~;Ml~Z_0 zu5>^s)GveyXxJry14D*lw2cCxXu5&%A{&&8qhE=f$D5)XFlvDHrsO$cnm41uRZdrp zHdK(smzNlirmI*uMHP9{4&0}ZQ0;9Ka|h9&ie!&9j$>Gqv?NIh#m zgOPOFli0KPncYT5I`f(-GUrQ&#LWbKZC2{P1roIfmxa`#ra?hR16s4i#;8|9aGZ`* z7jS6-DJY?fPi)gJ3oiFL3{MJubWciO)PoE@fA8W=R?QWnPumev8G>N6dkHXiK-Ajr zY7_1dfHx>C4nnHKTMP;EKLa~hllnpE#D-S$46i=a1Jk}z!dUJKqEkC zlL0BBUla}Y6Ke+)3>FXsL@TxBi!uf&f~H}bTF`^LD|K8fkYd~XY1L_(8}}K@(=<;d zv&~Xj>Y4?}oG?b()rK+TrN+}LLsIB!(yb=djo;B`Y-=C?DQ^wGT$nk{aU7=pqpq{d zr|7IzDpdxDHNO?tB4{)T`;DV;2|*C88L{(7`thcx5QB0tl9qKQx9;f*e{t{^<9}Zo Bl%@ax