From 4dbaef943763000a871d66990f6b85df73eeacf1 Mon Sep 17 00:00:00 2001 From: Weksa Date: Thu, 12 Nov 2026 12:47:18 +0000 Subject: [PATCH 1/3] data/original: ja-JP slice, 28 model-written Japanese passages and 8 Aozora Bunko Test sentences MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Train and Validation (14 each) are Japanese passages written in Japanese, ModelWritten Anthropic, generator claude-sonnet-5-5; none sits in a Test group. Test is eight one-sentence excerpts (Soseki, Kyoka, Akutagawa, Ogai, Arishima, Kenji, Kajii, Kunikida) taken from the card ruby zips (cp932, ruby and the pipe marker dropped, NFC) and checked verbatim against each card's XHTML file; licence strings keep 底本, 入力 and 校正 within the store's 200-byte cap. Entities appended (keeper gains 番人; eight pd-* authors). Items 1..28 and holdout:1..8 unchanged (digest pinned). Tests per language: counts, provenance, Test public-domain only, no markup, panel round reads both languages, speaker forms, recall 56, no Test text in any prompt. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/original_corpus.rs | 236 ++++++++++++++++++++------ data/original/corpus.cc | Bin 13160 -> 28749 bytes data/original/entity.cc | Bin 1378 -> 2600 bytes 3 files changed, 181 insertions(+), 55 deletions(-) diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 21f7a1d..1c8b391 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -1,7 +1,11 @@ -//! The original corpus (`data/original`): 28 passages written for the -//! non-memorised golf-loop target (Train and Validation, all model-written) -//! and 8 public-domain Test items, each with its full provenance. Test is only -//! read at a checkpoint: no round or recall prompt over this store holds it. +//! The original corpus (`data/original`), in en-US and ja-JP: per language, 28 +//! passages written for the non-memorised golf-loop target (Train and +//! Validation, all model-written) and 8 public-domain Test sentences, each with +//! its full provenance. Test is only read at a checkpoint: no round or recall +//! prompt over this store holds it. The ja-JP passages are written in Japanese, +//! not translated; the ja-JP Test sentences are Aozora Bunko excerpts. + +const PANEL: [&str; 2] = ["en-US", "ja-JP"]; mod common; @@ -28,15 +32,21 @@ fn stores() -> (CorpusStore, EntityStore) { fn every_original_item_records_its_author_and_the_public_domain_ones_their_provenance() { let (corpus, entities) = stores(); let items = corpus.all().unwrap(); - assert_eq!(items.len(), 36); - let (mut model, mut public) = (0, 0); + assert_eq!(items.len(), 72); + let (mut model, mut public) = ([0; 2], [0; 2]); for item in &items { + let lang = PANEL.iter().position(|l| *l == item.language).unwrap(); match &item.source { Source::ModelWritten { family, generator, .. } => { - model += 1; - assert!(item.id.starts_with("item:original:"), "{}", item.id); + model[lang] += 1; + assert!( + item.id + .starts_with(["item:original:", "item:original-ja:"][lang]), + "{}", + item.id + ); assert_eq!(*family, ModelFamily::Anthropic, "{}", item.id); let generator = generator.as_deref().unwrap_or_default(); assert!(generator.contains("claude-sonnet-5-5"), "{}", item.id); @@ -50,22 +60,42 @@ fn every_original_item_records_its_author_and_the_public_domain_ones_their_prove translator, edition_year, } => { - public += 1; - assert!(item.id.starts_with("item:holdout:"), "{}", item.id); - // The edition is the page the text was taken from. + public[lang] += 1; assert!( - edition_url.starts_with("https://www.gutenberg.org/cache/epub/") - && edition_url.ends_with(".txt"), + item.id + .starts_with(["item:holdout:", "item:holdout-ja:"][lang]), "{}", item.id ); assert!(licence.contains("Public domain"), "{}", item.id); - assert!( - licence.contains("trademark or licence matter"), - "{}", - item.id - ); assert!(jurisdiction.starts_with("United States"), "{}", item.id); + if lang == 0 { + // The edition is the page the text was taken from. + assert!( + edition_url.starts_with("https://www.gutenberg.org/cache/epub/") + && edition_url.ends_with(".txt"), + "{}", + item.id + ); + assert!( + licence.contains("trademark or licence matter"), + "{}", + item.id + ); + } else { + // The card file the text was taken from, its sources kept attached. + assert!( + edition_url.starts_with("https://www.aozora.gr.jp/cards/") + && edition_url.ends_with(".html"), + "{}", + item.id + ); + for kept in ["Aozora Bunko", "底本", "入力", "校正"] { + assert!(licence.contains(kept), "{} lacks {kept}", item.id); + } + assert!(jurisdiction.contains("Japan"), "{}", item.id); + assert!(translator.starts_with("none"), "{}", item.id); + } assert!( [work, author, translator] .iter() @@ -73,8 +103,10 @@ fn every_original_item_records_its_author_and_the_public_domain_ones_their_prove "{}", item.id ); - // Every edition and every author is out of copyright by date. - assert!((1600..=1900).contains(edition_year), "{}", item.id); + // Every edition and every author is out of copyright by date; + // a ja-JP edition year is the work's first appearance. + let years = [1600..=1900, 1868..=1930][lang].clone(); + assert!(years.contains(edition_year), "{}", item.id); } other => panic!("{}: not model-written or public-domain: {other:?}", item.id), } @@ -84,7 +116,7 @@ fn every_original_item_records_its_author_and_the_public_domain_ones_their_prove item.id ); } - assert_eq!((model, public), (28, 8)); + assert_eq!((model, public), ([28, 28], [8, 8])); } #[test] @@ -99,14 +131,30 @@ fn the_original_items_spread_over_train_and_validation_and_test_is_public_domain .filter(|i| SplitScheme::V1.live(i) == Some(s)) .collect::>() }; - assert_eq!(in_split(Split::Train).len(), 14); - assert_eq!(in_split(Split::Validation).len(), 14); + for lang in PANEL { + let of = |s| { + in_split(s) + .into_iter() + .filter(|i| i.language == lang) + .count() + }; + assert_eq!( + (of(Split::Train), of(Split::Validation), of(Split::Test)), + (14, 14, 8), + "{lang}" + ); + } let test = in_split(Split::Test); - assert_eq!(test.len(), 8); + assert_eq!(test.len(), 16); assert!( test.iter() .all(|i| matches!(i.source, Source::PublicDomain { .. })) ); + // No model-written passage was admitted into a Test group at all. + assert!(items.iter().all(|i| { + !matches!(i.source, Source::ModelWritten { .. }) + || SplitScheme::V1.of_bucket(SplitScheme::V1.bucket(&i.group)) != Split::Test + })); // Train and Validation hold no public-domain item. for s in [Split::Train, Split::Validation] { assert!( @@ -238,27 +286,27 @@ fn the_recall_probe_covers_every_live_original_item_and_names_no_work() { "bag", weksa::run::LowererReach::BriefedSubagent, ModelFamily::Unrecorded, - &["en-US".to_string()], + &PANEL.map(String::from), ) .unwrap(); let mut model = Unrelated(vec![]); let RecallOutcome::Read(reading) = run.recall(&mut model).unwrap() else { panic!("answered"); }; - let r = reading.languages[0].clone(); - assert_eq!((r.n, r.answered), (28, 28)); - assert!(r.chrf < 25.0, "{}", r.chrf); - assert_eq!(model.0.len(), 28); + assert_eq!(reading.languages.len(), 2); + for r in &reading.languages { + assert_eq!((r.n, r.answered), (28, 28), "{}", r.language); + assert!(r.chrf < 25.0, "{} {}", r.language, r.chrf); + } + assert_eq!(model.0.len(), 56); // No item outside Train and Validation is shown to the model. let all = CorpusStore::open(&c).unwrap().all().unwrap(); for item in &all { let live = SplitScheme::V1.live(item); // A recall prompt shows the first half of the item, cut at a word boundary. - let mut cut = item.text.len() / 4; - while !item.text.is_char_boundary(cut) { - cut -= 1; - } - let opening = item.text[..cut].trim_end().to_string(); + let quarter = item.text.chars().count() / 4; + let opening: String = item.text.chars().take(quarter).collect(); + let opening = opening.trim_end().to_string(); let shown = model.0.iter().any(|t| t.contains(&opening)); assert_eq!( shown, @@ -314,7 +362,7 @@ fn original_round(family: ModelFamily) -> (weksa::run::RunStores, weksa::run::Ro "bag", LowererReach::ToolLessRelay, family, - &["en-US".to_string()], + &PANEL.map(String::from), ) .unwrap(); let scorer = weksa::score::Scorer::new(Bag); @@ -328,14 +376,14 @@ fn original_round(family: ModelFamily) -> (weksa::run::RunStores, weksa::run::Ro fn own_family_counts_and_labels() { // The original corpus is Anthropic-written items, all offered. let (run, r) = original_round(ModelFamily::Anthropic); - assert_eq!(r.own_family(), 28); + assert_eq!(r.own_family(), 56); assert_eq!( r.parse_reasons().iter().map(|c| c.count).sum::(), - 28, + 56, "every item was refused at Parse and says why" ); let line = run.round_line(&r); - assert!(line.contains(" own-family 28"), "{line}"); + assert!(line.contains(" own-family 56"), "{line}"); // A different answerer family, or none recorded, labels nothing. for family in [ModelFamily::OpenAi, ModelFamily::Unrecorded] { @@ -471,7 +519,7 @@ fn no_test_item_is_offered_to_a_round_or_recall_or_recorded() { "bag", LowererReach::ToolLessRelay, ModelFamily::Unrecorded, - &["en-US".to_string()], + &PANEL.map(String::from), ) .unwrap(); let items = CorpusStore::open(&p("corpus.cc")).unwrap().all().unwrap(); @@ -479,28 +527,38 @@ fn no_test_item_is_offered_to_a_round_or_recall_or_recorded() { .iter() .filter(|i| SplitScheme::V1.live(i) == Some(Split::Test)) .collect(); - assert_eq!(held.len(), 8); - // A Test item's text, or any run of five of its words, in a prompt. + assert_eq!(held.len(), 16); + // A Test item's text, or any run of five of its words (twelve characters + // of a text written without spaces), in a prompt. let leaks = |prompts: &[String], item: &weksa::corpus::CorpusItem| { let w: Vec<&str> = item.text.split_whitespace().collect(); - prompts - .iter() - .any(|t| t.contains(&item.text) || w.windows(5).any(|k| t.contains(&k.join(" ")))) + let c: Vec = item.text.chars().collect(); + prompts.iter().any(|t| { + t.contains(&item.text) + || w.windows(5).any(|k| t.contains(&k.join(" "))) + || (w.len() == 1 + && c.windows(12) + .any(|k| t.contains(&k.iter().collect::()))) + }) }; let scorer = weksa::score::Scorer::new(Bag); let mut model = Record(vec![]); let RoundOutcome::Recorded(r) = round(&mut run, &mut model, &scorer).unwrap() else { panic!("answered"); }; - // The check bites: the round did show live items, 28 of them. - assert!(model.0.len() >= 28, "{}", model.0.len()); - assert!( - items - .iter() - .filter(|i| matches!(SplitScheme::V1.live(i), Some(Split::Train))) - .any(|i| leaks(&model.0, i)), - "the probe cannot see a shown Train item" - ); + // The check bites: the round did show live items, 56 of them, and the + // probe sees a shown Train item of each language. + assert!(model.0.len() >= 56, "{}", model.0.len()); + for lang in PANEL { + assert!( + items + .iter() + .filter(|i| i.language == lang) + .filter(|i| matches!(SplitScheme::V1.live(i), Some(Split::Train))) + .any(|i| leaks(&model.0, i)), + "the probe cannot see a shown {lang} Train item" + ); + } for item in &held { assert!(!leaks(&model.0, item), "round prompt held {}", item.id); assert!(r.calls().iter().all(|c| !c.items.contains(&item.id))); @@ -510,8 +568,9 @@ fn no_test_item_is_offered_to_a_round_or_recall_or_recorded() { let weksa::run::RecallOutcome::Read(reading) = run.recall(&mut recall).unwrap() else { panic!("answered"); }; - let rr = reading.languages[0].clone(); - assert_eq!((rr.n, rr.answered), (28, 28)); + for rr in &reading.languages { + assert_eq!((rr.n, rr.answered), (28, 28), "{}", rr.language); + } for item in &held { assert!(!leaks(&recall.0, item), "recall prompt held {}", item.id); } @@ -540,3 +599,70 @@ fn a_model_written_item_in_a_test_group_is_read_in_no_split() { let item = item_in("item:f:2", "x opens", &group_in_bucket(80), public); assert_eq!(SplitScheme::V1.live(&item), Some(Split::Test)); } + +/// An Aozora Bunko file carries ruby, 《》 and ｜, and ［＃…］ notes; the items +/// hold the base text only, in NFC. +#[test] +fn ja_test_text_carries_no_aozora_markup() { + use unicode_normalization::UnicodeNormalization; + let (corpus, _) = stores(); + let ja: Vec<_> = corpus + .all() + .unwrap() + .into_iter() + .filter(|i| i.language == "ja-JP") + .collect(); + assert_eq!(ja.len(), 36); + for item in &ja { + for bad in ["《", "》", "｜", "［＃", "※"] { + assert!(!item.text.contains(bad), "{} holds {bad}", item.id); + } + assert!(!item.text.starts_with('\u{3000}'), "{}", item.id); + assert_eq!( + item.text.nfc().collect::(), + item.text, + "{}", + item.id + ); + } +} + +/// A packet that cites an item's speaker lowers only if the speaker has a form +/// in the item's language. +#[test] +fn every_cited_speaker_has_a_form_in_its_items_language() { + let (corpus, entities) = stores(); + for item in corpus.all().unwrap() { + let speaker = entities.get(&item.speaker).unwrap().unwrap(); + assert!(speaker.forms_in(&item.language).is_some(), "{}", item.id); + } +} + +/// The panel [en-US, ja-JP] over `data/original`: both languages are read, each +/// with 14 Validation items, and no Test item is in any call. +#[test] +fn a_panel_round_reads_both_languages() { + let (run, r) = original_round(ModelFamily::Unrecorded); + assert_eq!(run.record().panel, PANEL); + let tags: Vec<&str> = r.languages().iter().map(|l| l.language.as_str()).collect(); + assert_eq!(tags, PANEL); + for l in r.languages() { + let refused = &l.validation_refused; + assert_eq!( + refused.parse + refused.admit + refused.lower, + 14, + "{}", + l.language + ); + } + let (corpus, _) = stores(); + for item in &corpus.all().unwrap() { + if SplitScheme::V1.live(item) == Some(Split::Test) { + assert!( + r.calls().iter().all(|c| !c.items.contains(&item.id)), + "{}", + item.id + ); + } + } +} diff --git a/data/original/corpus.cc b/data/original/corpus.cc index d778901e62568a8a52f6777b0c158717ccaf0df6..108afaa2bd95cb74f2aab9d17d6715b9d807bbc4 100644 GIT binary patch literal 28749 zcmdU2+jAS&eXi1J^w5WRh7l2`%Q zMRpg0>9nap-9=fV?vhDGcZm{3QWPbbqAV$G?Y{XB*qOd$z+C{H-lq?pzNFvpEbhAi zc2cPYXKWH&oLwxw{mys3-|zg+(W(0bsklBM3`E5)O;=R0Yq$Tzxw*6$(}b=8MNOu( zosuploJSrw+Tc`YGvkLpeMQ676T(&@A->Z8isQ6bRu>!mP*gQZk=@$h(rA_t6JfM@ zrT@>~|Em+uj~e@j-+1Hn9NleyREbBFl-{{Z=y$oN2fUB(?ezOQ{rq-6-yiVz`+9bK z{-vLeJ5M$?=5OA~Pv6gu9LZ*8a{I0n4)4odzMsw9GVdK8x*q|f?Nj_)1YF@}W6*e!9Zgp?jtR4&rR`TP969j|lMceI)_ z7tdG|cR9M_%<*jInRV_4{awm`BFB+{7hw|DpVm`P{A1 z)tq(tiZy+Wvo6hB_pe%q#tP&6Ugy+QczBJh=%U7{VnW$1Mz~ex@w0e7*5Nteoxge9 zT0Dz88WeTVF2YrwwXy35GT0H}w&?<$ziLoYHJwWm*cvtU_xXA|I6;nZZwX03uK${^ zb8%@&|Jd+h!gnD zbGe86%u{$ev-!zA=GdWZ228$`9l?tO+00aSPsW-dN@p`4FD>8&_;(2>;tF`nJouC@ zvM^08Gqdci%S#JKmKGPYnaSL(41O*x9%X-`!5zlkOXXfLTj>R{VI?LkRCR)wGAwjP zrCpLLbp{+A;Twgi^M!-Q@|m-4U7&~ecYI>aA1)lfl0P!Xt$OprR6^o5(?@tBzpuwv zBtNdSllS$6x^43RO%w8W_x(ELFW+Sg=KuU$b_8gSWi!uTI`e%Encv;_LYW_L#r$s1 z%s=L2ez)-fSd!awH-89>odVUNz`3V)7#n8_nPcWhAoDC}%*6S4cH{sWn9ui`&(0Mx zXXqgD9DM-|SV!1%F*^bgyPLapB0F+2cLaoowqIfwyp_#B{2|R1de2notqciLHM?~o zCTLfh{iJ#QmU(?DyZ1@?=F60hDT$<@X=NQZA?|adg^v%s zY${_wY9!)3|3az!m#wHA@J!_ooKy}NTeBkz*}a#+S99+fQs%goqf5RL(vGjBTKBUan!l4J&#H4xbWApeznX+sU z*T^kIUN&j@#`?>5zfjWdX+_!|Po(XIW4`MoZIAJ@!qrLeGn={1%*3QQ_LvbBZW#Xf zCWxB<2*v_Lg=+(Q*-sdb37iE&*>8cj&|>zp_!ioqhOvPEt$Hz8n18_B$+J3=SDOq` z-X+afF?=AJN+@ndo44jCVN!7$=5=(Ueb(6vxv>N0;yE{h2fBkj0eJ428MxE@{8j7z z6KJwDEPN=W1#UBas3&3eh5RL9rYr9B2YY+^Y?9A4A$fO0k~jJiWvZ8jxQgehgh)aY z(Dz?D(E|3$V=b-78FtH`A55EsCzdYrV4%cQ9z5*}@x2}lazg`Q z-!iNNc_?PV03)=+x-iQjPC*1=ew4|5GFj&BuCA1h{*@5QyhSiz9>10!or7q-Pr99`1(L^a~xh{GI#o3ZuG7-eVbdgx%hG11|^vUy4P}koFZ?Vrw-H@>n&eM+M;BMKAqhx%a?a!UQR5-hb_>6Q#=2l@GMgh!x zPLlgf?U*&^iDP0)Cxc#t-ovNTk6;fNjgPh+ef1)cV`Cy0ft^AQ2=wW!e zOOHpFo{XYl=|i}8hf}hmrDCEd4;%`XA~!!D-0AP(dwn)7|Go(=aSIKSa&`A*r{nw; zjy|48WS75l8Ms!N;Z`>`)TzHse3kQXGT#|=4WsplFx==eKs*=<= zq4-x)(K$)ukcy2{5((rvBSJY1T{+a*;~@Q}4doM7<4)HrxVZrx$xgZx z%H2~+b(=6iiP3e664U57-TFpB(^qr5l{m7+z7FGiYY|orCzKSLzazvYB(QM+4d@b_ zpZDVnt3pJgO9Q51PS6HaDU7S?3cKFAR0KELVFao*eJF+0D`p3Vx)gN?8Exc#*NDkf zcnuP>!8OSt`qia>pd07vdyPCG0TfP0@|JzFti=V)gm9a>xGlm5+!kOBNLi}e=R>c0 zM^$#=i*w&AeIh%ROKG^nw1m7kxG(7%`x2P;P-7e}hIGCt4~Ses(DArkveP~zRGXcq zG&IXM2YuYG6hh~S#Lxz6b+^EMmZvi z!fmuKi%P7?(@b|H+BfQ^+* zF)+JkGbkczx~kZtszMhn5~nJN4hsW{JX@j-`^&}Cy-K2CGly@O!*KHVNg*vh?;k|Shx*3vm(_$F{v)NZgp z4WGC1#81NFXRNqZH<}Cz;S{tPifKEP8gueJ5jIEUHVuc(e>=6bL#eoLQajfRT z_>JR?S8-wv{SSbNH*5qd+1aWJu!x|hCAl2i*p))gv1S2!L6>=(RT7bGZL@E2W_?sp zG2S4Kp$mCs4$p^(Ir!C5IbN!l^-c-<`xVEw)>gJ&JMOc`Tq?x!SP|&!=eu`&zQQ-| za<(y$yMxKdnH;Nrvi96ylbx7@zlqdX?%-ZFvqRHBNQ2kp9%*{$9@1*r4AN()F~Jn$ zWPS{_R9A8rk-9`F3TjDk9I3YvyI^p-7z@Wl5eDd9)tBguVIBm6vPFTFc~3>MN0d+9 zUMuTSB$A{7V`v~Qq#`2f3}so=J9|2NxK-~_Q788XxCvKVx>j9f8ke@Og?tn2Ie3S7 zu@iNdXioW!AtgT88CR4frwi~OddDv+-I1qM!@Lu1U+a`_5$>;!yPa;lkvoXF;G^az z*JyTvCWSF6fe8vgW!LdwMtH`Yd4x8Fix*e}luX#=Yj`nfp@IixZSJ7Xmw*5>I#Dng zMwROFz|})xYZ2Fg8?Jt1&^ost$YDN`rhF);JB5iH99dd;!gA{ix$B<-a+C<#Gx^|d z>8;t^{#$H%2D3qUI1J#-6hxIhB2-La`je`p8lYZ7WnBnM%LDSJsWZElxClAt203qh zKR+>HU7|WN^DuPdI2GuOoV2E|vT`%3jlhhX`52{|v$;Er*4L?EvoL;#p2JO0|9Y_{ z&AT;1cdnY#2`MJX%LCq~>7^QYSGmF4ZR}uknzPpVNmLx8=bK}Y-1F=pCfp&~OqNlb zf;pozc1d7;<8lqGbsK0;<)#?s?%Gkq;uxlnmIusD(>XOTf87mcl%?hFUM30`XJ}cO zJLxUJF4f_nRt=*ctIMY1G_aKv0W-#!Va5mv@}ecvYERH=%LLk{sg4?GUvq=D$9UgF z@f3}6zy@fQRuE8CHm2vNK-~P^1t>zPcnVW+D5?VRKoqZW9Jm7SQ8p6db?G7v#iEA9 zSTL|mq_}BbpayAIt>`s&Fy}UA&Om#i#y}CUVaOsEM=&3HT(~?9(I`AAJ_>6JCmZ5m*vOIKC%|r`$u%$!q6Bz(XhqXly9VZOxe+>OY|ovZ2Tu7({6zg8 zMl-wbkJGF%^TLI_lc);@cVVathv{ipk5Ke+u1a2xFvv^V;;tAqMlLEL%^O`NRBjry z)?nIecKeuW)%UTm6A& zf&mf-N4*#}7>Z>=-#*W=$5-6w+h=^2IWEA4(d(!ktPx0dbODuk0;#rZPAJ|ejDx-q zV~7JD2nB#{aa;{k5pt-4+K$7UGxYUtf4}Y$IP=EVLI$3876n?czIGW;{tpB9m3?vJOU0{R5V+P>!w&N(nMzKtnYHs0N#6$+~Go0-eMio=22?Xtz7T`W0Dvd_8nd&HG zIJGRsGv0g9>}DHyzp*2K27~@g{w$C=TDWqWa6Sl%+D?FAXJJ~=MVPl?+#m%Qx{#^> zswj3vCIV&ym6K!}TJgEgHpG_4kj8sciU3k<-EMT|je)`=(AgfzKwd1zqG61gUG)A* z>pW%Mtc&<*CxU?GBx^vK2$t|+#+Nc`I354N8DoS)^l^~oG8&C@_p3>r8cIppG9k9N z*qIRbxgp+d{5b#o^wJ{acea$$gbqU-Zss12S+{T;@Lj-&2H$J$-)r3i;c=ukuCp_o z$Oh4iTPWW)4}t&16WxZw8n|_F2-6bFLr1*D&;++zCWsFh-_Jo25J(i=?uB9m0*(`B z51`J~T!j9TMxbz?{2N5)6lIHYS0Oief!=@y2tts%Lq@~ZYSaveSbeB26L`JF(FE`B zxv{y&*l0~$EYKhT@BajX2 zD2`mIZR5(_nxPMKUCV^J-Xdl~+AW&ngNWv4(ZR8xXx^H6NcwOFt1l3&0p_E?J9ps; zi$Lt*4j3aNJVFaHXC9K7F6Jg6+z9;|jdC^6VkWa4!w=_F5brHeCTRc2O%)I*=YM{c zh>Om$Q;Es5R9hKPxa ztG{hIS?5I8n_VDuUvoh>VEhczQc(gcE3Q#({wTeHWnN8}dgmTAj=< zKtMr7duW4=uss$=u1Rxo+FStEv)BO0mIU(}p3SYoWz3Z#z0qj)zQ&57lrg+K0QF`Z z8Wl_QqFCY*GCWiH{cH&#Ggg>(wzE1=hr&hpiJ9DkF@TOhBbz~EfDREse1R7<;uXjR z1M8XG{tMJ>VzFd?G5R&iPSgx>R65#*afOq<-pm7`{W~u74I1n0wfE+AWGsOgOOye| zJtHNNn6(~$g2%y97B;zAMxr=_)F@+F17=M4upP6%PB6XM1p@PHE?|a?&Dco@SXt*m zKSGkSJ;db4L_(SuM)+0C^RSk)Ol2uOK~8rB@Zu3LkljbJ8mkb)`op#iex0~_Ppbc+ z3$A^}=KR%4QG<1ujX)Ii!jydsNid&&YTkehRA$~-r4@!c1Rzk6DRU8m%&6^-YYkg9 z6iBsU*PpGalWMuJFwpN0_WMFRKKtkK`ZEpRTJ9$)35z9A(ut}=4TZ}wtjf}<;MQI+ zfl5Syi=c)Kg^KA?*>Y`d*>pLor6x@Pd%s~}*)}mgSXEb+q>ax07v*GZ<=VYEn3|@^ zS$(IkyPx;(_{+D3=%pQBdV-7r$eDjVKQRwlZEQH>L7dwieYDue)V7VGVWL zV%0QBU5(X0IUWUN1mql_H9ii%heiOv?rzJ%<+pd`j4t`JqsHEYwh zAT^B(t3mn~2T0w9Z#zJ-i&trdVFERz07~J+L=uZL39+!CQJpl_*{iCOc15ZpO;PmU zwgswbh*u5NFC9PyjBgT9Di+DA*ws?QHm6kBjg?J;&ekSt!jRYko0t@DR^|8qE89(@ zvTAJp%YjXg@#;HqtN=vKy%uAXDe{S}@G9y#tw(mnr;;p)3z3MZwh4PqoT?pPe)|tj z*AG`s>MT%-fusa#ihxIuLCET>`lE!$pMnQ{<$0*)n@Z|{Hd;g@rsdzYcbBN8P{lPNdmqNW0Bws=(}<=TqyKhb z)MtF>y^^i8-L{4`RoHn08*&hCMYT#}YcsIvRCo2ZHcr%n+~N>3%^7G0AKscizO zL%Nz5{Z!M=VL(yv)A1%b1X}#g7DX3FLJiATFjC?j^f@eCg^hP*U8QegDLchNZ3~j; zh$dIxH&LzAK|CuN2JEhmO=*Z{Ow39voLH}QC zd1F;0!BPau-gbx`*(J~>4OmN$O>kTxs!}g()e>nFjOx{ERTwkbmN9AUkWwiysgYd&iS#R_298mN7PW8QN(^D9TU3m9wPvzfGv-RCUk6j#JfpjMYU$irh+) z_A6l9DTSFUWgAy8#TVouQGL~&Ulovakg;t+`t`#b)S|Go0^7NPi(m^UGVn=NMCQdd z@wN>cM7{wbHPQ=1uF6#fR8&Z}El{4roBwe1!=Mq^Xs=wS_z!}FCF_6+>eYX+ZUh-Hh?BR;LyhRkODNo^_ zz_F9?b~mLRIP8eYJPtHH7a+~+I4T#sCj@zxJyj#F|C+J9jMxKE{vE{Wquc1ePCNT*N$2N5EpJ&09y^DB>+Dw)Z^D{ zytWow9mzx3+r)Ml*wP^d+Qk*@96_*R6C^>#u4|y4D?k-5%g%3Ch&&IW|5guC(BO+v zHcf}wv33a=>YPmY?SWKKX_rCJ%{APvAccQqt?A;q|Jy`;os4-MKR^4|dZa>ze?9yv z;zZlz+ruQo2OE_50=4-}1{*h|KUw1-kaTL%LTwr)&&xN$DzV#khb2gIgHg#Mzi zQ8K4#x3)siKwD-)A*f|(M|t3%h_tuFPV!&uI-$ti;f=$^a=UaaabJvrAr}^V(@DiLtUXR`s1$Ni?i>L ztYArW9g|q(ix`T9$#GZc$De*pIbv`8_rZth{RUBpM<|YD$N4eqXFv5ZjE`Zb7&a8} zhnRus^J|(@@`tvAMBlfQsXaA{KqJhVwlfEP*CtZCmHT_;Lg6f#9mVq~m8;YHBpJ_Q z5en&pi{$Z$44qspO0y7QR+Rw&J`=hSyG=!gaynENim4U)Z3}3 zj1c%DAx2~r0rhXX6)*zB0#ed>ihxh3a277WHWaoA{nO`A^AZx0M%f3|)g9yYliKO; z<$u=7Z`RyQ#Y>gbTb1-?EwcnJUy**6WC#WmA};|B$`Qb@{XOX;icH2z6vsLN<1`PQ z&83@cll(=TzOYY=BlZ!>c&kkG?rm{=(dF+Cy8wgqucfa-L|@@ z*|tF|Mk~z}8K+@Ifr|=|WN=kVhzb&vUT_u9V*x-h0>+4MLi%OXs0fr7a3?rlX)P+7 zXLqO`#SL^aE&^-9$Lqrjq!}<7M;s*)1fW#p|K%7eUfCXo*1f+n=laRBC2yLUKPwxzjlu&@x=;Fpgpa!4d_u%LZaZtVVRQ6ec(1c54~$3unvb7X z(l0A(y1D8i)DXEi%URt{4g~$Z_15O7QCBEHNpz*Fd9JE9>+Y2*) zw{~xZe%m=jKz;g*_9=|y0Yn2b|A|o75xp0*PmN@4=a*__)hAF^d4@gYzK5Bd#~~+XTCkGZ06mm zi`~YP)l%|lDY;czzEN7%ysYRCvX1ShM#;Z1nCQR?j^1t;GTV%2Q;j$+Oup+-h#-+H eWAZqWxv=aRN~nuqeCFcS8p4VcMiGoe4*d^lIxP Date: Thu, 12 Nov 2026 12:57:45 +0000 Subject: [PATCH 2/3] data/original: ja passages rewritten onto the Dao r3 concepts; ja Test sentences pinned verbatim The 28 Train/Validation ja passages are new Japanese sentences on the concepts the pinned inventory holds (water, mind, belly, bone, sage, will, heaven, earth, fill, empty, weaken, strengthen, compare, resemble ...), written in Japanese idiom, not translated. The branch was unmerged, so corpus.cc was restored to 21eaa07 and rebuilt through weksa corpus add; holdout-ja:1..8 and original:1..28 come out byte-identical. Tests: the eight ja Test texts, groups, speakers, split and edition URLs are literal expectations; every passage of both languages rests on a live inventory concept (en 28/28, 18 concepts; ja 28/28, 21 concepts). Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/original_corpus.rs | 160 ++++++++++++++++++++++++++ data/original/corpus.cc | Bin 28749 -> 28446 bytes 2 files changed, 160 insertions(+) diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 1c8b391..47ed2d6 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -666,3 +666,163 @@ fn a_panel_round_reads_both_languages() { } } } + +/// The eight ja-JP Test sentences, byte for byte: each is the Aozora Bunko +/// card file's text with only the ruby (《》, ｜) removed and NFC applied, a whole +/// sentence, checked against the card's XHTML on 2026-11-12. A change to one +/// is a new item, never an edit of these. +#[test] +fn the_eight_ja_test_sentences_are_pinned_verbatim() { + let want = [ + ( + "item:holdout-ja:1", + "親譲りの無鉄砲で小供の時から損ばかりしている。", + "holdout-ja-bocchan-6", + "entity:pd-soseki", + "https://www.aozora.gr.jp/cards/000148/files/752_14964.html", + ), + ( + "item:holdout-ja:2", + "尾張の停車場で他の乗組員は言合せたように、残らず下りたので、函の中にはただ上人と私と二人になった。", + "holdout-ja-koya-hijiri-4", + "entity:pd-kyoka", + "https://www.aozora.gr.jp/cards/000050/files/521_20583.html", + ), + ( + "item:holdout-ja:3", + "高瀬舟に乗る罪人の過半は、いわゆる心得違いのために、思わぬ科を犯した人であった。", + "holdout-ja-takasebune-14", + "entity:pd-ogai", + "https://www.aozora.gr.jp/cards/000129/files/45245_22007.html", + ), + ( + "item:holdout-ja:4", + "ジムは僕より身長が高いくせに、絵はずっと下手でした。", + "holdout-ja-hitofusa-no-budo-4", + "entity:pd-arishima", + "https://www.aozora.gr.jp/cards/000025/files/211_20472.html", + ), + ( + "item:holdout-ja:5", + "風がどうと吹いてきて、草はざわざわ、木の葉はかさかさ、木はごとんごとんと鳴りました。", + "holdout-ja-chumon-6", + "entity:pd-kenji", + "https://www.aozora.gr.jp/cards/000081/files/43754_17659.html", + ), + ( + "item:holdout-ja:6", + "えたいの知れない不吉な塊が私の心を始終圧えつけていた。", + "holdout-ja-lemon-3", + "entity:pd-kajii", + "https://www.aozora.gr.jp/cards/000074/files/424_19826.html", + ), + ( + "item:holdout-ja:7", + "多摩川の二子の渡しをわたって少しばかり行くと溝口という宿場がある。", + "holdout-ja-wasureenu-7", + "entity:pd-kunikida", + "https://www.aozora.gr.jp/cards/000038/files/1409_34798.html", + ), + ( + "item:holdout-ja:8", + "一人の下人が、羅生門の下で雨やみを待っていた。", + "holdout-ja-rashomon-1", + "entity:pd-akutagawa", + "https://www.aozora.gr.jp/cards/000879/files/127_15260.html", + ), + ]; + let (corpus, _) = stores(); + for (id, text, group, speaker, url) in want { + let item = corpus.get(id).unwrap().unwrap(); + assert_eq!(item.text, text, "{id}"); + assert_eq!( + (item.group.as_str(), item.speaker.as_str()), + (group, speaker), + "{id}" + ); + assert_eq!(item.language, "ja-JP", "{id}"); + assert_eq!(SplitScheme::V1.live(&item), Some(Split::Test), "{id}"); + let Source::PublicDomain { edition_url, .. } = &item.source else { + panic!("{id}: not public-domain"); + }; + assert_eq!(edition_url, url, "{id}"); + } +} + +/// The concepts the pinned inventory holds, as each language names them in the +/// passages (a surface that appears in a passage). Every passage of both +/// languages rests on at least one live concept, so the two panel legs are +/// measured on the same inventory. +const LEXICON: [(&str, &[&str], &[&str]); 21] = [ + ("endanger", &["endanger"], &["危う", "危な"]), + ("prevent", &["prevent"], &["防げ", "妨げ"]), + ("control", &["control"], &["操", "支配"]), + ( + "request", + &["please ask", "request"], + &["くれませんか", "いただけませんか", "お願い"], + ), + ("sage", &["sage"], &["賢者"]), + ("mind", &["mind"], &["心"]), + ("belly", &["belly"], &["腹"]), + ("will", &["will"], &["意志"]), + ("bone", &["bone"], &["骨"]), + ("empty", &["empt"], &["空にし", "空の器", "空なのに"]), + ("fill", &["fill"], &["満た", "いっぱい"]), + ("weaken", &["weaken"], &["弱ら", "弱っ"]), + ("strengthen", &["strengthen"], &["強く", "強い"]), + ("heaven", &["heaven"], &["天"]), + ("earth", &["earth"], &["大地", "地は", "天と地"]), + ("water", &["water"], &["水"]), + ("compare", &["compare"], &["比べ"]), + ("resemble", &["resemble"], &["似"]), + ("continues_long", &[], &["長く続", "久しく続"]), + ("true_words", &[], &["誠の言葉", "真実の言葉"]), + ("seem_paradoxical", &[], &["逆さま"]), +]; + +#[test] +fn every_original_passage_rests_on_a_live_inventory_concept() { + use weksa::inventory::Family; + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/original"); + let tmp = tempfile::tempdir().unwrap(); + std::fs::copy(dir.join("inventory.cc"), tmp.path().join("inventory.cc")).unwrap(); + let inventory = InventoryStore::open(&tmp.path().join("inventory.cc")) + .unwrap() + .inventory() + .unwrap(); + let rev = inventory.latest().unwrap().n; + let live: Vec<&str> = inventory + .entries() + .filter(|e| e.family() == Family::Concept && e.live_at(rev)) + .map(|e| e.gloss.as_str()) + .collect(); + for (concept, _, _) in LEXICON { + assert!(live.contains(&concept), "{concept} is not a live concept"); + } + let (corpus, _) = stores(); + for (lang, col) in [("en-US", 0), ("ja-JP", 1)] { + let mut hit = std::collections::BTreeSet::new(); + let mut passages = 0; + for item in corpus.all().unwrap() { + if item.language != lang || !matches!(item.source, Source::ModelWritten { .. }) { + continue; + } + passages += 1; + let text = item.text.to_lowercase(); + let found: Vec<&str> = LEXICON + .iter() + .filter(|(_, en, ja)| [en, ja][col].iter().any(|s| text.contains(s))) + .map(|(c, _, _)| *c) + .collect(); + assert!(!found.is_empty(), "{} rests on no live concept", item.id); + hit.extend(found); + } + eprintln!( + "{lang}: {passages} passages on a live concept, {} distinct concepts", + hit.len() + ); + assert_eq!(passages, 28); + assert!(hit.len() >= 12, "{lang}: {} distinct concepts", hit.len()); + } +} diff --git a/data/original/corpus.cc b/data/original/corpus.cc index 108afaa2bd95cb74f2aab9d17d6715b9d807bbc4..4e28fcae918a2755c39ce726e937f36c62623870 100644 GIT binary patch delta 3371 zcmZu!TX0iV6rBP^%ZLb+$KWtv%8Y|fY5JrTuz*q!l!r3Dl?hNwj7_jjN8yK@P)Z-c zP)^gdw52@y2rW$l0W2vG`Opz(oZ-ihJHzmpn|pKf<&XY3>e}b#HYw((lgG{3YpuQZ z+NZxg$vXQeD?B4ByR@{Tv@nvB6`7WNd2jk(#>vn2RF*+@rZUIJacYCqHGq%$KcPP_b1FUrY2|keMgZ*DbRmowdelw|t%dl_kH{dZfW-sj=|6 zMFGD5HPbAjU`DCAJ^X3G{7SR5NjrX1WN^GK=8I^n%)n5v92aPbznh z$?ntf%k8MoU#VEaAC$j1t)ZSjDIez1*YhSTWQhvqLdmRkHmczta9-@DUyunX?YD7; za`KAIa3l))1;cLH-OQPJ5&yMh_RNO*h8jm5e_B4o^Yc^3zABD=zew5>i+W@>07P*X z!f9%#LuQ~dB)i=JF0)HAJ1@IiKuhiVUes-r8KexUy?sQEjmU1C5EiuBWH)!27caP; zuAb9T>!?3y-6ueJRy_-1d$IJPa`vKn4pl>O4^9lm?p#-n4+2f1>#EF7Qw=JKMnO^! zV8wkSv1k;}%dC&eLxs}-$M~wv=4_kAX0`KsRULfSYxDV^1*vu|Cf2!cOWV~(uW}b^ zg%TqHQuNj>V$2$epUbVbc8NpVl$Mi%Vu)He0dlTHe+0$iqd)_Nh%0~m!K`d}1R@(H zcNUt2v(5a@hDC1{mP(tb+YFT;T@wk*f`r}0RM((j1{Y31>I6F{`Z%6&;d$^Dn3z{| z^QHB(XWAS#N1cOTSv$zbi&8=^5#8Ke24a33=uyuErw{=8^B(Fi8&$8~q%KEYLQ6=1 z60HIlUcf@<_`h40!i=-CE%pNryC}7+BvopTC{?yvs?ttrk4N> z5lI*2Vi;7Fu-8C5!%ibgPJdLEI&+o)yl#!OP9uc)alaBBQ^v_>Y76>E$YVE=4C1R^ zI4tQ0bsXNu1C5Y>#dN#Xi3Vee%1s{`Ni(bGs?}LtTao7WiXv&98peR4djm4Vcn{(H zL~k#!YHesvBN-ijy>&%QRZntcQOMmQ`b^qmTxv<4A>O>kbks+-uIE zu1$x&{BBhz#%dn_qayE91J_HLa1~0G@k7rNyQ8PDF zO=MSWqEqk+iWL`_++g%}a$k_gM5ZB3+~DY^g3k!sP-od!onIt`)i#=Hjl!l(h0CPf zGP^DaVj%@%<74D(G8D+75^YsoQ|&s83%=cn3osfoPNz?uj(5NfXn(q)zHG48IFcyd zds82-Lag$FOej`MpBt;fBL)O|qX_;Y;F1|GR)|OG0krgQfe23qlv8LI+@hl%rXB4* zA%3n;8UJV9ys0|D3x^%nB*L+`OrMcXgm6wKgymAD01#3_NSx&SCifQnnH*{i7^c28 zElq)B@}5A6e9&R9&M!u9IG?j^9?z*VS&jZI&5&}%(he~S0Wkqk0y_=tu@PWLeY2qr zx+#(TsEiT8+kg^1h9^ZMu_>U9J2?jwFwSZ_e_Y-v5Y%ijePJM&p8-LMv< z#DK$rK#;VC-i|o7HSMA1w6ui|Py=z_M6;heW%dtSzQyd@_{NPL0>gu?sqoFqfT2{{ zBQ8iVLz~ic8GO_}Tw9EL1I;3A5Y!za_E?h=M$-9>zToZzX#j?LAzT4pr)RH|pE5sB z;XA|bIO4$l(+6NBI}Ks7Gk_|S*61k}yX`|wWe7IZuTYT3kFn7|S6gxCff57AUcUF; zrF4&(iO{DW7%uQ`Pw_WrD79MJuZsT&|3sJbcGwG3aMxA99bb9IeXTO}uRkoyr-7ZRkw;yw$}70?&W5IdcGO%*3YIWgT(G1OzAacp zuyMiWgsPgCI|aKU*n(geMOf7zbP0Bg{)B(3xV9w1Ot2ojF4%R!VpiXM!KTg0uwcF1 zAN2EoR2KPKl~W4;WA$x*==DwOC34f(1<&!uy(I?%Wm=tic?gjDy<0*Z5bUa8{es<; zUrfrcbDZ_WGVa)uWAefj{i@x9;Se*zlY(91XSZ%BO`Sg1a-yY8`FQK!l&$Tdjw8yc zV-k3Biw7Je;1hv>cH9_2HVN~tD%4(~B8j-3c!($2^lCPf5iD)aeuv9U2E27 zVgM8jZ{+2RfD!*Rf=8NGXBvzO20)Z_Rqwa+YFXdM$rHE@$IH<$%Z6?ar9~IY^++ zx4S7yV;MCaSyPrC8H!sIDsIUmEq1cyh;sB~Uf2H%dQy|MJ`mWUHR!WwkQI_^-zfEP z3NofVNMz{YFf*>Cfo%_WyJ`nLq^*ngRHxoY2f?EJOzn$qncbwTE4`$DPf1q=c53@% zC*VWSO2qLZp&~`;7T|mp5Az>|pR_9AjRq;tw3?2z)V-}2p3?!K|thn@$eD$0r%`;cbzu(~{vl+>h zHkWD5#tPZ8@dbnlNs_Zj`-L7=Ny-$!^4*!eKLlM74CxnE9LMW=KV@L5S86u%H-km%I!<)7gpm4o zGG#(6BgDfM+F|SAU^at#=4T-a3r^?EyM0C!&*6|`Feh}7-qmR=feJjqI9%gT&S=C~ zphHx@#`G&-hF;{#&R2Zx$4)4&aTN1c_imEYAjp5I-q;w_8cmRgY2>!!m}?zK8jZm& zWz%6jgF2EYWHUq5Jg5(+fnm;|GB|)9T!4FEszszFx9xeluvIyiXZrj;kLhB{yLP)) zYfMbq6NnLRhm5bTRo7#KBJch7F!YAj1uHbPC04`1qC!P_@f`o*O{bfPUozv;zgB9^ z_GQ#2Va~$`;${lrN9^!*NuTT>!l%Jc!<5e0L{K>jhvqC13M*H^yAi>Y={0w9v!L+g znpC&dc&U;pmDX3mlagR6Vctv`iMvGf&{A&8i69VjY*o63BP4)hx~kwEm#Ob*Q$VDU z!~zeLJue4A`!To&4T54m@sC zbYCZbX|A-ZOJj2vO>AE0o#MsnJ(TAOrGAH2Z)3Z_Xc%>O337DGc!h$c4GzBHdbzxr zl+_}mrIu5Hi`WQV+E}Y1Z7j+Q4|z}-kR}tRHGXeL7`z_=z!~BQ@{VEej5yW l>yP~Z2UG-eKmp=<2K6NxWZ^)cFLe4N+IM*BK*K-%zX7(vMZo|7 -- 2.47.3 From 20863615b35c9fdbb7adf2b7e71eb7e091f220ec Mon Sep 17 00:00:00 2001 From: Weksa Date: Thu, 12 Nov 2026 13:05:16 +0000 Subject: [PATCH 3/3] data/original: ja passages replaced, none a rendering of an en-US original or a Dao line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All 28 Train/Validation ja passages are new Japanese sentences on the inventory's concepts with their own imagery (paddies, charcoal, kites, the bath, the market); none renders an en-US original, and none restates a data/daodejing line (ch3, 5, 7, 8, 78) or other canon (the old ones on 'true words', 'long-enduring', 'bellows', 'water vs hard things' are gone). corpus.cc restored from 21eaa07 and rebuilt through weksa corpus add: numbering 1..28 unbroken, holdout-ja:1..8 and original:1..28 byte-identical. The concept test now matches Japanese by multi-character surface and English by word stem, strips false friends (天気, 心配, 腹が立つ, 水曜 ...), and has a negative test. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/original_corpus.rs | 103 +++++++++++++++++++------- data/original/corpus.cc | Bin 28446 -> 28720 bytes 2 files changed, 77 insertions(+), 26 deletions(-) diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 47ed2d6..9213939 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -750,37 +750,93 @@ fn the_eight_ja_test_sentences_are_pinned_verbatim() { } /// The concepts the pinned inventory holds, as each language names them in the -/// passages (a surface that appears in a passage). Every passage of both -/// languages rests on at least one live concept, so the two panel legs are -/// measured on the same inventory. +/// passages: English by word stem, Japanese by a multi-character surface. This +/// is a tripwire, not the encoder: it fails a passage that names no live +/// concept, and `FALSE_FRIENDS` strips the idioms that hold a concept's +/// character without its meaning (天気, 心配, 腹が立つ ...) before matching. const LEXICON: [(&str, &[&str], &[&str]); 21] = [ - ("endanger", &["endanger"], &["危う", "危な"]), - ("prevent", &["prevent"], &["防げ", "妨げ"]), - ("control", &["control"], &["操", "支配"]), + ("endanger", &["endanger"], &["危う", "危ない"]), + ("prevent", &["prevent"], &["防い", "防げ", "妨げ"]), + ("control", &["control"], &["操ろ", "操る", "操れ", "支配"]), ( "request", - &["please ask", "request"], - &["くれませんか", "いただけませんか", "お願い"], + &["request"], + &["いただけませんか", "もらえますか", "くれませんか", "お願い"], ), ("sage", &["sage"], &["賢者"]), - ("mind", &["mind"], &["心"]), - ("belly", &["belly"], &["腹"]), + ("mind", &["mind"], &["心が", "心は", "心を", "心の", "心も"]), + ( + "belly", + &["belly", "bellies"], + &["お腹", "空腹", "満腹", "腹の虫"], + ), ("will", &["will"], &["意志"]), - ("bone", &["bone"], &["骨"]), - ("empty", &["empt"], &["空にし", "空の器", "空なのに"]), + ("bone", &["bone"], &["骨が", "骨を", "骨は", "骨の"]), + ("empty", &["empt"], &["空っぽ", "空にし", "空の器"]), ("fill", &["fill"], &["満た", "いっぱい"]), - ("weaken", &["weaken"], &["弱ら", "弱っ"]), - ("strengthen", &["strengthen"], &["強く", "強い"]), - ("heaven", &["heaven"], &["天"]), - ("earth", &["earth"], &["大地", "地は", "天と地"]), - ("water", &["water"], &["水"]), + ("weaken", &["weaken"], &["弱ら", "弱め", "弱っ"]), + ("strengthen", &["strengthen"], &["強くな", "強め", "強くし"]), + ( + "heaven", + &["heaven"], + &["天が", "天を", "天は", "天上", "天の"], + ), + ("earth", &["earth"], &["大地"]), + ("water", &["water"], &["水が", "水を", "水は", "水面"]), ("compare", &["compare"], &["比べ"]), - ("resemble", &["resemble"], &["似"]), + ("resemble", &["resemble"], &["似て", "似た", "似る"]), ("continues_long", &[], &["長く続", "久しく続"]), ("true_words", &[], &["誠の言葉", "真実の言葉"]), ("seem_paradoxical", &[], &["逆さま"]), ]; +/// Compounds and idioms that contain a surface above without naming its concept. +const FALSE_FRIENDS: [&str; 8] = [ + "天気", + "心配", + "腹が立", + "腹を立", + "水曜", + "骨が折れ", + "天の川", + "天井", +]; + +/// The live concepts a passage names, by `LEXICON`, in `lang_col` (0 en-US, 1 ja-JP). +fn concepts_in(text: &str, col: usize) -> Vec<&'static str> { + let mut text = text.to_lowercase(); + for f in FALSE_FRIENDS { + text = text.replace(f, ""); + } + let words: Vec<&str> = text.split(|c: char| !c.is_alphabetic()).collect(); + LEXICON + .iter() + .filter(|(_, en, ja)| { + if col == 0 { + en.iter().any(|s| words.iter().any(|w| w.starts_with(s))) + } else { + ja.iter().any(|s| text.contains(s)) + } + }) + .map(|(c, _, _)| *c) + .collect() +} + +#[test] +fn the_concept_lexicon_does_not_match_a_word_that_merely_holds_a_character() { + for ja in [ + "今日の天気はどうだろう。", + "試験が心配でならない。", + "腹が立って仕方がない。", + "水曜に会いましょう。", + "骨が折れる仕事だ。", + ] { + assert!(concepts_in(ja, 1).is_empty(), "{ja}"); + } + assert!(concepts_in("I attempt it, with a reminder.", 0).is_empty()); + assert_eq!(concepts_in("水が澄んでいる。", 1), ["water"]); +} + #[test] fn every_original_passage_rests_on_a_live_inventory_concept() { use weksa::inventory::Family; @@ -801,20 +857,15 @@ fn every_original_passage_rests_on_a_live_inventory_concept() { assert!(live.contains(&concept), "{concept} is not a live concept"); } let (corpus, _) = stores(); - for (lang, col) in [("en-US", 0), ("ja-JP", 1)] { + for (col, lang) in PANEL.iter().enumerate() { let mut hit = std::collections::BTreeSet::new(); let mut passages = 0; for item in corpus.all().unwrap() { - if item.language != lang || !matches!(item.source, Source::ModelWritten { .. }) { + if item.language != *lang || !matches!(item.source, Source::ModelWritten { .. }) { continue; } passages += 1; - let text = item.text.to_lowercase(); - let found: Vec<&str> = LEXICON - .iter() - .filter(|(_, en, ja)| [en, ja][col].iter().any(|s| text.contains(s))) - .map(|(c, _, _)| *c) - .collect(); + let found = concepts_in(&item.text, col); assert!(!found.is_empty(), "{} rests on no live concept", item.id); hit.extend(found); } diff --git a/data/original/corpus.cc b/data/original/corpus.cc index 4e28fcae918a2755c39ce726e937f36c62623870..9c53141206b52a394c56f05303a5e2b7230c6d54 100644 GIT binary patch delta 3391 zcmZWsTTEQn6&1#IoHmfec4XQrGFF{dDaB?6%)_C?!p5D(F$on?B}cAuoj5ign`u1e z!ylFAjv3zWV2-hQhSx9yGvUs_;CR4b6shT766K>(uas6*@0|PSN7cti)xWNN?hM?f zKWOgj?7i1sYwgWPe=11*xgfQx;F-#k6;&rItJBXGEbrRB__Xj}I~Py({&wf0>scj# zQIme{xxegO9DRQKfSUBzFRt!9d3f;G2fuB{sk-o4!4KJ|KRwXwQh%YZC9|16O&-vK z_^TKhnaiu19Mb|a-@1eQJatc#A8PWLCP&OCo0=TfwGPJH@3!ul01Xua1=- zNc(QKU1@8Vu9shv%5Qf_SEQS)#dDv%?mobpe^ly7;Sg(SIB?cgp*C6LaWjSk5lGQf zh(2IG8RDTHEjX0jnAGGhYbtH0WDLi%>+71F)JNv=)R>wVnM|<{%MY=y&lW$^;ca{0 z=e^91)X%fYvLlXgXDHk?mrK3LGeJI$D2g=}vX3>>vt|IllYDIj#p+XAI9!waHF-q9 zf_i~ZFEFtJW8w}T^=rX?N`^jE%umuh{D2+>hxFl;Cik!pUoBDV34$laV?YcjQ82!b@$ML%6~@gWMAGC1glGRObF))bMSE{aUSD~Y zIP&zdBRS5TU|$_C@l>iF0E&!_#k*QCm}?b&1Y}LdrNWo{_31TKYmG1H8(}+XYXp^} zoiXZYa6pr%m{fOk?`^65axTu|@qFqhK%nE4`a7;F^@1k%nIRm9Z$`xtQ-I2nr}Yqu z85hJ!mjw)&uPl%hUj4=z1Egp^g#0tDrW@-svz1Xid&) zazY=O2JSqvh}VPyQbYkcY)1QqVunPAzpO6WeM{=F8Q0==rgV%_`lYK{y~raI)*N6# zBRjRAtOb!$7j=|Ecj=vBzJ)ZBl-pWTPvN&>B+&9mHnV}RcrVIHh~@zq0M*A=k(Fo? zi2n`EN&c%oFDtE&vUsI4oia-2=dK#{qAm~OYaW7xfc;a^w*dq)GndIDEp$;3FHf?@L*kq+Gxx_(33z=5pwC&ACR0WL=1o` z`a=*Lt$8Z?n-GPBvl|=Rg!TI_5r9$VzxZ;oyVYgS1!~pv6rs_jHVX3>z))T!NRi;8 zCZqoekuQXrG6yjcW{gS58*>3gKr_gzD-YW)LB$vgyymRq2t_&PR$KGbSNO)VVD^x? z^080|FoS^ms53;pF?|BQsPN1kK0ZezrF@_$n~Ym*rlFLHF8K+!-{pLtD%wY}Dk{}F zzP3uJlS>06a#QLUTyPS$g*ypj6*iQcZtS+Jl!0u!Z~6RJa|3KYd!=E=YX4SW@^+Ql zVl1u;cmRW9_AMLilUyg_%zTPRph8mEOW62Lr}co0CEx5QagHnnZeWX${i?i#q856k zOYGeXFSEZlBpjt2rc#Yv$zDXqs&$DU9yV88y)$rU_U$7=oIl$s?k%Uwpgy4aepMk?vL?FoL%65Bcpt_k#)lXjQ8sznNk)Ncb1hIEevFBTGp(@+ zzBGssMw;e1ag*9K4$8n5^-&-UrX!^p>Lyl+;D9l%(3|UbXj)`*aLNMiP6rU{Q zuV?b{X%#z6t5|EL+M2UpLdv!#P#kVZ&S+44#k#YA*^LDAg`)ovJ%dpI9>K|D`Z{!o zuP`F=!+{lhzqJRe*iD}Z-E{W6 z83C9DcL4PgSPg0aBlhm`OKl%?Alzq;_3JpQeCeHcc_hXs@cEFLL77{^W})z?zJ$=Y zL$ajFjOcT?BhF9%rfF zA(R`d)pN!?#ty*3adB&d$cqs#20BeRzKln?JoHHvrkHW03q%I8TWI%)@K7r9mPAvC z+IjK~XP_4-P*aWClwT^b#s!U86qztrh?(2=lg;!B@h0s>TTjp0Ltl^$J3yx}(Omzo z*s&JQ?Mxr(jYF-q>U$6hzZcYJU_1$|55QJ9H7py#LwBGR97anSu+^s-UNC0iOu&|q z&w)HgO`+cy=%2fPKq}q+K}U`?h3B0W)>4Jn>(uK)B!YcF6aOqgt^v3-VxpN)4xmiv z_mfr#0?v)oZ5f-gVq!&xU8OY$ahfdYR}r~X zQ9ck*5V;5yW?QW~EMAjsf0nTLV?R!^m~E%$w9S4kOZIQcp7%Q~J!bqNd_Cv9@B2LO z^FHqv`pt0jwIMXkFw0`g&$s29!*dMxrcEARPX9Z77@zrl>R<`i&k3Xtg>V0nK4j0I z43HcCb-@Giq`;PA33n{|X3<+BPoDMg4~7+d)!M~dcB`QWiJpF$`4T-{GJ7U_o9Qnb zmOU&U?vHye$Sjx`$D;w64aLTo!XC(MREZ4Z7rx47T=K|W?WlCP>?ciM*-cfBbG1%; zrM-$9wh!^o*R9qP=I0>c0=~L%q0O8l?TBA^EVDqoYn0N45wAsNy)p|Uus&`4;)v2v zk2jU)=VfmzcjXxQi;~P4Rkf#2i1@(|_3;f9e~H;F<;1VmsiE%>E&8}a0P^#0lZD?} zn>n-gbZw=(T6}FW>A(JfzW&r~kq$*8KA8<7ZHxtxRt>hv4D}AlUN5SX*&Uhn$=*iP zu6AA);p=1u)`Dtx4`o;eT};H0Mg6 zDK`hyTSy#?`4BV^efChfa2o|BI`7M@mC{gFv<1}pP+6>fBpQj}eVO%81>RPr74@Ow z5$;)QdzO;in z%a8~R))T)hK=pf=>bWh@L3iU|Ji%@Wk&GogcpunBNvwsKC0sL{Zl{}nvcH`hO$+&i zMW1XrCEH>y0JaVU^{LkbZ(sm`J|DH3jjHz_Q^O-3!6{fknkE#-SL}bEl$T+59e2C< z@1`MMn3tw*W#Q6o=1o#@d<0bn;(ky3D%3$R1U6zS;en3Z#bk}f*-e>onKclnNzAuW zNHVJIPInD2+utUj+|JiS$sj0Z^Jb|a(GGdH#2#HIT6DaL!a*kSqYQ)qR|&Q7S4AtS z{OK-xW$X4~{#SusaC0?&tWr^&3dYC1O60CGMw(L_(M|#qeGE?!kGi3@q%71ii~?S$ z6U7(XkibzRT7Ko@CSJPPcsT7gb8p;t)Kpa2bW>onNJVN01B!kQ%Iqpt?O}=TZd4s_ zMyu*b{!sCgdt%&f2>PH1u+}J2z9^YD{e+_`$yOF0vg(0l1K6TNInp7u6Tt$YEO7%; z3&z?ZP+e4VysKhl?@;~-xC>~5C(t!$)~7rn#>H@pffnNgLSc0@PDi&k^M7nY*OE{XF2WX5dR#1RYH=> z;saas;f)l&ELSR12GQ(bVziT3@Gzwt;tDF40iX`XkTxC=eGO2Dp*x}Ykr*rybO(|= z()&qlewBdKC` z@AiX2K2c;8ZeLU3KJHF8Xzw=tG%X@%R+vz9{F%!kpj;USR1E zNxoj6IonxVZ9i6l{bH~VasPIGigyKj1yY&J9*Xj@N&?ZbyQD%=2B07&Ej^8;_(O;c zXu1;pP>e8gI#S%j7(&A0Oas>@Q&$UXi^f9j&#pk5Y?Ad7FS+^ru!qJaXJv8%`h0Cm) zrD731s01B+^4oE0DQII}Hu1V))!~C9U-OLkiSA{7TZp-=TF3U;nhC~;Ml~Z_0 zu5>^s)GveyXxJry14D*lw2cCxXu5&%A{&&8qhE=f$D5)XFlvDHrsO$cnm41uRZdrp zHdK(smzNlirmI*uMHP9{4&0}ZQ0;9Ka|h9&ie!&9j$>Gqv?NIh#m zgOPOFli0KPncYT5I`f(-GUrQ&#LWbKZC2{P1roIfmxa`#ra?hR16s4i#;8|9aGZ`* z7jS6-DJY?fPi)gJ3oiFL3{MJubWciO)PoE@fA8W=R?QWnPumev8G>N6dkHXiK-Ajr zY7_1dfHx>C4nnHKTMP;EKLa~hllnpE#D-S$46i=a1Jk}z!dUJKqEkC zlL0BBUla}Y6Ke+)3>FXsL@TxBi!uf&f~H}bTF`^LD|K8fkYd~XY1L_(8}}K@(=<;d zv&~Xj>Y4?}oG?b()rK+TrN+}LLsIB!(yb=djo;B`Y-=C?DQ^wGT$nk{aU7=pqpq{d zr|7IzDpdxDHNO?tB4{)T`;DV;2|*C88L{(7`thcx5QB0tl9qKQx9;f*e{t{^<9}Zo Bl%@ax -- 2.47.3