From e30ee3886a268ab04f2cc6729615212f203ac4fb Mon Sep 17 00:00:00 2001 From: Weksa Date: Thu, 12 Nov 2026 12:57:45 +0000 Subject: [PATCH] data/original: ja passages rewritten onto the Dao r3 concepts; ja Test sentences pinned verbatim The 28 Train/Validation ja passages are new Japanese sentences on the concepts the pinned inventory holds (water, mind, belly, bone, sage, will, heaven, earth, fill, empty, weaken, strengthen, compare, resemble ...), written in Japanese idiom, not translated. The branch was unmerged, so corpus.cc was restored to 21eaa07 and rebuilt through weksa corpus add; holdout-ja:1..8 and original:1..28 come out byte-identical. Tests: the eight ja Test texts, groups, speakers, split and edition URLs are literal expectations; every passage of both languages rests on a live inventory concept (en 28/28, 18 concepts; ja 28/28, 21 concepts). Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/original_corpus.rs | 160 ++++++++++++++++++++++++++ data/original/corpus.cc | Bin 28749 -> 28446 bytes 2 files changed, 160 insertions(+) diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 1c8b391..47ed2d6 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -666,3 +666,163 @@ fn a_panel_round_reads_both_languages() { } } } + +/// The eight ja-JP Test sentences, byte for byte: each is the Aozora Bunko +/// card file's text with only the ruby (《》, |) removed and NFC applied, a whole +/// sentence, checked against the card's XHTML on 2026-11-12. A change to one +/// is a new item, never an edit of these. +#[test] +fn the_eight_ja_test_sentences_are_pinned_verbatim() { + let want = [ + ( + "item:holdout-ja:1", + "親譲りの無鉄砲で小供の時から損ばかりしている。", + "holdout-ja-bocchan-6", + "entity:pd-soseki", + "https://www.aozora.gr.jp/cards/000148/files/752_14964.html", + ), + ( + "item:holdout-ja:2", + "尾張の停車場で他の乗組員は言合せたように、残らず下りたので、函の中にはただ上人と私と二人になった。", + "holdout-ja-koya-hijiri-4", + "entity:pd-kyoka", + "https://www.aozora.gr.jp/cards/000050/files/521_20583.html", + ), + ( + "item:holdout-ja:3", + "高瀬舟に乗る罪人の過半は、いわゆる心得違いのために、思わぬ科を犯した人であった。", + "holdout-ja-takasebune-14", + "entity:pd-ogai", + "https://www.aozora.gr.jp/cards/000129/files/45245_22007.html", + ), + ( + "item:holdout-ja:4", + "ジムは僕より身長が高いくせに、絵はずっと下手でした。", + "holdout-ja-hitofusa-no-budo-4", + "entity:pd-arishima", + "https://www.aozora.gr.jp/cards/000025/files/211_20472.html", + ), + ( + "item:holdout-ja:5", + "風がどうと吹いてきて、草はざわざわ、木の葉はかさかさ、木はごとんごとんと鳴りました。", + "holdout-ja-chumon-6", + "entity:pd-kenji", + "https://www.aozora.gr.jp/cards/000081/files/43754_17659.html", + ), + ( + "item:holdout-ja:6", + "えたいの知れない不吉な塊が私の心を始終圧えつけていた。", + "holdout-ja-lemon-3", + "entity:pd-kajii", + "https://www.aozora.gr.jp/cards/000074/files/424_19826.html", + ), + ( + "item:holdout-ja:7", + "多摩川の二子の渡しをわたって少しばかり行くと溝口という宿場がある。", + "holdout-ja-wasureenu-7", + "entity:pd-kunikida", + "https://www.aozora.gr.jp/cards/000038/files/1409_34798.html", + ), + ( + "item:holdout-ja:8", + "一人の下人が、羅生門の下で雨やみを待っていた。", + "holdout-ja-rashomon-1", + "entity:pd-akutagawa", + "https://www.aozora.gr.jp/cards/000879/files/127_15260.html", + ), + ]; + let (corpus, _) = stores(); + for (id, text, group, speaker, url) in want { + let item = corpus.get(id).unwrap().unwrap(); + assert_eq!(item.text, text, "{id}"); + assert_eq!( + (item.group.as_str(), item.speaker.as_str()), + (group, speaker), + "{id}" + ); + assert_eq!(item.language, "ja-JP", "{id}"); + assert_eq!(SplitScheme::V1.live(&item), Some(Split::Test), "{id}"); + let Source::PublicDomain { edition_url, .. } = &item.source else { + panic!("{id}: not public-domain"); + }; + assert_eq!(edition_url, url, "{id}"); + } +} + +/// The concepts the pinned inventory holds, as each language names them in the +/// passages (a surface that appears in a passage). Every passage of both +/// languages rests on at least one live concept, so the two panel legs are +/// measured on the same inventory. +const LEXICON: [(&str, &[&str], &[&str]); 21] = [ + ("endanger", &["endanger"], &["危う", "危な"]), + ("prevent", &["prevent"], &["防げ", "妨げ"]), + ("control", &["control"], &["操", "支配"]), + ( + "request", + &["please ask", "request"], + &["くれませんか", "いただけませんか", "お願い"], + ), + ("sage", &["sage"], &["賢者"]), + ("mind", &["mind"], &["心"]), + ("belly", &["belly"], &["腹"]), + ("will", &["will"], &["意志"]), + ("bone", &["bone"], &["骨"]), + ("empty", &["empt"], &["空にし", "空の器", "空なのに"]), + ("fill", &["fill"], &["満た", "いっぱい"]), + ("weaken", &["weaken"], &["弱ら", "弱っ"]), + ("strengthen", &["strengthen"], &["強く", "強い"]), + ("heaven", &["heaven"], &["天"]), + ("earth", &["earth"], &["大地", "地は", "天と地"]), + ("water", &["water"], &["水"]), + ("compare", &["compare"], &["比べ"]), + ("resemble", &["resemble"], &["似"]), + ("continues_long", &[], &["長く続", "久しく続"]), + ("true_words", &[], &["誠の言葉", "真実の言葉"]), + ("seem_paradoxical", &[], &["逆さま"]), +]; + +#[test] +fn every_original_passage_rests_on_a_live_inventory_concept() { + use weksa::inventory::Family; + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/original"); + let tmp = tempfile::tempdir().unwrap(); + std::fs::copy(dir.join("inventory.cc"), tmp.path().join("inventory.cc")).unwrap(); + let inventory = InventoryStore::open(&tmp.path().join("inventory.cc")) + .unwrap() + .inventory() + .unwrap(); + let rev = inventory.latest().unwrap().n; + let live: Vec<&str> = inventory + .entries() + .filter(|e| e.family() == Family::Concept && e.live_at(rev)) + .map(|e| e.gloss.as_str()) + .collect(); + for (concept, _, _) in LEXICON { + assert!(live.contains(&concept), "{concept} is not a live concept"); + } + let (corpus, _) = stores(); + for (lang, col) in [("en-US", 0), ("ja-JP", 1)] { + let mut hit = std::collections::BTreeSet::new(); + let mut passages = 0; + for item in corpus.all().unwrap() { + if item.language != lang || !matches!(item.source, Source::ModelWritten { .. }) { + continue; + } + passages += 1; + let text = item.text.to_lowercase(); + let found: Vec<&str> = LEXICON + .iter() + .filter(|(_, en, ja)| [en, ja][col].iter().any(|s| text.contains(s))) + .map(|(c, _, _)| *c) + .collect(); + assert!(!found.is_empty(), "{} rests on no live concept", item.id); + hit.extend(found); + } + eprintln!( + "{lang}: {passages} passages on a live concept, {} distinct concepts", + hit.len() + ); + assert_eq!(passages, 28); + assert!(hit.len() >= 12, "{lang}: {} distinct concepts", hit.len()); + } +} diff --git a/data/original/corpus.cc b/data/original/corpus.cc index 108afaa2bd95cb74f2aab9d17d6715b9d807bbc4..4e28fcae918a2755c39ce726e937f36c62623870 100644 GIT binary patch delta 3371 zcmZu!TX0iV6rBP^%ZLb+$KWtv%8Y|fY5JrTuz*q!l!r3Dl?hNwj7_jjN8yK@P)Z-c zP)^gdw52@y2rW$l0W2vG`Opz(oZ-ihJHzmpn|pKf<&XY3>e}b#HYw((lgG{3YpuQZ z+NZxg$vXQeD?B4ByR@{Tv@nvB6`7WNd2jk(#>vn2RF*+@rZUIJacYCqHGq%$KcPP_b1FUrY2|keMgZ*DbRmowdelw|t%dl_kH{dZfW-sj=|6 zMFGD5HPbAjU`DCAJ^X3G{7SR5NjrX1WN^GK=8I^n%)n5v92aPbznh z$?ntf%k8MoU#VEaAC$j1t)ZSjDIez1*YhSTWQhvqLdmRkHmczta9-@DUyunX?YD7; za`KAIa3l))1;cLH-OQPJ5&yMh_RNO*h8jm5e_B4o^Yc^3zABD=zew5>i+W@>07P*X z!f9%#LuQ~dB)i=JF0)HAJ1@IiKuhiVUes-r8KexUy?sQEjmU1C5EiuBWH)!27caP; zuAb9T>!?3y-6ueJRy_-1d$IJPa`vKn4pl>O4^9lm?p#-n4+2f1>#EF7Qw=JKMnO^! zV8wkSv1k;}%dC&eLxs}-$M~wv=4_kAX0`KsRULfSYxDV^1*vu|Cf2!cOWV~(uW}b^ zg%TqHQuNj>V$2$epUbVbc8NpVl$Mi%Vu)He0dlTHe+0$iqd)_Nh%0~m!K`d}1R@(H zcNUt2v(5a@hDC1{mP(tb+YFT;T@wk*f`r}0RM((j1{Y31>I6F{`Z%6&;d$^Dn3z{| z^QHB(XWAS#N1cOTSv$zbi&8=^5#8Ke24a33=uyuErw{=8^B(Fi8&$8~q%KEYLQ6=1 z60HIlUcf@<_`h40!i=-CE%pNryC}7+BvopTC{?yvs?ttrk4N> z5lI*2Vi;7Fu-8C5!%ibgPJdLEI&+o)yl#!OP9uc)alaBBQ^v_>Y76>E$YVE=4C1R^ zI4tQ0bsXNu1C5Y>#dN#Xi3Vee%1s{`Ni(bGs?}LtTao7WiXv&98peR4djm4Vcn{(H zL~k#!YHesvBN-ijy>&%QRZntcQOMmQ`b^qmTxv<4A>O>kbks+-uIE zu1$x&{BBhz#%dn_qayE91J_HLa1~0G@k7rNyQ8PDF zO=MSWqEqk+iWL`_++g%}a$k_gM5ZB3+~DY^g3k!sP-od!onIt`)i#=Hjl!l(h0CPf zGP^DaVj%@%<74D(G8D+75^YsoQ|&s83%=cn3osfoPNz?uj(5NfXn(q)zHG48IFcyd zds82-Lag$FOej`MpBt;fBL)O|qX_;Y;F1|GR)|OG0krgQfe23qlv8LI+@hl%rXB4* zA%3n;8UJV9ys0|D3x^%nB*L+`OrMcXgm6wKgymAD01#3_NSx&SCifQnnH*{i7^c28 zElq)B@}5A6e9&R9&M!u9IG?j^9?z*VS&jZI&5&}%(he~S0Wkqk0y_=tu@PWLeY2qr zx+#(TsEiT8+kg^1h9^ZMu_>U9J2?jwFwSZ_e_Y-v5Y%ijePJM&p8-LMv< z#DK$rK#;VC-i|o7HSMA1w6ui|Py=z_M6;heW%dtSzQyd@_{NPL0>gu?sqoFqfT2{{ zBQ8iVLz~ic8GO_}Tw9EL1I;3A5Y!za_E?h=M$-9>zToZzX#j?LAzT4pr)RH|pE5sB z;XA|bIO4$l(+6NBI}Ks7Gk_|S*61k}yX`|wWe7IZuTYT3kFn7|S6gxCff57AUcUF; zrF4&(iO{DW7%uQ`Pw_WrD79MJuZsT&|3sJbcGwG3aMxA99bb9IeXTO}uRkoyr-7ZRkw;yw$}70?&W5IdcGO%*3YIWgT(G1OzAacp zuyMiWgsPgCI|aKU*n(geMOf7zbP0Bg{)B(3xV9w1Ot2ojF4%R!VpiXM!KTg0uwcF1 zAN2EoR2KPKl~W4;WA$x*==DwOC34f(1<&!uy(I?%Wm=tic?gjDy<0*Z5bUa8{es<; zUrfrcbDZ_WGVa)uWAefj{i@x9;Se*zlY(91XSZ%BO`Sg1a-yY8`FQK!l&$Tdjw8yc zV-k3Biw7Je;1hv>cH9_2HVN~tD%4(~B8j-3c!($2^lCPf5iD)aeuv9U2E27 zVgM8jZ{+2RfD!*Rf=8NGXBvzO20)Z_Rqwa+YFXdM$rHE@$IH<$%Z6?ar9~IY^++ zx4S7yV;MCaSyPrC8H!sIDsIUmEq1cyh;sB~Uf2H%dQy|MJ`mWUHR!WwkQI_^-zfEP z3NofVNMz{YFf*>Cfo%_WyJ`nLq^*ngRHxoY2f?EJOzn$qncbwTE4`$DPf1q=c53@% zC*VWSO2qLZp&~`;7T|mp5Az>|pR_9AjRq;tw3?2z)V-}2p3?!K|thn@$eD$0r%`;cbzu(~{vl+>h zHkWD5#tPZ8@dbnlNs_Zj`-L7=Ny-$!^4*!eKLlM74CxnE9LMW=KV@L5S86u%H-km%I!<)7gpm4o zGG#(6BgDfM+F|SAU^at#=4T-a3r^?EyM0C!&*6|`Feh}7-qmR=feJjqI9%gT&S=C~ zphHx@#`G&-hF;{#&R2Zx$4)4&aTN1c_imEYAjp5I-q;w_8cmRgY2>!!m}?zK8jZm& zWz%6jgF2EYWHUq5Jg5(+fnm;|GB|)9T!4FEszszFx9xeluvIyiXZrj;kLhB{yLP)) zYfMbq6NnLRhm5bTRo7#KBJch7F!YAj1uHbPC04`1qC!P_@f`o*O{bfPUozv;zgB9^ z_GQ#2Va~$`;${lrN9^!*NuTT>!l%Jc!<5e0L{K>jhvqC13M*H^yAi>Y={0w9v!L+g znpC&dc&U;pmDX3mlagR6Vctv`iMvGf&{A&8i69VjY*o63BP4)hx~kwEm#Ob*Q$VDU z!~zeLJue4A`!To&4T54m@sC zbYCZbX|A-ZOJj2vO>AE0o#MsnJ(TAOrGAH2Z)3Z_Xc%>O337DGc!h$c4GzBHdbzxr zl+_}mrIu5Hi`WQV+E}Y1Z7j+Q4|z}-kR}tRHGXeL7`z_=z!~BQ@{VEej5yW l>yP~Z2UG-eKmp=<2K6NxWZ^)cFLe4N+IM*BK*K-%zX7(vMZo|7