From 4dbaef943763000a871d66990f6b85df73eeacf1 Mon Sep 17 00:00:00 2001 From: Weksa Date: Thu, 12 Nov 2026 12:47:18 +0000 Subject: [PATCH 01/57] data/original: ja-JP slice, 28 model-written Japanese passages and 8 Aozora Bunko Test sentences MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Train and Validation (14 each) are Japanese passages written in Japanese, ModelWritten Anthropic, generator claude-sonnet-5-5; none sits in a Test group. Test is eight one-sentence excerpts (Soseki, Kyoka, Akutagawa, Ogai, Arishima, Kenji, Kajii, Kunikida) taken from the card ruby zips (cp932, ruby and the pipe marker dropped, NFC) and checked verbatim against each card's XHTML file; licence strings keep 底本, 入力 and 校正 within the store's 200-byte cap. Entities appended (keeper gains 番人; eight pd-* authors). Items 1..28 and holdout:1..8 unchanged (digest pinned). Tests per language: counts, provenance, Test public-domain only, no markup, panel round reads both languages, speaker forms, recall 56, no Test text in any prompt. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/original_corpus.rs | 236 ++++++++++++++++++++------ data/original/corpus.cc | Bin 13160 -> 28749 bytes data/original/entity.cc | Bin 1378 -> 2600 bytes 3 files changed, 181 insertions(+), 55 deletions(-) diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 21f7a1d..1c8b391 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -1,7 +1,11 @@ -//! The original corpus (`data/original`): 28 passages written for the -//! non-memorised golf-loop target (Train and Validation, all model-written) -//! and 8 public-domain Test items, each with its full provenance. Test is only -//! read at a checkpoint: no round or recall prompt over this store holds it. +//! The original corpus (`data/original`), in en-US and ja-JP: per language, 28 +//! passages written for the non-memorised golf-loop target (Train and +//! Validation, all model-written) and 8 public-domain Test sentences, each with +//! its full provenance. Test is only read at a checkpoint: no round or recall +//! prompt over this store holds it. The ja-JP passages are written in Japanese, +//! not translated; the ja-JP Test sentences are Aozora Bunko excerpts. + +const PANEL: [&str; 2] = ["en-US", "ja-JP"]; mod common; @@ -28,15 +32,21 @@ fn stores() -> (CorpusStore, EntityStore) { fn every_original_item_records_its_author_and_the_public_domain_ones_their_provenance() { let (corpus, entities) = stores(); let items = corpus.all().unwrap(); - assert_eq!(items.len(), 36); - let (mut model, mut public) = (0, 0); + assert_eq!(items.len(), 72); + let (mut model, mut public) = ([0; 2], [0; 2]); for item in &items { + let lang = PANEL.iter().position(|l| *l == item.language).unwrap(); match &item.source { Source::ModelWritten { family, generator, .. } => { - model += 1; - assert!(item.id.starts_with("item:original:"), "{}", item.id); + model[lang] += 1; + assert!( + item.id + .starts_with(["item:original:", "item:original-ja:"][lang]), + "{}", + item.id + ); assert_eq!(*family, ModelFamily::Anthropic, "{}", item.id); let generator = generator.as_deref().unwrap_or_default(); assert!(generator.contains("claude-sonnet-5-5"), "{}", item.id); @@ -50,22 +60,42 @@ fn every_original_item_records_its_author_and_the_public_domain_ones_their_prove translator, edition_year, } => { - public += 1; - assert!(item.id.starts_with("item:holdout:"), "{}", item.id); - // The edition is the page the text was taken from. + public[lang] += 1; assert!( - edition_url.starts_with("https://www.gutenberg.org/cache/epub/") - && edition_url.ends_with(".txt"), + item.id + .starts_with(["item:holdout:", "item:holdout-ja:"][lang]), "{}", item.id ); assert!(licence.contains("Public domain"), "{}", item.id); - assert!( - licence.contains("trademark or licence matter"), - "{}", - item.id - ); assert!(jurisdiction.starts_with("United States"), "{}", item.id); + if lang == 0 { + // The edition is the page the text was taken from. + assert!( + edition_url.starts_with("https://www.gutenberg.org/cache/epub/") + && edition_url.ends_with(".txt"), + "{}", + item.id + ); + assert!( + licence.contains("trademark or licence matter"), + "{}", + item.id + ); + } else { + // The card file the text was taken from, its sources kept attached. + assert!( + edition_url.starts_with("https://www.aozora.gr.jp/cards/") + && edition_url.ends_with(".html"), + "{}", + item.id + ); + for kept in ["Aozora Bunko", "底本", "入力", "校正"] { + assert!(licence.contains(kept), "{} lacks {kept}", item.id); + } + assert!(jurisdiction.contains("Japan"), "{}", item.id); + assert!(translator.starts_with("none"), "{}", item.id); + } assert!( [work, author, translator] .iter() @@ -73,8 +103,10 @@ fn every_original_item_records_its_author_and_the_public_domain_ones_their_prove "{}", item.id ); - // Every edition and every author is out of copyright by date. - assert!((1600..=1900).contains(edition_year), "{}", item.id); + // Every edition and every author is out of copyright by date; + // a ja-JP edition year is the work's first appearance. + let years = [1600..=1900, 1868..=1930][lang].clone(); + assert!(years.contains(edition_year), "{}", item.id); } other => panic!("{}: not model-written or public-domain: {other:?}", item.id), } @@ -84,7 +116,7 @@ fn every_original_item_records_its_author_and_the_public_domain_ones_their_prove item.id ); } - assert_eq!((model, public), (28, 8)); + assert_eq!((model, public), ([28, 28], [8, 8])); } #[test] @@ -99,14 +131,30 @@ fn the_original_items_spread_over_train_and_validation_and_test_is_public_domain .filter(|i| SplitScheme::V1.live(i) == Some(s)) .collect::>() }; - assert_eq!(in_split(Split::Train).len(), 14); - assert_eq!(in_split(Split::Validation).len(), 14); + for lang in PANEL { + let of = |s| { + in_split(s) + .into_iter() + .filter(|i| i.language == lang) + .count() + }; + assert_eq!( + (of(Split::Train), of(Split::Validation), of(Split::Test)), + (14, 14, 8), + "{lang}" + ); + } let test = in_split(Split::Test); - assert_eq!(test.len(), 8); + assert_eq!(test.len(), 16); assert!( test.iter() .all(|i| matches!(i.source, Source::PublicDomain { .. })) ); + // No model-written passage was admitted into a Test group at all. + assert!(items.iter().all(|i| { + !matches!(i.source, Source::ModelWritten { .. }) + || SplitScheme::V1.of_bucket(SplitScheme::V1.bucket(&i.group)) != Split::Test + })); // Train and Validation hold no public-domain item. for s in [Split::Train, Split::Validation] { assert!( @@ -238,27 +286,27 @@ fn the_recall_probe_covers_every_live_original_item_and_names_no_work() { "bag", weksa::run::LowererReach::BriefedSubagent, ModelFamily::Unrecorded, - &["en-US".to_string()], + &PANEL.map(String::from), ) .unwrap(); let mut model = Unrelated(vec![]); let RecallOutcome::Read(reading) = run.recall(&mut model).unwrap() else { panic!("answered"); }; - let r = reading.languages[0].clone(); - assert_eq!((r.n, r.answered), (28, 28)); - assert!(r.chrf < 25.0, "{}", r.chrf); - assert_eq!(model.0.len(), 28); + assert_eq!(reading.languages.len(), 2); + for r in &reading.languages { + assert_eq!((r.n, r.answered), (28, 28), "{}", r.language); + assert!(r.chrf < 25.0, "{} {}", r.language, r.chrf); + } + assert_eq!(model.0.len(), 56); // No item outside Train and Validation is shown to the model. let all = CorpusStore::open(&c).unwrap().all().unwrap(); for item in &all { let live = SplitScheme::V1.live(item); // A recall prompt shows the first half of the item, cut at a word boundary. - let mut cut = item.text.len() / 4; - while !item.text.is_char_boundary(cut) { - cut -= 1; - } - let opening = item.text[..cut].trim_end().to_string(); + let quarter = item.text.chars().count() / 4; + let opening: String = item.text.chars().take(quarter).collect(); + let opening = opening.trim_end().to_string(); let shown = model.0.iter().any(|t| t.contains(&opening)); assert_eq!( shown, @@ -314,7 +362,7 @@ fn original_round(family: ModelFamily) -> (weksa::run::RunStores, weksa::run::Ro "bag", LowererReach::ToolLessRelay, family, - &["en-US".to_string()], + &PANEL.map(String::from), ) .unwrap(); let scorer = weksa::score::Scorer::new(Bag); @@ -328,14 +376,14 @@ fn original_round(family: ModelFamily) -> (weksa::run::RunStores, weksa::run::Ro fn own_family_counts_and_labels() { // The original corpus is Anthropic-written items, all offered. let (run, r) = original_round(ModelFamily::Anthropic); - assert_eq!(r.own_family(), 28); + assert_eq!(r.own_family(), 56); assert_eq!( r.parse_reasons().iter().map(|c| c.count).sum::(), - 28, + 56, "every item was refused at Parse and says why" ); let line = run.round_line(&r); - assert!(line.contains(" own-family 28"), "{line}"); + assert!(line.contains(" own-family 56"), "{line}"); // A different answerer family, or none recorded, labels nothing. for family in [ModelFamily::OpenAi, ModelFamily::Unrecorded] { @@ -471,7 +519,7 @@ fn no_test_item_is_offered_to_a_round_or_recall_or_recorded() { "bag", LowererReach::ToolLessRelay, ModelFamily::Unrecorded, - &["en-US".to_string()], + &PANEL.map(String::from), ) .unwrap(); let items = CorpusStore::open(&p("corpus.cc")).unwrap().all().unwrap(); @@ -479,28 +527,38 @@ fn no_test_item_is_offered_to_a_round_or_recall_or_recorded() { .iter() .filter(|i| SplitScheme::V1.live(i) == Some(Split::Test)) .collect(); - assert_eq!(held.len(), 8); - // A Test item's text, or any run of five of its words, in a prompt. + assert_eq!(held.len(), 16); + // A Test item's text, or any run of five of its words (twelve characters + // of a text written without spaces), in a prompt. let leaks = |prompts: &[String], item: &weksa::corpus::CorpusItem| { let w: Vec<&str> = item.text.split_whitespace().collect(); - prompts - .iter() - .any(|t| t.contains(&item.text) || w.windows(5).any(|k| t.contains(&k.join(" ")))) + let c: Vec = item.text.chars().collect(); + prompts.iter().any(|t| { + t.contains(&item.text) + || w.windows(5).any(|k| t.contains(&k.join(" "))) + || (w.len() == 1 + && c.windows(12) + .any(|k| t.contains(&k.iter().collect::()))) + }) }; let scorer = weksa::score::Scorer::new(Bag); let mut model = Record(vec![]); let RoundOutcome::Recorded(r) = round(&mut run, &mut model, &scorer).unwrap() else { panic!("answered"); }; - // The check bites: the round did show live items, 28 of them. - assert!(model.0.len() >= 28, "{}", model.0.len()); - assert!( - items - .iter() - .filter(|i| matches!(SplitScheme::V1.live(i), Some(Split::Train))) - .any(|i| leaks(&model.0, i)), - "the probe cannot see a shown Train item" - ); + // The check bites: the round did show live items, 56 of them, and the + // probe sees a shown Train item of each language. + assert!(model.0.len() >= 56, "{}", model.0.len()); + for lang in PANEL { + assert!( + items + .iter() + .filter(|i| i.language == lang) + .filter(|i| matches!(SplitScheme::V1.live(i), Some(Split::Train))) + .any(|i| leaks(&model.0, i)), + "the probe cannot see a shown {lang} Train item" + ); + } for item in &held { assert!(!leaks(&model.0, item), "round prompt held {}", item.id); assert!(r.calls().iter().all(|c| !c.items.contains(&item.id))); @@ -510,8 +568,9 @@ fn no_test_item_is_offered_to_a_round_or_recall_or_recorded() { let weksa::run::RecallOutcome::Read(reading) = run.recall(&mut recall).unwrap() else { panic!("answered"); }; - let rr = reading.languages[0].clone(); - assert_eq!((rr.n, rr.answered), (28, 28)); + for rr in &reading.languages { + assert_eq!((rr.n, rr.answered), (28, 28), "{}", rr.language); + } for item in &held { assert!(!leaks(&recall.0, item), "recall prompt held {}", item.id); } @@ -540,3 +599,70 @@ fn a_model_written_item_in_a_test_group_is_read_in_no_split() { let item = item_in("item:f:2", "x opens", &group_in_bucket(80), public); assert_eq!(SplitScheme::V1.live(&item), Some(Split::Test)); } + +/// An Aozora Bunko file carries ruby, 《》 and ｜, and ［＃…］ notes; the items +/// hold the base text only, in NFC. +#[test] +fn ja_test_text_carries_no_aozora_markup() { + use unicode_normalization::UnicodeNormalization; + let (corpus, _) = stores(); + let ja: Vec<_> = corpus + .all() + .unwrap() + .into_iter() + .filter(|i| i.language == "ja-JP") + .collect(); + assert_eq!(ja.len(), 36); + for item in &ja { + for bad in ["《", "》", "｜", "［＃", "※"] { + assert!(!item.text.contains(bad), "{} holds {bad}", item.id); + } + assert!(!item.text.starts_with('\u{3000}'), "{}", item.id); + assert_eq!( + item.text.nfc().collect::(), + item.text, + "{}", + item.id + ); + } +} + +/// A packet that cites an item's speaker lowers only if the speaker has a form +/// in the item's language. +#[test] +fn every_cited_speaker_has_a_form_in_its_items_language() { + let (corpus, entities) = stores(); + for item in corpus.all().unwrap() { + let speaker = entities.get(&item.speaker).unwrap().unwrap(); + assert!(speaker.forms_in(&item.language).is_some(), "{}", item.id); + } +} + +/// The panel [en-US, ja-JP] over `data/original`: both languages are read, each +/// with 14 Validation items, and no Test item is in any call. +#[test] +fn a_panel_round_reads_both_languages() { + let (run, r) = original_round(ModelFamily::Unrecorded); + assert_eq!(run.record().panel, PANEL); + let tags: Vec<&str> = r.languages().iter().map(|l| l.language.as_str()).collect(); + assert_eq!(tags, PANEL); + for l in r.languages() { + let refused = &l.validation_refused; + assert_eq!( + refused.parse + refused.admit + refused.lower, + 14, + "{}", + l.language + ); + } + let (corpus, _) = stores(); + for item in &corpus.all().unwrap() { + if SplitScheme::V1.live(item) == Some(Split::Test) { + assert!( + r.calls().iter().all(|c| !c.items.contains(&item.id)), + "{}", + item.id + ); + } + } +} diff --git a/data/original/corpus.cc b/data/original/corpus.cc index d778901e62568a8a52f6777b0c158717ccaf0df6..108afaa2bd95cb74f2aab9d17d6715b9d807bbc4 100644 GIT binary patch literal 28749 zcmdU2+jAS&eXi1J^w5WRh7l2`%Q zMRpg0>9nap-9=fV?vhDGcZm{3QWPbbqAV$G?Y{XB*qOd$z+C{H-lq?pzNFvpEbhAi zc2cPYXKWH&oLwxw{mys3-|zg+(W(0bsklBM3`E5)O;=R0Yq$Tzxw*6$(}b=8MNOu( zosuploJSrw+Tc`YGvkLpeMQ676T(&@A->Z8isQ6bRu>!mP*gQZk=@$h(rA_t6JfM@ zrT@>~|Em+uj~e@j-+1Hn9NleyREbBFl-{{Z=y$oN2fUB(?ezOQ{rq-6-yiVz`+9bK z{-vLeJ5M$?=5OA~Pv6gu9LZ*8a{I0n4)4odzMsw9GVdK8x*q|f?Nj_)1YF@}W6*e!9Zgp?jtR4&rR`TP969j|lMceI)_ z7tdG|cR9M_%<*jInRV_4{awm`BFB+{7hw|DpVm`P{A1 z)tq(tiZy+Wvo6hB_pe%q#tP&6Ugy+QczBJh=%U7{VnW$1Mz~ex@w0e7*5Nteoxge9 zT0Dz88WeTVF2YrwwXy35GT0H}w&?<$ziLoYHJwWm*cvtU_xXA|I6;nZZwX03uK${^ zb8%@&|Jd+h!gnD zbGe86%u{$ev-!zA=GdWZ228$`9l?tO+00aSPsW-dN@p`4FD>8&_;(2>;tF`nJouC@ zvM^08Gqdci%S#JKmKGPYnaSL(41O*x9%X-`!5zlkOXXfLTj>R{VI?LkRCR)wGAwjP zrCpLLbp{+A;Twgi^M!-Q@|m-4U7&~ecYI>aA1)lfl0P!Xt$OprR6^o5(?@tBzpuwv zBtNdSllS$6x^43RO%w8W_x(ELFW+Sg=KuU$b_8gSWi!uTI`e%Encv;_LYW_L#r$s1 z%s=L2ez)-fSd!awH-89>odVUNz`3V)7#n8_nPcWhAoDC}%*6S4cH{sWn9ui`&(0Mx zXXqgD9DM-|SV!1%F*^bgyPLapB0F+2cLaoowqIfwyp_#B{2|R1de2notqciLHM?~o zCTLfh{iJ#QmU(?DyZ1@?=F60hDT$<@X=NQZA?|adg^v%s zY${_wY9!)3|3az!m#wHA@J!_ooKy}NTeBkz*}a#+S99+fQs%goqf5RL(vGjBTKBUan!l4J&#H4xbWApeznX+sU z*T^kIUN&j@#`?>5zfjWdX+_!|Po(XIW4`MoZIAJ@!qrLeGn={1%*3QQ_LvbBZW#Xf zCWxB<2*v_Lg=+(Q*-sdb37iE&*>8cj&|>zp_!ioqhOvPEt$Hz8n18_B$+J3=SDOq` z-X+afF?=AJN+@ndo44jCVN!7$=5=(Ueb(6vxv>N0;yE{h2fBkj0eJ428MxE@{8j7z z6KJwDEPN=W1#UBas3&3eh5RL9rYr9B2YY+^Y?9A4A$fO0k~jJiWvZ8jxQgehgh)aY z(Dz?D(E|3$V=b-78FtH`A55EsCzdYrV4%cQ9z5*}@x2}lazg`Q z-!iNNc_?PV03)=+x-iQjPC*1=ew4|5GFj&BuCA1h{*@5QyhSiz9>10!or7q-Pr99`1(L^a~xh{GI#o3ZuG7-eVbdgx%hG11|^vUy4P}koFZ?Vrw-H@>n&eM+M;BMKAqhx%a?a!UQR5-hb_>6Q#=2l@GMgh!x zPLlgf?U*&^iDP0)Cxc#t-ovNTk6;fNjgPh+ef1)cV`Cy0ft^AQ2=wW!e zOOHpFo{XYl=|i}8hf}hmrDCEd4;%`XA~!!D-0AP(dwn)7|Go(=aSIKSa&`A*r{nw; zjy|48WS75l8Ms!N;Z`>`)TzHse3kQXGT#|=4WsplFx==eKs*=<= zq4-x)(K$)ukcy2{5((rvBSJY1T{+a*;~@Q}4doM7<4)HrxVZrx$xgZx z%H2~+b(=6iiP3e664U57-TFpB(^qr5l{m7+z7FGiYY|orCzKSLzazvYB(QM+4d@b_ zpZDVnt3pJgO9Q51PS6HaDU7S?3cKFAR0KELVFao*eJF+0D`p3Vx)gN?8Exc#*NDkf zcnuP>!8OSt`qia>pd07vdyPCG0TfP0@|JzFti=V)gm9a>xGlm5+!kOBNLi}e=R>c0 zM^$#=i*w&AeIh%ROKG^nw1m7kxG(7%`x2P;P-7e}hIGCt4~Ses(DArkveP~zRGXcq zG&IXM2YuYG6hh~S#Lxz6b+^EMmZvi z!fmuKi%P7?(@b|H+BfQ^+* zF)+JkGbkczx~kZtszMhn5~nJN4hsW{JX@j-`^&}Cy-K2CGly@O!*KHVNg*vh?;k|Shx*3vm(_$F{v)NZgp z4WGC1#81NFXRNqZH<}Cz;S{tPifKEP8gueJ5jIEUHVuc(e>=6bL#eoLQajfRT z_>JR?S8-wv{SSbNH*5qd+1aWJu!x|hCAl2i*p))gv1S2!L6>=(RT7bGZL@E2W_?sp zG2S4Kp$mCs4$p^(Ir!C5IbN!l^-c-<`xVEw)>gJ&JMOc`Tq?x!SP|&!=eu`&zQQ-| za<(y$yMxKdnH;Nrvi96ylbx7@zlqdX?%-ZFvqRHBNQ2kp9%*{$9@1*r4AN()F~Jn$ zWPS{_R9A8rk-9`F3TjDk9I3YvyI^p-7z@Wl5eDd9)tBguVIBm6vPFTFc~3>MN0d+9 zUMuTSB$A{7V`v~Qq#`2f3}so=J9|2NxK-~_Q788XxCvKVx>j9f8ke@Og?tn2Ie3S7 zu@iNdXioW!AtgT88CR4frwi~OddDv+-I1qM!@Lu1U+a`_5$>;!yPa;lkvoXF;G^az z*JyTvCWSF6fe8vgW!LdwMtH`Yd4x8Fix*e}luX#=Yj`nfp@IixZSJ7Xmw*5>I#Dng zMwROFz|})xYZ2Fg8?Jt1&^ost$YDN`rhF);JB5iH99dd;!gA{ix$B<-a+C<#Gx^|d z>8;t^{#$H%2D3qUI1J#-6hxIhB2-La`je`p8lYZ7WnBnM%LDSJsWZElxClAt203qh zKR+>HU7|WN^DuPdI2GuOoV2E|vT`%3jlhhX`52{|v$;Er*4L?EvoL;#p2JO0|9Y_{ z&AT;1cdnY#2`MJX%LCq~>7^QYSGmF4ZR}uknzPpVNmLx8=bK}Y-1F=pCfp&~OqNlb zf;pozc1d7;<8lqGbsK0;<)#?s?%Gkq;uxlnmIusD(>XOTf87mcl%?hFUM30`XJ}cO zJLxUJF4f_nRt=*ctIMY1G_aKv0W-#!Va5mv@}ecvYERH=%LLk{sg4?GUvq=D$9UgF z@f3}6zy@fQRuE8CHm2vNK-~P^1t>zPcnVW+D5?VRKoqZW9Jm7SQ8p6db?G7v#iEA9 zSTL|mq_}BbpayAIt>`s&Fy}UA&Om#i#y}CUVaOsEM=&3HT(~?9(I`AAJ_>6JCmZ5m*vOIKC%|r`$u%$!q6Bz(XhqXly9VZOxe+>OY|ovZ2Tu7({6zg8 zMl-wbkJGF%^TLI_lc);@cVVathv{ipk5Ke+u1a2xFvv^V;;tAqMlLEL%^O`NRBjry z)?nIecKeuW)%UTm6A& zf&mf-N4*#}7>Z>=-#*W=$5-6w+h=^2IWEA4(d(!ktPx0dbODuk0;#rZPAJ|ejDx-q zV~7JD2nB#{aa;{k5pt-4+K$7UGxYUtf4}Y$IP=EVLI$3876n?czIGW;{tpB9m3?vJOU0{R5V+P>!w&N(nMzKtnYHs0N#6$+~Go0-eMio=22?Xtz7T`W0Dvd_8nd&HG zIJGRsGv0g9>}DHyzp*2K27~@g{w$C=TDWqWa6Sl%+D?FAXJJ~=MVPl?+#m%Qx{#^> zswj3vCIV&ym6K!}TJgEgHpG_4kj8sciU3k<-EMT|je)`=(AgfzKwd1zqG61gUG)A* z>pW%Mtc&<*CxU?GBx^vK2$t|+#+Nc`I354N8DoS)^l^~oG8&C@_p3>r8cIppG9k9N z*qIRbxgp+d{5b#o^wJ{acea$$gbqU-Zss12S+{T;@Lj-&2H$J$-)r3i;c=ukuCp_o z$Oh4iTPWW)4}t&16WxZw8n|_F2-6bFLr1*D&;++zCWsFh-_Jo25J(i=?uB9m0*(`B z51`J~T!j9TMxbz?{2N5)6lIHYS0Oief!=@y2tts%Lq@~ZYSaveSbeB26L`JF(FE`B zxv{y&*l0~$EYKhT@BajX2 zD2`mIZR5(_nxPMKUCV^J-Xdl~+AW&ngNWv4(ZR8xXx^H6NcwOFt1l3&0p_E?J9ps; zi$Lt*4j3aNJVFaHXC9K7F6Jg6+z9;|jdC^6VkWa4!w=_F5brHeCTRc2O%)I*=YM{c zh>Om$Q;Es5R9hKPxa ztG{hIS?5I8n_VDuUvoh>VEhczQc(gcE3Q#({wTeHWnN8}dgmTAj=< zKtMr7duW4=uss$=u1Rxo+FStEv)BO0mIU(}p3SYoWz3Z#z0qj)zQ&57lrg+K0QF`Z z8Wl_QqFCY*GCWiH{cH&#Ggg>(wzE1=hr&hpiJ9DkF@TOhBbz~EfDREse1R7<;uXjR z1M8XG{tMJ>VzFd?G5R&iPSgx>R65#*afOq<-pm7`{W~u74I1n0wfE+AWGsOgOOye| zJtHNNn6(~$g2%y97B;zAMxr=_)F@+F17=M4upP6%PB6XM1p@PHE?|a?&Dco@SXt*m zKSGkSJ;db4L_(SuM)+0C^RSk)Ol2uOK~8rB@Zu3LkljbJ8mkb)`op#iex0~_Ppbc+ z3$A^}=KR%4QG<1ujX)Ii!jydsNid&&YTkehRA$~-r4@!c1Rzk6DRU8m%&6^-YYkg9 z6iBsU*PpGalWMuJFwpN0_WMFRKKtkK`ZEpRTJ9$)35z9A(ut}=4TZ}wtjf}<;MQI+ zfl5Syi=c)Kg^KA?*>Y`d*>pLor6x@Pd%s~}*)}mgSXEb+q>ax07v*GZ<=VYEn3|@^ zS$(IkyPx;(_{+D3=%pQBdV-7r$eDjVKQRwlZEQH>L7dwieYDue)V7VGVWL zV%0QBU5(X0IUWUN1mql_H9ii%heiOv?rzJ%<+pd`j4t`JqsHEYwh zAT^B(t3mn~2T0w9Z#zJ-i&trdVFERz07~J+L=uZL39+!CQJpl_*{iCOc15ZpO;PmU zwgswbh*u5NFC9PyjBgT9Di+DA*ws?QHm6kBjg?J;&ekSt!jRYko0t@DR^|8qE89(@ zvTAJp%YjXg@#;HqtN=vKy%uAXDe{S}@G9y#tw(mnr;;p)3z3MZwh4PqoT?pPe)|tj z*AG`s>MT%-fusa#ihxIuLCET>`lE!$pMnQ{<$0*)n@Z|{Hd;g@rsdzYcbBN8P{lPNdmqNW0Bws=(}<=TqyKhb z)MtF>y^^i8-L{4`RoHn08*&hCMYT#}YcsIvRCo2ZHcr%n+~N>3%^7G0AKscizO zL%Nz5{Z!M=VL(yv)A1%b1X}#g7DX3FLJiATFjC?j^f@eCg^hP*U8QegDLchNZ3~j; zh$dIxH&LzAK|CuN2JEhmO=*Z{Ow39voLH}QC zd1F;0!BPau-gbx`*(J~>4OmN$O>kTxs!}g()e>nFjOx{ERTwkbmN9AUkWwiysgYd&iS#R_298mN7PW8QN(^D9TU3m9wPvzfGv-RCUk6j#JfpjMYU$irh+) z_A6l9DTSFUWgAy8#TVouQGL~&Ulovakg;t+`t`#b)S|Go0^7NPi(m^UGVn=NMCQdd z@wN>cM7{wbHPQ=1uF6#fR8&Z}El{4roBwe1!=Mq^Xs=wS_z!}FCF_6+>eYX+ZUh-Hh?BR;LyhRkODNo^_ zz_F9?b~mLRIP8eYJPtHH7a+~+I4T#sCj@zxJyj#F|C+J9jMxKE{vE{Wquc1ePCNT*N$2N5EpJ&09y^DB>+Dw)Z^D{ zytWow9mzx3+r)Ml*wP^d+Qk*@96_*R6C^>#u4|y4D?k-5%g%3Ch&&IW|5guC(BO+v zHcf}wv33a=>YPmY?SWKKX_rCJ%{APvAccQqt?A;q|Jy`;os4-MKR^4|dZa>ze?9yv z;zZlz+ruQo2OE_50=4-}1{*h|KUw1-kaTL%LTwr)&&xN$DzV#khb2gIgHg#Mzi zQ8K4#x3)siKwD-)A*f|(M|t3%h_tuFPV!&uI-$ti;f=$^a=UaaabJvrAr}^V(@DiLtUXR`s1$Ni?i>L ztYArW9g|q(ix`T9$#GZc$De*pIbv`8_rZth{RUBpM<|YD$N4eqXFv5ZjE`Zb7&a8} zhnRus^J|(@@`tvAMBlfQsXaA{KqJhVwlfEP*CtZCmHT_;Lg6f#9mVq~m8;YHBpJ_Q z5en&pi{$Z$44qspO0y7QR+Rw&J`=hSyG=!gaynENim4U)Z3}3 zj1c%DAx2~r0rhXX6)*zB0#ed>ihxh3a277WHWaoA{nO`A^AZx0M%f3|)g9yYliKO; z<$u=7Z`RyQ#Y>gbTb1-?EwcnJUy**6WC#WmA};|B$`Qb@{XOX;icH2z6vsLN<1`PQ z&83@cll(=TzOYY=BlZ!>c&kkG?rm{=(dF+Cy8wgqucfa-L|@@ z*|tF|Mk~z}8K+@Ifr|=|WN=kVhzb&vUT_u9V*x-h0>+4MLi%OXs0fr7a3?rlX)P+7 zXLqO`#SL^aE&^-9$Lqrjq!}<7M;s*)1fW#p|K%7eUfCXo*1f+n=laRBC2yLUKPwxzjlu&@x=;Fpgpa!4d_u%LZaZtVVRQ6ec(1c54~$3unvb7X z(l0A(y1D8i)DXEi%URt{4g~$Z_15O7QCBEHNpz*Fd9JE9>+Y2*) zw{~xZe%m=jKz;g*_9=|y0Yn2b|A|o75xp0*PmN@4=a*__)hAF^d4@gYzK5Bd#~~+XTCkGZ06mm zi`~YP)l%|lDY;czzEN7%ysYRCvX1ShM#;Z1nCQR?j^1t;GTV%2Q;j$+Oup+-h#-+H eWAZqWxv=aRN~nuqeCFcS8p4VcMiGoe4*d^lIxP Date: Thu, 12 Nov 2026 12:57:45 +0000 Subject: [PATCH 02/57] data/original: ja passages rewritten onto the Dao r3 concepts; ja Test sentences pinned verbatim The 28 Train/Validation ja passages are new Japanese sentences on the concepts the pinned inventory holds (water, mind, belly, bone, sage, will, heaven, earth, fill, empty, weaken, strengthen, compare, resemble ...), written in Japanese idiom, not translated. The branch was unmerged, so corpus.cc was restored to 21eaa07 and rebuilt through weksa corpus add; holdout-ja:1..8 and original:1..28 come out byte-identical. Tests: the eight ja Test texts, groups, speakers, split and edition URLs are literal expectations; every passage of both languages rests on a live inventory concept (en 28/28, 18 concepts; ja 28/28, 21 concepts). Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/original_corpus.rs | 160 ++++++++++++++++++++++++++ data/original/corpus.cc | Bin 28749 -> 28446 bytes 2 files changed, 160 insertions(+) diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 1c8b391..47ed2d6 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -666,3 +666,163 @@ fn a_panel_round_reads_both_languages() { } } } + +/// The eight ja-JP Test sentences, byte for byte: each is the Aozora Bunko +/// card file's text with only the ruby (《》, ｜) removed and NFC applied, a whole +/// sentence, checked against the card's XHTML on 2026-11-12. A change to one +/// is a new item, never an edit of these. +#[test] +fn the_eight_ja_test_sentences_are_pinned_verbatim() { + let want = [ + ( + "item:holdout-ja:1", + "親譲りの無鉄砲で小供の時から損ばかりしている。", + "holdout-ja-bocchan-6", + "entity:pd-soseki", + "https://www.aozora.gr.jp/cards/000148/files/752_14964.html", + ), + ( + "item:holdout-ja:2", + "尾張の停車場で他の乗組員は言合せたように、残らず下りたので、函の中にはただ上人と私と二人になった。", + "holdout-ja-koya-hijiri-4", + "entity:pd-kyoka", + "https://www.aozora.gr.jp/cards/000050/files/521_20583.html", + ), + ( + "item:holdout-ja:3", + "高瀬舟に乗る罪人の過半は、いわゆる心得違いのために、思わぬ科を犯した人であった。", + "holdout-ja-takasebune-14", + "entity:pd-ogai", + "https://www.aozora.gr.jp/cards/000129/files/45245_22007.html", + ), + ( + "item:holdout-ja:4", + "ジムは僕より身長が高いくせに、絵はずっと下手でした。", + "holdout-ja-hitofusa-no-budo-4", + "entity:pd-arishima", + "https://www.aozora.gr.jp/cards/000025/files/211_20472.html", + ), + ( + "item:holdout-ja:5", + "風がどうと吹いてきて、草はざわざわ、木の葉はかさかさ、木はごとんごとんと鳴りました。", + "holdout-ja-chumon-6", + "entity:pd-kenji", + "https://www.aozora.gr.jp/cards/000081/files/43754_17659.html", + ), + ( + "item:holdout-ja:6", + "えたいの知れない不吉な塊が私の心を始終圧えつけていた。", + "holdout-ja-lemon-3", + "entity:pd-kajii", + "https://www.aozora.gr.jp/cards/000074/files/424_19826.html", + ), + ( + "item:holdout-ja:7", + "多摩川の二子の渡しをわたって少しばかり行くと溝口という宿場がある。", + "holdout-ja-wasureenu-7", + "entity:pd-kunikida", + "https://www.aozora.gr.jp/cards/000038/files/1409_34798.html", + ), + ( + "item:holdout-ja:8", + "一人の下人が、羅生門の下で雨やみを待っていた。", + "holdout-ja-rashomon-1", + "entity:pd-akutagawa", + "https://www.aozora.gr.jp/cards/000879/files/127_15260.html", + ), + ]; + let (corpus, _) = stores(); + for (id, text, group, speaker, url) in want { + let item = corpus.get(id).unwrap().unwrap(); + assert_eq!(item.text, text, "{id}"); + assert_eq!( + (item.group.as_str(), item.speaker.as_str()), + (group, speaker), + "{id}" + ); + assert_eq!(item.language, "ja-JP", "{id}"); + assert_eq!(SplitScheme::V1.live(&item), Some(Split::Test), "{id}"); + let Source::PublicDomain { edition_url, .. } = &item.source else { + panic!("{id}: not public-domain"); + }; + assert_eq!(edition_url, url, "{id}"); + } +} + +/// The concepts the pinned inventory holds, as each language names them in the +/// passages (a surface that appears in a passage). Every passage of both +/// languages rests on at least one live concept, so the two panel legs are +/// measured on the same inventory. +const LEXICON: [(&str, &[&str], &[&str]); 21] = [ + ("endanger", &["endanger"], &["危う", "危な"]), + ("prevent", &["prevent"], &["防げ", "妨げ"]), + ("control", &["control"], &["操", "支配"]), + ( + "request", + &["please ask", "request"], + &["くれませんか", "いただけませんか", "お願い"], + ), + ("sage", &["sage"], &["賢者"]), + ("mind", &["mind"], &["心"]), + ("belly", &["belly"], &["腹"]), + ("will", &["will"], &["意志"]), + ("bone", &["bone"], &["骨"]), + ("empty", &["empt"], &["空にし", "空の器", "空なのに"]), + ("fill", &["fill"], &["満た", "いっぱい"]), + ("weaken", &["weaken"], &["弱ら", "弱っ"]), + ("strengthen", &["strengthen"], &["強く", "強い"]), + ("heaven", &["heaven"], &["天"]), + ("earth", &["earth"], &["大地", "地は", "天と地"]), + ("water", &["water"], &["水"]), + ("compare", &["compare"], &["比べ"]), + ("resemble", &["resemble"], &["似"]), + ("continues_long", &[], &["長く続", "久しく続"]), + ("true_words", &[], &["誠の言葉", "真実の言葉"]), + ("seem_paradoxical", &[], &["逆さま"]), +]; + +#[test] +fn every_original_passage_rests_on_a_live_inventory_concept() { + use weksa::inventory::Family; + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/original"); + let tmp = tempfile::tempdir().unwrap(); + std::fs::copy(dir.join("inventory.cc"), tmp.path().join("inventory.cc")).unwrap(); + let inventory = InventoryStore::open(&tmp.path().join("inventory.cc")) + .unwrap() + .inventory() + .unwrap(); + let rev = inventory.latest().unwrap().n; + let live: Vec<&str> = inventory + .entries() + .filter(|e| e.family() == Family::Concept && e.live_at(rev)) + .map(|e| e.gloss.as_str()) + .collect(); + for (concept, _, _) in LEXICON { + assert!(live.contains(&concept), "{concept} is not a live concept"); + } + let (corpus, _) = stores(); + for (lang, col) in [("en-US", 0), ("ja-JP", 1)] { + let mut hit = std::collections::BTreeSet::new(); + let mut passages = 0; + for item in corpus.all().unwrap() { + if item.language != lang || !matches!(item.source, Source::ModelWritten { .. }) { + continue; + } + passages += 1; + let text = item.text.to_lowercase(); + let found: Vec<&str> = LEXICON + .iter() + .filter(|(_, en, ja)| [en, ja][col].iter().any(|s| text.contains(s))) + .map(|(c, _, _)| *c) + .collect(); + assert!(!found.is_empty(), "{} rests on no live concept", item.id); + hit.extend(found); + } + eprintln!( + "{lang}: {passages} passages on a live concept, {} distinct concepts", + hit.len() + ); + assert_eq!(passages, 28); + assert!(hit.len() >= 12, "{lang}: {} distinct concepts", hit.len()); + } +} diff --git a/data/original/corpus.cc b/data/original/corpus.cc index 108afaa2bd95cb74f2aab9d17d6715b9d807bbc4..4e28fcae918a2755c39ce726e937f36c62623870 100644 GIT binary patch delta 3371 zcmZu!TX0iV6rBP^%ZLb+$KWtv%8Y|fY5JrTuz*q!l!r3Dl?hNwj7_jjN8yK@P)Z-c zP)^gdw52@y2rW$l0W2vG`Opz(oZ-ihJHzmpn|pKf<&XY3>e}b#HYw((lgG{3YpuQZ z+NZxg$vXQeD?B4ByR@{Tv@nvB6`7WNd2jk(#>vn2RF*+@rZUIJacYCqHGq%$KcPP_b1FUrY2|keMgZ*DbRmowdelw|t%dl_kH{dZfW-sj=|6 zMFGD5HPbAjU`DCAJ^X3G{7SR5NjrX1WN^GK=8I^n%)n5v92aPbznh z$?ntf%k8MoU#VEaAC$j1t)ZSjDIez1*YhSTWQhvqLdmRkHmczta9-@DUyunX?YD7; za`KAIa3l))1;cLH-OQPJ5&yMh_RNO*h8jm5e_B4o^Yc^3zABD=zew5>i+W@>07P*X z!f9%#LuQ~dB)i=JF0)HAJ1@IiKuhiVUes-r8KexUy?sQEjmU1C5EiuBWH)!27caP; zuAb9T>!?3y-6ueJRy_-1d$IJPa`vKn4pl>O4^9lm?p#-n4+2f1>#EF7Qw=JKMnO^! zV8wkSv1k;}%dC&eLxs}-$M~wv=4_kAX0`KsRULfSYxDV^1*vu|Cf2!cOWV~(uW}b^ zg%TqHQuNj>V$2$epUbVbc8NpVl$Mi%Vu)He0dlTHe+0$iqd)_Nh%0~m!K`d}1R@(H zcNUt2v(5a@hDC1{mP(tb+YFT;T@wk*f`r}0RM((j1{Y31>I6F{`Z%6&;d$^Dn3z{| z^QHB(XWAS#N1cOTSv$zbi&8=^5#8Ke24a33=uyuErw{=8^B(Fi8&$8~q%KEYLQ6=1 z60HIlUcf@<_`h40!i=-CE%pNryC}7+BvopTC{?yvs?ttrk4N> z5lI*2Vi;7Fu-8C5!%ibgPJdLEI&+o)yl#!OP9uc)alaBBQ^v_>Y76>E$YVE=4C1R^ zI4tQ0bsXNu1C5Y>#dN#Xi3Vee%1s{`Ni(bGs?}LtTao7WiXv&98peR4djm4Vcn{(H zL~k#!YHesvBN-ijy>&%QRZntcQOMmQ`b^qmTxv<4A>O>kbks+-uIE zu1$x&{BBhz#%dn_qayE91J_HLa1~0G@k7rNyQ8PDF zO=MSWqEqk+iWL`_++g%}a$k_gM5ZB3+~DY^g3k!sP-od!onIt`)i#=Hjl!l(h0CPf zGP^DaVj%@%<74D(G8D+75^YsoQ|&s83%=cn3osfoPNz?uj(5NfXn(q)zHG48IFcyd zds82-Lag$FOej`MpBt;fBL)O|qX_;Y;F1|GR)|OG0krgQfe23qlv8LI+@hl%rXB4* zA%3n;8UJV9ys0|D3x^%nB*L+`OrMcXgm6wKgymAD01#3_NSx&SCifQnnH*{i7^c28 zElq)B@}5A6e9&R9&M!u9IG?j^9?z*VS&jZI&5&}%(he~S0Wkqk0y_=tu@PWLeY2qr zx+#(TsEiT8+kg^1h9^ZMu_>U9J2?jwFwSZ_e_Y-v5Y%ijePJM&p8-LMv< z#DK$rK#;VC-i|o7HSMA1w6ui|Py=z_M6;heW%dtSzQyd@_{NPL0>gu?sqoFqfT2{{ zBQ8iVLz~ic8GO_}Tw9EL1I;3A5Y!za_E?h=M$-9>zToZzX#j?LAzT4pr)RH|pE5sB z;XA|bIO4$l(+6NBI}Ks7Gk_|S*61k}yX`|wWe7IZuTYT3kFn7|S6gxCff57AUcUF; zrF4&(iO{DW7%uQ`Pw_WrD79MJuZsT&|3sJbcGwG3aMxA99bb9IeXTO}uRkoyr-7ZRkw;yw$}70?&W5IdcGO%*3YIWgT(G1OzAacp zuyMiWgsPgCI|aKU*n(geMOf7zbP0Bg{)B(3xV9w1Ot2ojF4%R!VpiXM!KTg0uwcF1 zAN2EoR2KPKl~W4;WA$x*==DwOC34f(1<&!uy(I?%Wm=tic?gjDy<0*Z5bUa8{es<; zUrfrcbDZ_WGVa)uWAefj{i@x9;Se*zlY(91XSZ%BO`Sg1a-yY8`FQK!l&$Tdjw8yc zV-k3Biw7Je;1hv>cH9_2HVN~tD%4(~B8j-3c!($2^lCPf5iD)aeuv9U2E27 zVgM8jZ{+2RfD!*Rf=8NGXBvzO20)Z_Rqwa+YFXdM$rHE@$IH<$%Z6?ar9~IY^++ zx4S7yV;MCaSyPrC8H!sIDsIUmEq1cyh;sB~Uf2H%dQy|MJ`mWUHR!WwkQI_^-zfEP z3NofVNMz{YFf*>Cfo%_WyJ`nLq^*ngRHxoY2f?EJOzn$qncbwTE4`$DPf1q=c53@% zC*VWSO2qLZp&~`;7T|mp5Az>|pR_9AjRq;tw3?2z)V-}2p3?!K|thn@$eD$0r%`;cbzu(~{vl+>h zHkWD5#tPZ8@dbnlNs_Zj`-L7=Ny-$!^4*!eKLlM74CxnE9LMW=KV@L5S86u%H-km%I!<)7gpm4o zGG#(6BgDfM+F|SAU^at#=4T-a3r^?EyM0C!&*6|`Feh}7-qmR=feJjqI9%gT&S=C~ zphHx@#`G&-hF;{#&R2Zx$4)4&aTN1c_imEYAjp5I-q;w_8cmRgY2>!!m}?zK8jZm& zWz%6jgF2EYWHUq5Jg5(+fnm;|GB|)9T!4FEszszFx9xeluvIyiXZrj;kLhB{yLP)) zYfMbq6NnLRhm5bTRo7#KBJch7F!YAj1uHbPC04`1qC!P_@f`o*O{bfPUozv;zgB9^ z_GQ#2Va~$`;${lrN9^!*NuTT>!l%Jc!<5e0L{K>jhvqC13M*H^yAi>Y={0w9v!L+g znpC&dc&U;pmDX3mlagR6Vctv`iMvGf&{A&8i69VjY*o63BP4)hx~kwEm#Ob*Q$VDU z!~zeLJue4A`!To&4T54m@sC zbYCZbX|A-ZOJj2vO>AE0o#MsnJ(TAOrGAH2Z)3Z_Xc%>O337DGc!h$c4GzBHdbzxr zl+_}mrIu5Hi`WQV+E}Y1Z7j+Q4|z}-kR}tRHGXeL7`z_=z!~BQ@{VEej5yW l>yP~Z2UG-eKmp=<2K6NxWZ^)cFLe4N+IM*BK*K-%zX7(vMZo|7 From 20863615b35c9fdbb7adf2b7e71eb7e091f220ec Mon Sep 17 00:00:00 2001 From: Weksa Date: Thu, 12 Nov 2026 13:05:16 +0000 Subject: [PATCH 03/57] data/original: ja passages replaced, none a rendering of an en-US original or a Dao line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All 28 Train/Validation ja passages are new Japanese sentences on the inventory's concepts with their own imagery (paddies, charcoal, kites, the bath, the market); none renders an en-US original, and none restates a data/daodejing line (ch3, 5, 7, 8, 78) or other canon (the old ones on 'true words', 'long-enduring', 'bellows', 'water vs hard things' are gone). corpus.cc restored from 21eaa07 and rebuilt through weksa corpus add: numbering 1..28 unbroken, holdout-ja:1..8 and original:1..28 byte-identical. The concept test now matches Japanese by multi-character surface and English by word stem, strips false friends (天気, 心配, 腹が立つ, 水曜 ...), and has a negative test. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/original_corpus.rs | 103 +++++++++++++++++++------- data/original/corpus.cc | Bin 28446 -> 28720 bytes 2 files changed, 77 insertions(+), 26 deletions(-) diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 47ed2d6..9213939 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -750,37 +750,93 @@ fn the_eight_ja_test_sentences_are_pinned_verbatim() { } /// The concepts the pinned inventory holds, as each language names them in the -/// passages (a surface that appears in a passage). Every passage of both -/// languages rests on at least one live concept, so the two panel legs are -/// measured on the same inventory. +/// passages: English by word stem, Japanese by a multi-character surface. This +/// is a tripwire, not the encoder: it fails a passage that names no live +/// concept, and `FALSE_FRIENDS` strips the idioms that hold a concept's +/// character without its meaning (天気, 心配, 腹が立つ ...) before matching. const LEXICON: [(&str, &[&str], &[&str]); 21] = [ - ("endanger", &["endanger"], &["危う", "危な"]), - ("prevent", &["prevent"], &["防げ", "妨げ"]), - ("control", &["control"], &["操", "支配"]), + ("endanger", &["endanger"], &["危う", "危ない"]), + ("prevent", &["prevent"], &["防い", "防げ", "妨げ"]), + ("control", &["control"], &["操ろ", "操る", "操れ", "支配"]), ( "request", - &["please ask", "request"], - &["くれませんか", "いただけませんか", "お願い"], + &["request"], + &["いただけませんか", "もらえますか", "くれませんか", "お願い"], ), ("sage", &["sage"], &["賢者"]), - ("mind", &["mind"], &["心"]), - ("belly", &["belly"], &["腹"]), + ("mind", &["mind"], &["心が", "心は", "心を", "心の", "心も"]), + ( + "belly", + &["belly", "bellies"], + &["お腹", "空腹", "満腹", "腹の虫"], + ), ("will", &["will"], &["意志"]), - ("bone", &["bone"], &["骨"]), - ("empty", &["empt"], &["空にし", "空の器", "空なのに"]), + ("bone", &["bone"], &["骨が", "骨を", "骨は", "骨の"]), + ("empty", &["empt"], &["空っぽ", "空にし", "空の器"]), ("fill", &["fill"], &["満た", "いっぱい"]), - ("weaken", &["weaken"], &["弱ら", "弱っ"]), - ("strengthen", &["strengthen"], &["強く", "強い"]), - ("heaven", &["heaven"], &["天"]), - ("earth", &["earth"], &["大地", "地は", "天と地"]), - ("water", &["water"], &["水"]), + ("weaken", &["weaken"], &["弱ら", "弱め", "弱っ"]), + ("strengthen", &["strengthen"], &["強くな", "強め", "強くし"]), + ( + "heaven", + &["heaven"], + &["天が", "天を", "天は", "天上", "天の"], + ), + ("earth", &["earth"], &["大地"]), + ("water", &["water"], &["水が", "水を", "水は", "水面"]), ("compare", &["compare"], &["比べ"]), - ("resemble", &["resemble"], &["似"]), + ("resemble", &["resemble"], &["似て", "似た", "似る"]), ("continues_long", &[], &["長く続", "久しく続"]), ("true_words", &[], &["誠の言葉", "真実の言葉"]), ("seem_paradoxical", &[], &["逆さま"]), ]; +/// Compounds and idioms that contain a surface above without naming its concept. +const FALSE_FRIENDS: [&str; 8] = [ + "天気", + "心配", + "腹が立", + "腹を立", + "水曜", + "骨が折れ", + "天の川", + "天井", +]; + +/// The live concepts a passage names, by `LEXICON`, in `lang_col` (0 en-US, 1 ja-JP). +fn concepts_in(text: &str, col: usize) -> Vec<&'static str> { + let mut text = text.to_lowercase(); + for f in FALSE_FRIENDS { + text = text.replace(f, ""); + } + let words: Vec<&str> = text.split(|c: char| !c.is_alphabetic()).collect(); + LEXICON + .iter() + .filter(|(_, en, ja)| { + if col == 0 { + en.iter().any(|s| words.iter().any(|w| w.starts_with(s))) + } else { + ja.iter().any(|s| text.contains(s)) + } + }) + .map(|(c, _, _)| *c) + .collect() +} + +#[test] +fn the_concept_lexicon_does_not_match_a_word_that_merely_holds_a_character() { + for ja in [ + "今日の天気はどうだろう。", + "試験が心配でならない。", + "腹が立って仕方がない。", + "水曜に会いましょう。", + "骨が折れる仕事だ。", + ] { + assert!(concepts_in(ja, 1).is_empty(), "{ja}"); + } + assert!(concepts_in("I attempt it, with a reminder.", 0).is_empty()); + assert_eq!(concepts_in("水が澄んでいる。", 1), ["water"]); +} + #[test] fn every_original_passage_rests_on_a_live_inventory_concept() { use weksa::inventory::Family; @@ -801,20 +857,15 @@ fn every_original_passage_rests_on_a_live_inventory_concept() { assert!(live.contains(&concept), "{concept} is not a live concept"); } let (corpus, _) = stores(); - for (lang, col) in [("en-US", 0), ("ja-JP", 1)] { + for (col, lang) in PANEL.iter().enumerate() { let mut hit = std::collections::BTreeSet::new(); let mut passages = 0; for item in corpus.all().unwrap() { - if item.language != lang || !matches!(item.source, Source::ModelWritten { .. }) { + if item.language != *lang || !matches!(item.source, Source::ModelWritten { .. }) { continue; } passages += 1; - let text = item.text.to_lowercase(); - let found: Vec<&str> = LEXICON - .iter() - .filter(|(_, en, ja)| [en, ja][col].iter().any(|s| text.contains(s))) - .map(|(c, _, _)| *c) - .collect(); + let found = concepts_in(&item.text, col); assert!(!found.is_empty(), "{} rests on no live concept", item.id); hit.extend(found); } diff --git a/data/original/corpus.cc b/data/original/corpus.cc index 4e28fcae918a2755c39ce726e937f36c62623870..9c53141206b52a394c56f05303a5e2b7230c6d54 100644 GIT binary patch delta 3391 zcmZWsTTEQn6&1#IoHmfec4XQrGFF{dDaB?6%)_C?!p5D(F$on?B}cAuoj5ign`u1e z!ylFAjv3zWV2-hQhSx9yGvUs_;CR4b6shT766K>(uas6*@0|PSN7cti)xWNN?hM?f zKWOgj?7i1sYwgWPe=11*xgfQx;F-#k6;&rItJBXGEbrRB__Xj}I~Py({&wf0>scj# zQIme{xxegO9DRQKfSUBzFRt!9d3f;G2fuB{sk-o4!4KJ|KRwXwQh%YZC9|16O&-vK z_^TKhnaiu19Mb|a-@1eQJatc#A8PWLCP&OCo0=TfwGPJH@3!ul01Xua1=- zNc(QKU1@8Vu9shv%5Qf_SEQS)#dDv%?mobpe^ly7;Sg(SIB?cgp*C6LaWjSk5lGQf zh(2IG8RDTHEjX0jnAGGhYbtH0WDLi%>+71F)JNv=)R>wVnM|<{%MY=y&lW$^;ca{0 z=e^91)X%fYvLlXgXDHk?mrK3LGeJI$D2g=}vX3>>vt|IllYDIj#p+XAI9!waHF-q9 zf_i~ZFEFtJW8w}T^=rX?N`^jE%umuh{D2+>hxFl;Cik!pUoBDV34$laV?YcjQ82!b@$ML%6~@gWMAGC1glGRObF))bMSE{aUSD~Y zIP&zdBRS5TU|$_C@l>iF0E&!_#k*QCm}?b&1Y}LdrNWo{_31TKYmG1H8(}+XYXp^} zoiXZYa6pr%m{fOk?`^65axTu|@qFqhK%nE4`a7;F^@1k%nIRm9Z$`xtQ-I2nr}Yqu z85hJ!mjw)&uPl%hUj4=z1Egp^g#0tDrW@-svz1Xid&) zazY=O2JSqvh}VPyQbYkcY)1QqVunPAzpO6WeM{=F8Q0==rgV%_`lYK{y~raI)*N6# zBRjRAtOb!$7j=|Ecj=vBzJ)ZBl-pWTPvN&>B+&9mHnV}RcrVIHh~@zq0M*A=k(Fo? zi2n`EN&c%oFDtE&vUsI4oia-2=dK#{qAm~OYaW7xfc;a^w*dq)GndIDEp$;3FHf?@L*kq+Gxx_(33z=5pwC&ACR0WL=1o` z`a=*Lt$8Z?n-GPBvl|=Rg!TI_5r9$VzxZ;oyVYgS1!~pv6rs_jHVX3>z))T!NRi;8 zCZqoekuQXrG6yjcW{gS58*>3gKr_gzD-YW)LB$vgyymRq2t_&PR$KGbSNO)VVD^x? z^080|FoS^ms53;pF?|BQsPN1kK0ZezrF@_$n~Ym*rlFLHF8K+!-{pLtD%wY}Dk{}F zzP3uJlS>06a#QLUTyPS$g*ypj6*iQcZtS+Jl!0u!Z~6RJa|3KYd!=E=YX4SW@^+Ql zVl1u;cmRW9_AMLilUyg_%zTPRph8mEOW62Lr}co0CEx5QagHnnZeWX${i?i#q856k zOYGeXFSEZlBpjt2rc#Yv$zDXqs&$DU9yV88y)$rU_U$7=oIl$s?k%Uwpgy4aepMk?vL?FoL%65Bcpt_k#)lXjQ8sznNk)Ncb1hIEevFBTGp(@+ zzBGssMw;e1ag*9K4$8n5^-&-UrX!^p>Lyl+;D9l%(3|UbXj)`*aLNMiP6rU{Q zuV?b{X%#z6t5|EL+M2UpLdv!#P#kVZ&S+44#k#YA*^LDAg`)ovJ%dpI9>K|D`Z{!o zuP`F=!+{lhzqJRe*iD}Z-E{W6 z83C9DcL4PgSPg0aBlhm`OKl%?Alzq;_3JpQeCeHcc_hXs@cEFLL77{^W})z?zJ$=Y zL$ajFjOcT?BhF9%rfF zA(R`d)pN!?#ty*3adB&d$cqs#20BeRzKln?JoHHvrkHW03q%I8TWI%)@K7r9mPAvC z+IjK~XP_4-P*aWClwT^b#s!U86qztrh?(2=lg;!B@h0s>TTjp0Ltl^$J3yx}(Omzo z*s&JQ?Mxr(jYF-q>U$6hzZcYJU_1$|55QJ9H7py#LwBGR97anSu+^s-UNC0iOu&|q z&w)HgO`+cy=%2fPKq}q+K}U`?h3B0W)>4Jn>(uK)B!YcF6aOqgt^v3-VxpN)4xmiv z_mfr#0?v)oZ5f-gVq!&xU8OY$ahfdYR}r~X zQ9ck*5V;5yW?QW~EMAjsf0nTLV?R!^m~E%$w9S4kOZIQcp7%Q~J!bqNd_Cv9@B2LO z^FHqv`pt0jwIMXkFw0`g&$s29!*dMxrcEARPX9Z77@zrl>R<`i&k3Xtg>V0nK4j0I z43HcCb-@Giq`;PA33n{|X3<+BPoDMg4~7+d)!M~dcB`QWiJpF$`4T-{GJ7U_o9Qnb zmOU&U?vHye$Sjx`$D;w64aLTo!XC(MREZ4Z7rx47T=K|W?WlCP>?ciM*-cfBbG1%; zrM-$9wh!^o*R9qP=I0>c0=~L%q0O8l?TBA^EVDqoYn0N45wAsNy)p|Uus&`4;)v2v zk2jU)=VfmzcjXxQi;~P4Rkf#2i1@(|_3;f9e~H;F<;1VmsiE%>E&8}a0P^#0lZD?} zn>n-gbZw=(T6}FW>A(JfzW&r~kq$*8KA8<7ZHxtxRt>hv4D}AlUN5SX*&Uhn$=*iP zu6AA);p=1u)`Dtx4`o;eT};H0Mg6 zDK`hyTSy#?`4BV^efChfa2o|BI`7M@mC{gFv<1}pP+6>fBpQj}eVO%81>RPr74@Ow z5$;)QdzO;in z%a8~R))T)hK=pf=>bWh@L3iU|Ji%@Wk&GogcpunBNvwsKC0sL{Zl{}nvcH`hO$+&i zMW1XrCEH>y0JaVU^{LkbZ(sm`J|DH3jjHz_Q^O-3!6{fknkE#-SL}bEl$T+59e2C< z@1`MMn3tw*W#Q6o=1o#@d<0bn;(ky3D%3$R1U6zS;en3Z#bk}f*-e>onKclnNzAuW zNHVJIPInD2+utUj+|JiS$sj0Z^Jb|a(GGdH#2#HIT6DaL!a*kSqYQ)qR|&Q7S4AtS z{OK-xW$X4~{#SusaC0?&tWr^&3dYC1O60CGMw(L_(M|#qeGE?!kGi3@q%71ii~?S$ z6U7(XkibzRT7Ko@CSJPPcsT7gb8p;t)Kpa2bW>onNJVN01B!kQ%Iqpt?O}=TZd4s_ zMyu*b{!sCgdt%&f2>PH1u+}J2z9^YD{e+_`$yOF0vg(0l1K6TNInp7u6Tt$YEO7%; z3&z?ZP+e4VysKhl?@;~-xC>~5C(t!$)~7rn#>H@pffnNgLSc0@PDi&k^M7nY*OE{XF2WX5dR#1RYH=> z;saas;f)l&ELSR12GQ(bVziT3@Gzwt;tDF40iX`XkTxC=eGO2Dp*x}Ykr*rybO(|= z()&qlewBdKC` z@AiX2K2c;8ZeLU3KJHF8Xzw=tG%X@%R+vz9{F%!kpj;USR1E zNxoj6IonxVZ9i6l{bH~VasPIGigyKj1yY&J9*Xj@N&?ZbyQD%=2B07&Ej^8;_(O;c zXu1;pP>e8gI#S%j7(&A0Oas>@Q&$UXi^f9j&#pk5Y?Ad7FS+^ru!qJaXJv8%`h0Cm) zrD731s01B+^4oE0DQII}Hu1V))!~C9U-OLkiSA{7TZp-=TF3U;nhC~;Ml~Z_0 zu5>^s)GveyXxJry14D*lw2cCxXu5&%A{&&8qhE=f$D5)XFlvDHrsO$cnm41uRZdrp zHdK(smzNlirmI*uMHP9{4&0}ZQ0;9Ka|h9&ie!&9j$>Gqv?NIh#m zgOPOFli0KPncYT5I`f(-GUrQ&#LWbKZC2{P1roIfmxa`#ra?hR16s4i#;8|9aGZ`* z7jS6-DJY?fPi)gJ3oiFL3{MJubWciO)PoE@fA8W=R?QWnPumev8G>N6dkHXiK-Ajr zY7_1dfHx>C4nnHKTMP;EKLa~hllnpE#D-S$46i=a1Jk}z!dUJKqEkC zlL0BBUla}Y6Ke+)3>FXsL@TxBi!uf&f~H}bTF`^LD|K8fkYd~XY1L_(8}}K@(=<;d zv&~Xj>Y4?}oG?b()rK+TrN+}LLsIB!(yb=djo;B`Y-=C?DQ^wGT$nk{aU7=pqpq{d zr|7IzDpdxDHNO?tB4{)T`;DV;2|*C88L{(7`thcx5QB0tl9qKQx9;f*e{t{^<9}Zo Bl%@ax From 598cbf62d6e7ab2795e13e57b4a4327ac1c17d0d Mon Sep 17 00:00:00 2001 From: Weksa Date: Thu, 12 Nov 2026 14:06:16 +0000 Subject: [PATCH 04/57] Map: cut 5 language-ontology specced as ontology store and seed cuts Co-Authored-By: Claude Sonnet 5.5 --- docs/targets/interlingua-lowering-map.md | 29 ++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 9c18e09..22db5cd 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -33,9 +33,9 @@ means no store holds it and nothing writes it. | Schema version | `weksa.interlingua_packet.v` constant in `crates/weksa`; published JSON Schema at `schemas/cultnet/` | Bumped by a commit that changes the Rust types; the published schema is regenerated in the same commit and a test fails on drift | A cut (Hands, under a spec); never the loop | | Inventory entry | Key `:`, `n` canonical decimal >= 1 (families: concept, frame, role, relation, speech_act, feature, referent_kind, extension); type `weksa.inventory_entry`, with a bounded gloss, value kind (option labels bounded), the slots a feature may sit in (`Discourse`, `Referent`, `Predication`, `Relation`; `Stance` from cut `speaker-state`, `Speaker` and `Scene` from cut `speaker-scene`), `introduced_rev` and optional `retired_rev`. Speaker axes, scene relations and affects are features of those slots, so the loop grows them like any closed-class family and convergence covers them | Added or retired by an inventory revision; never edited in place (a changed meaning is a retire plus an add); a retired id still resolves for packets of earlier revisions | The loop's accepted rounds, whose minting call the context builder assembles from Train items only; Self on merge to main. Admission checks form, not text | | Inventory revision | Key `rev:`, type `weksa.inventory_revision`, with parent, change list and `frozen` | Appended per accepted round; exactly one revision is ever marked frozen, by the convergence test | The loop's accept rule appends; the convergence test freezes; Self records the freeze as a ruling | -| Target profile | Locale or language-project id (`en-US`, `ja-JP`, a conlang's slug) plus a revision `profile::` | Authored, revised by commit or by an accepted round that changed the lowering; each lowering names the revision it used | Profile authors and accepted rounds; never the lowerer's defaults (`persona-drives-register`) | -| Language ontology entry (cut `language-ontology`) | Key `onto::`; type `weksa.ontology_entry`: conditions over feature values (stance, speaker, scene, packet features) and community ids, and the surface choice they select (layer: lexicon, grammar, discourse, phonology, orthography; a bounded descriptor and at most three bounded examples); introduced and retired by a profile revision | Added or retired by a profile revision, never edited; converted from `data/target-language-ontology/*.yaml`, which is deleted in the same cut so the YAML is not a second owner | The language's owner (the operator for her settings' languages) and accepted rounds; an ontology-minting call is assembled from Train items only, like an inventory-minting call, and its text is in the scorer's leak diagnostic. Selection is deterministic code (`select(profile, stance, speaker, scene)`); the lowerer realizes the selected choices and the trace lists their ids (`lowering-traced`) | -| Community (cut `language-ontology`) | Key `community:`; type `weksa.community`, converted from `data/cultural-ontology/cultures/` and `referents/` (YAML deleted in the same cut) | Revised by commit; a speaker names the communities it belongs to by id | The cultural ontology's author (owner-attested evidence only; Discord-only content stays deleted, `discord-only-content-deleted`) | +| Target profile | Locale or language-project id (`en-US`, `ja-JP`, a conlang's slug). Until entries grow in rounds there is no profile revision document: a run names the profile it lowered with by the ontology store's path and digest in its `RunRecord` (cut `language-ontology`); `profile::` revision documents arrive with follow-up `language-ontology-growth` | The ontology store is revised by commit; a run pins it and refuses drift | Profile authors and accepted rounds; never the lowerer's defaults (`persona-drives-register`) | +| Language ontology entry (cut `language-ontology`) | Key `onto::`, `n` from 1 per language; type `weksa.ontology_entry` in `data/ontology/ontology.cc`: the language, the inventory revision its conditions are read at, a layer (lexicon, grammar, discourse, phonology, orthography), at most four conditions (a feature id of slot `Stance` or `Discourse` with `AtLeast`/`AtMost` a grade or `Is` an option or flag; slots `Speaker` and `Scene` and community ids join with follow-up `language-ontology-view-assembly`), a choice of 1 to 200 bytes, at most three examples of 1 to 120 bytes, and an `Annotator` | Admitted in all-or-nothing batches through `OntologyStore::admit`, immutable, never edited; retire arrives with `language-ontology-growth`. The run pins the store like the entity store (path and digest, `Drift(Ontology)`), never copies it, because no loop step writes it yet. Cut `language-ontology-seed` converts by drafting, not by parse: a fresh model drafts each language's seed from that language's YAML and the inventory's feature lines with no corpus text, then deletes `data/target-language-ontology/` and `.weksa/cultural-ontology/` in the same cut | Seed: a labelled model (`Annotator::Model`), no operator gate, never Test evidence (`ja-ontology-owner-answer`); attestation (operator, native reader) appends later. Accepted rounds later, through a mint call from Train items only. Selection is `ontology::select(ontology, language, view)`, called only inside `Prompt::lower`, so no caller picks entries; the prompt prints each selected entry's id, layer, choice and examples. Its text is minted text for the leak diagnostic, which today has no runtime caller (follow-up `leak-diagnostic-in-round`) | +| Community (cut `language-ontology`, follow-up `language-ontology-communities`) | Key `community:`; type `weksa.community`, converted from `data/cultural-ontology/cultures/` and `referents/` (YAML deleted in the same cut) | Revised by commit; a speaker names the communities it belongs to by id | The cultural ontology's author (owner-attested evidence only; Discord-only content stays deleted, `discord-only-content-deleted`) | | Speaker (cut `speaker-scene`) | Key `speaker:@`; type `weksa.speaker`: stable axes as feature values of slot `Speaker` (origin, socioeconomic standing, age cohort, education and whatever the loop adds), at an inventory revision, and an `Annotator` | A new revision appends, never edits; a corpus item and every prior turn name a speaker slug, and a round records the speaker revisions it read | The annotator it names: `PersonaSnapshot` (a projection of a `gamecult.persona_state.v0` document, naming its source document and `updatedAt`), `Operator`, `WorkMetadata` (a public-domain work's own cast list and narration), or `Model { family, generator }` (the 2b `ModelFamily`) | | Scene (cut `speaker-scene`) | Key `scene:@`, the item's split group; type `weksa.scene`: participants (speaker slugs) and, per ordered pair, relation features of slot `Scene` (relative status, intimacy, in-group, audience and whatever the loop adds), at an inventory revision, and an `Annotator` | As Speaker. Keyed by the group, so a scene never straddles the split | As Speaker | | Corpus item | Key `item::`, `n` canonical decimal >= 1 (no sign, no leading zero, from cut 2b); type `weksa.corpus_item` with text, language, speaker, split group, prior turns, source. Source `ModelWritten` carries a `ModelFamily` enum, the generator string as the fixture wrote it, and a prompt digest (cut 2b). The item carries no speaker state of its own: its speaker slug and group name the Speaker and Scene documents | Admitted once; the whole item is immutable (re-adding an identical item is a no-op, any difference is refused; a correction is a new item that `supersedes` the old); text already held under another key is refused unless the new item supersedes that key (cut 2b); withdrawn, never deleted, so old scores stay explainable | Importers and `weksa corpus add`, under `corpus-cast-plus-public-domain`; the source enum has no variant for Discord or any training-forbidden source (`corpus-clean`) | @@ -309,6 +309,27 @@ line two", "その　制御" against the or socioeconomic standing; `extensions` are non-authoritative by the schema's own description. +- Probes of 2026-11-12 at `d9ebbcf` (map pass for cut 5): no code reads + `data/target-language-ontology/`, `data/cultural-ontology/` or + `.weksa/cultural-ontology/starter-targets.cc` (a third, typed copy of en, + ja-JP and pt-BR profiles); `README.md`, `docs/multilingual-utterance-lowering.md`, + `examples/cultcache-lowering/` and `prompts/nibu-cultcache-target-*.md` cite + them. `LoweringInput::with_scene` has no caller in `run.rs` or `main.rs`, the + run pins no `SpeakerStateStore`, and no Speaker or Scene document is committed + (`speaker-scene-seeds` open), so only the packet's `Stance` and `Discourse` + features reach a loop's lowering view. `score::leak_diagnostic` and + `TestReadings::at_checkpoint` take minted text as `&[&str]` and have no + caller outside tests: no round reads glosses or any minted text against + Validation. `data/original/inventory.cc` and `data/daodejing/inventory.cc` are + byte-identical; with `data/interlingua/inventory.cc` they share revision 1, + where features 6 to 8 (hedge, certainty grades, irony flag) sit in + `Discourse` and `Predication` and features 9 to 18 (anger to restraint, + grades) in `Stance`. `PROMPT_SOURCE_DIGEST` covers `context.rs`, `split.rs`, + `lower.rs` and `packet.rs`. 25 `RunPaths {` literals across `run.rs`, + `context.rs` doctests, `main.rs`, `tests/common/mod.rs`, `tests/golf_loop.rs` + and `tests/original_corpus.rs`; 10 `Prompt::lower(` calls. `huginn known` + timed out (Unavailable, 15 s deadline) twice this pass. + ## Where things live and run (`rust-single-crate`, `loop-runs-on-yggdrasil`) - Code: one Cargo workspace at the repo root, one package `crates/weksa` (a @@ -403,7 +424,7 @@ One line per cut; specs exist for the cuts about to run (2d and 4, siblings on ` 4. `speaker-state`: packet `stance` (felt affect and restraint, slot `Stance`, packet schema v4); formality, warmth and intensity out of the seed and the standard; only `PacketStore::admit` yields a `LoweringInput`, and it refuses packets for withdrawn items (follow-up `lowering-input-from-packet-store`). (spec r1) 4a. `speaker-relation-input`: slot `Scene` and its first seed features (`relative_status`, `intimacy`, `in_group`; inventory revision 3, now `speaker_scene_revision`); `LoweringInput::with_scene` carries the speaker's relation to the addressee as typed `Scene` feature values. Since 4b it takes the `SpeakerStateStore` and the `CorpusStore`, reads the scene of the view's own item group for the item's speaker, and selects the relation by the addressee referent's entity (V16 when the scene does not list the item's speaker or the speaker of any live item of the group, the rule `put_scene` applies too; V17 refuses an addressee that names a predication; V13 is retired). The English lowerer does not render it. (PR for follow-up `chat-s3352-3352`) 4b. `speaker-scene` (spec r1; `speaker-scene-seeds` is its cut 4c: CLI, inventory revision 3, cast speakers and scenes; `Party::Unnamed` covers an addressee with no entity; Speaker and Scene share one `SpeakerStateStore`): slots `Speaker` and `Scene` with their seed features; `weksa.speaker` (`speaker:@`) and `weksa.scene` (`scene:@`, participants and ordered-pair relations) documents with `Annotator` `Operator` and `Model { family, generator }`, validated by the packet's shared feature check, append-only per key; their stores, `weksa speaker` and `weksa scene` commands, published schemas; Model-annotated cast speakers and scenes for the committed items. Waits on 2d and 4; `pd-speaker-annotation` blocks only the public-domain annotator. Part 1 landed (branch `weksa-lowering/speaker-scene`): `speaker.rs`, `SpeakerStateStore`, slots `Speaker` and `Scene` exclusive to their own feature lists, `speaker_scene_revision()` (Speaker features 22-24), `with_scene` on the store, schemas. The CLI, the committed inventory revision 3 and the seeds stay with 4c. -5. `language-ontology`: `weksa.community` from `data/cultural-ontology/` (owns follow-up `edda-referent-yaml-parse`), `weksa.ontology_entry` for en-US and ja-JP from `data/target-language-ontology/` (YAML deleted), speaker communities, deterministic `select`, the lowering view (packet without trace, speaker, scene, selected entries). Waits on 4. +5. `language-ontology`, in sequential cuts on `d9ebbcf`. 5a `language-ontology` (spec r1): `weksa.ontology_entry`, `OntologyStore`, `ontology::select` over the view's `Stance` and `Discourse` features, `Prompt::lower` printing the selected entries of the item's language, `RunRecord` pinning the ontology store (keys 15 and 16), `weksa ontology admit|dump`; `ontology.rs` joins `PROMPT_SOURCE_DIGEST`. 5b `language-ontology-seed` (spec r1, after 5a): Model-annotated en-US and ja-JP seed entries drafted by an isolated model, `data/target-language-ontology/`, `.weksa/cultural-ontology/` and `schemas/target-language-cultural-ontology.md` deleted. Lost until 5d: keigo, hierarchy and uchi-soto hang on scene relations, so ja-JP politeness is still the lowerer's guess. 5c `language-ontology-view-assembly` (follow-up): the run pins the `SpeakerStateStore`, the round reads the item's Speaker and Scene through the stores, `select` widens to slots `Speaker` and `Scene`; waits on `speaker-scene-seeds`. 5d `language-ontology-communities` (follow-up): `weksa.community` from `data/cultural-ontology/` (YAML deleted, owns `edda-referent-yaml-parse`), community ids on `weksa.speaker`, community conditions; waits on 5c and question `persona-speech-community-home`. 6. `pd-importer`: public-domain play and fiction dialogue into the corpus with speaker, scene group, edition provenance, and Speaker and Scene annotations by the ruled annotator. Waits on 4, follow-up `gutenberg-terms` and question `pd-speaker-annotation`. 7a. `loop-context-builder`: `context.rs` (`TrainItems`, `EncoderOffer`, `Prompt`: corpus text reaches a model call only through `Prompt::mint` over live Train items or `Prompt::encode` over one offered item; instructions are the consts `MINT_INSTRUCTIONS` and `ENCODE_INSTRUCTIONS` in `context.rs`; `Checkpoint` has no public constructor, offers each item once, and its offers borrow it (the gate is typed `TestReadings` and Test offers on the round path; operator tools such as `weksa corpus dump` and `weksa score` still read corpus text of any split); prior turns print the held Train text), `SplitScheme::live`, `CorpusItem::cited_entities` (one owner of the id set `PacketStore::admit` and the offer share), and checkpoint-only Test readings (`Checkpoint`, `TestReadings`; `Aggregate` and `leak_diagnostic` carry no Test). Library only; since 7b `TrainItems::from_corpus` and `EncoderOffer::for_round` are crate-private and `RunStores` is their one public producer. Merges follow-ups `loop-context-builder-2` and `scorer-test-readings-type`. (spec r1) 7b. `golf-loop-round` (spec r1; branch `weksa-lowering/golf-loop-round`): the smallest loop that scores one round. `run.rs`: `RunStores` pins the corpus and entity stores of a run by canonical path and FNV-1a 64 digest at `create`, reopens only from `/run.cc` and refuses drift (corpus, entities, `PROMPT_SOURCE_DIGEST` of `context.rs`, a second pin on `lower.rs` retired in 7d, model, embedder); `round` encodes every live Train and Validation item through a `ModelPort` (`model.rs`; the one adapter `AnswerFiles` writes `.prompt` and reads `.answer`, so the model is the Claude session), reads each answer into a `PacketDraft` (`encode.rs`, the one place a model's JSON is read), admits through `PacketStore::admit` into `/round-/packets.cc`, lowers, scores once, and keeps a `RoundRecord` (`weksa.loop_round`, no text, no Test; Train rows, Validation only as means and refusal counts). `weksa loop open|round`. Not here: the spec-edit step, accept rule and composite, noise floor, Score and lowering documents, a subprocess adapter, the checkpoint door, a model lowerer (follow-ups `golf-loop-spec-edit`, `golf-loop-accept-rule`, `golf-loop-score-documents`, `golf-loop-subprocess-adapter`, `golf-loop-checkpoint-door-bound`, `golf-loop-model-lowerer`). `packet-store-inventory-binding` is met for loop packets only (`RunStores` admits with the run's inventory); `validate`, `packet proof|dao` and `lower` still take caller stores. Waits on 7a, `checkpoint-gate`, 4b. From 0ea07d21faba1d14ada8698ba11a4219bd639bc9 Mon Sep 17 00:00:00 2001 From: Weksa Date: Thu, 12 Nov 2026 14:11:51 +0000 Subject: [PATCH 05/57] Ontology entries per language, selected inside Prompt::lower and pinned by the run OntologyEntry, OntologyStore::admit, ontology::select over the view's stance and discourse, OntologyDraft, run keys 15 and 16 with Drift(Ontology), weksa ontology admit and dump. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/context.rs | 51 ++-- crates/weksa/src/encode.rs | 55 ++++ crates/weksa/src/lib.rs | 4 +- crates/weksa/src/main.rs | 79 +++++- crates/weksa/src/ontology.rs | 164 +++++++++++ crates/weksa/src/run.rs | 70 ++++- crates/weksa/src/schema.rs | 2 + crates/weksa/src/store.rs | 134 ++++++++- crates/weksa/tests/cli.rs | 4 + crates/weksa/tests/common/mod.rs | 12 + crates/weksa/tests/context.rs | 12 +- crates/weksa/tests/golf_loop.rs | 44 ++- crates/weksa/tests/original_corpus.rs | 7 + schemas/cultnet/weksa.loop_run.schema.json | 12 +- .../cultnet/weksa.ontology_entry.schema.json | 264 ++++++++++++++++++ 15 files changed, 869 insertions(+), 45 deletions(-) create mode 100644 crates/weksa/src/ontology.rs create mode 100644 schemas/cultnet/weksa.ontology_entry.schema.json diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index 877c974..2b88a08 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -10,7 +10,9 @@ //! [`Inventory`] and an [`EditFeedback`] and nothing else, so no held-out text, //! key or per-item score can reach it. A lowering prompt names the language it //! is said in on a language line; every language is named by its tag, none by a -//! default. +//! default. A lowering prompt also prints the target language's ontology +//! entries that `ontology::select` chose for the view; the caller never names +//! entries. //! //! Item text may hold line breaks; the renderer indents every continuation line //! (`indented`), so item text cannot forge a `[key]` line or a paragraph. @@ -28,6 +30,7 @@ use serde::Serialize; use crate::corpus::{CorpusItem, CorpusItemId}; use crate::inventory::{Inventory, InventoryEntry, OptionLabel, ValueKind}; use crate::lower::{EntityForms, LowerError, entity}; +use crate::ontology::{Ontology, select}; use crate::packet::{ Context, Discourse, EntityId, Extension, Predication, Referent, Relation, Stance, }; @@ -68,7 +71,7 @@ A feature is {feature, value}; value is {\"Grade\": 0 to 1000}, {\"Option\": lab /// Fixed per run by `PROMPT_SOURCE_DIGEST`. pub const LOWER_INSTRUCTIONS: &str = "\ Say the packet below in the language on the language line, as one utterance of its speaker: all of its meaning and nothing it does not carry. -Every id is the inventory's listed below and every feature value is as listed; a grade is permille, 0 to 1000. The stance is the speaker's felt affect and restraint: it shapes how the utterance is said, not what it says. +Every id is the inventory's listed below and every feature value is as listed; a grade is permille, 0 to 1000. The stance is the speaker's felt affect and restraint: it shapes how the utterance is said, not what it says. The ontology lines are choices this language makes for this speaker and stance: realize each where the packet allows; they add no meaning. A referent with an entity is named by one of that entity's forms on the entities line. Answer with one JSON object and nothing else: {\"text\": \"...\"}. No code fence, no commentary."; @@ -86,7 +89,7 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat /// ```no_run /// # let dir = tempfile::tempdir().unwrap(); /// # let p = |n: &str| dir.path().join(n); -/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; +/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc"), ontology: &p("o.cc") }; /// # let panel = ["en-US".to_string()]; /// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &panel).unwrap(); /// let t = run.train().unwrap(); @@ -100,8 +103,8 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat /// ```compile_fail /// # let dir = tempfile::tempdir().unwrap(); /// # let p = |n: &str| dir.path().join(n); -/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } -/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; +/// # for n in ["c.cc", "i.cc", "e.cc", "o.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } +/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc"), ontology: &p("o.cc") }; /// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &["en-US".to_string()]).unwrap(); /// let t = run.train().unwrap(); /// let _u = t.clone(); @@ -111,8 +114,8 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat /// fn s(_: &T) {} /// # let dir = tempfile::tempdir().unwrap(); /// # let p = |n: &str| dir.path().join(n); -/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } -/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; +/// # for n in ["c.cc", "i.cc", "e.cc", "o.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } +/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc"), ontology: &p("o.cc") }; /// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &["en-US".to_string()]).unwrap(); /// let t = run.train().unwrap(); /// s(&t); @@ -334,7 +337,7 @@ impl EditFeedback { /// ```no_run /// # let dir = tempfile::tempdir().unwrap(); /// # let p = |n: &str| dir.path().join(n); -/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; +/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc"), ontology: &p("o.cc") }; /// # let panel = ["en-US".to_string()]; /// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &panel).unwrap(); /// let t = run.train().unwrap(); @@ -352,8 +355,8 @@ impl EditFeedback { /// ```compile_fail,E0061 /// # let dir = tempfile::tempdir().unwrap(); /// # let p = |n: &str| dir.path().join(n); -/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } -/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; +/// # for n in ["c.cc", "i.cc", "e.cc", "o.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } +/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc"), ontology: &p("o.cc") }; /// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &["en-US".to_string()]).unwrap(); /// let t = run.train().unwrap(); /// let inv = run.inventory().inventory().unwrap(); @@ -365,8 +368,8 @@ impl EditFeedback { /// fn s(_: &T) {} /// # let dir = tempfile::tempdir().unwrap(); /// # let p = |n: &str| dir.path().join(n); -/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } -/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; +/// # for n in ["c.cc", "i.cc", "e.cc", "o.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } +/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc"), ontology: &p("o.cc") }; /// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &["en-US".to_string()]).unwrap(); /// let t = run.train().unwrap(); /// let inv = run.inventory().inventory().unwrap(); @@ -523,9 +526,13 @@ impl Prompt { /// [`LOWER_INSTRUCTIONS`], the `language: ` line, the meaning of /// `input` (no kind, constraints, scene, speaker or group), the forms in - /// `language` of every entity its referents cite, and the entries of `inventory` live at the view's revision. `item` - /// goes only to [`Prompt::item_keys`], never into the text. The view's - /// speaker and group fields are never read; a speaker's entity id appears + /// `language` of every entity its referents cite, the `ontology:` section + /// and the entries of `inventory` live at the view's revision. The section + /// holds the entries `ontology::select` chooses from the view's stance and + /// discourse features and `language`, never the caller; the view's speaker + /// and group are still never read. `item` + /// goes only to [`Prompt::item_keys`], never into the text. A speaker's + /// entity id appears /// only when the encoder cites that entity on a referent, which is naming a /// cited entity like any other. A cited entity the port lacks, or one with /// no form in `language`, is an error and no prompt. The caller chooses the @@ -536,6 +543,7 @@ impl Prompt { language: &str, inventory: &Inventory, entities: &dyn EntityForms, + ontology: &Ontology, ) -> Result { let mut forms: BTreeMap<&str, Vec> = BTreeMap::new(); for id in input.referents().iter().filter_map(|r| r.entity.as_ref()) { @@ -556,8 +564,19 @@ impl Prompt { stance: input.stance(), }; let rev = input.inventory_rev(); + let chosen = select(ontology, language, input); + let mut lines = String::new(); + for e in &chosen { + lines.push_str(&format!("\n{} [{}] {}", e.key(), e.layer.name(), e.choice)); + for x in &e.examples { + lines.push_str(&format!(" | e.g. {x}")); + } + } + if chosen.is_empty() { + lines.push_str(" none"); + } let text = format!( - "{LOWER_INSTRUCTIONS}\n\nlanguage: {language}\n\npacket: {}\n\nentities: {}\n\ninventory rev:{rev}{}", + "{LOWER_INSTRUCTIONS}\n\nlanguage: {language}\n\npacket: {}\n\nentities: {}\n\nontology:{lines}\n\ninventory rev:{rev}{}", serde_json::to_string(&meaning).expect("a meaning is plain data"), serde_json::to_string(&forms).expect("forms are plain data"), inventory_lines(inventory, rev) diff --git a/crates/weksa/src/encode.rs b/crates/weksa/src/encode.rs index c9b245e..d616d3d 100644 --- a/crates/weksa/src/encode.rs +++ b/crates/weksa/src/encode.rs @@ -19,10 +19,12 @@ use crate::corpus::CorpusItemId; use crate::inventory::{ Change, Family, InvId, Inventory, InventoryEntry, InventoryRevision, Slot, ValueKind, }; +use crate::ontology::{Condition, Layer, OntologyEntry}; use crate::packet::{ Constraints, Context, Discourse, EncoderId, Grade, Packet, PacketId, PacketKind, Predication, Provenance, ProvenanceSource, Referent, Relation, SCHEMA_VERSION, Stance, Trace, }; +use crate::speaker::Annotator; use crate::split::{fnv1a64, hex16}; /// The meaning fields of a packet, as a model writes them. @@ -271,6 +273,59 @@ impl EditDraft { } } +/// One ontology entry as a model writes it: no language, number, inventory +/// revision or annotator. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EntryOntologyDraft { + pub layer: Layer, + #[serde(default)] + pub when: Vec, + pub choice: String, + #[serde(default)] + pub examples: Vec, +} + +/// A batch of ontology entries as a model drafts them: the xenos door of a +/// seed answer. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OntologyDraft { + pub entries: Vec, +} + +impl OntologyDraft { + pub fn parse(answer: &str) -> Result { + from_json(answer) + } + + /// The entries this draft becomes: numbered from `first_n` in draft order, + /// all of `language` read at `inventory_rev`, annotated `annotator`. + /// Whether they are admissible is `OntologyStore::admit`'s to refuse. + pub fn into_entries( + self, + language: &str, + inventory_rev: u32, + first_n: u32, + annotator: &Annotator, + ) -> Vec { + self.entries + .into_iter() + .zip(first_n..) + .map(|(d, n)| OntologyEntry { + language: language.to_string(), + n, + inventory_rev, + layer: d.layer, + when: d.when, + choice: d.choice, + examples: d.examples, + annotator: annotator.clone(), + }) + .collect() + } +} + /// A lowering as the lowerer writes it: the English text, nothing else. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] diff --git a/crates/weksa/src/lib.rs b/crates/weksa/src/lib.rs index efea9ef..2510fae 100644 --- a/crates/weksa/src/lib.rs +++ b/crates/weksa/src/lib.rs @@ -12,6 +12,7 @@ pub mod form; pub mod inventory; pub mod lower; pub mod model; +pub mod ontology; pub mod packet; pub mod phones; pub mod run; @@ -32,6 +33,7 @@ pub use inventory::{ }; pub use lower::{EntityForms, LowerError, PhoneLine, TARGET_LANGUAGE, lower, phones, pronounce}; pub use model::{AnswerFiles, ModelPort}; +pub use ontology::{Condition, Layer, Ontology, OntologyEntry, Test, select}; pub use packet::{ EncoderId, EntityId, Grade, NodeId, Packet, PacketId, PacketKind, SCHEMA_VERSION, Stance, }; @@ -41,7 +43,7 @@ pub use score::{Embedder, ItemScore, OllamaEmbedder, Scorer, chrf}; pub use speaker::{Annotator, PairRelation, Party, Scene, Speaker}; pub use split::{Split, SplitScheme}; pub use store::{ - AddOutcome, CorpusStore, EntityBound, EntityStore, InventoryStore, PacketStore, + AddOutcome, CorpusStore, EntityBound, EntityStore, InventoryStore, OntologyStore, PacketStore, SpeakerStateStore, StoreError, StoreRefusal, }; pub use validate::{LoweringInput, Refusal, SceneRead, validate}; diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 8270565..b9b094e 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -8,6 +8,7 @@ use clap::error::{ContextKind, ContextValue, ErrorKind}; use clap::{Args, Parser, Subcommand, ValueEnum}; use unicode_normalization::UnicodeNormalization; use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn}; +use weksa::encode::OntologyDraft; use weksa::entity::{Entity, EntityOrigin, LanguageForms, Pronunciation}; use weksa::inventory::{proof_revision, seed}; use weksa::packet::proof_packet; @@ -17,7 +18,7 @@ use weksa::run::{ }; use weksa::score::ScoreInput; use weksa::{ - AnswerFiles, CorpusStore, EntityId, EntityStore, InventoryStore, OllamaEmbedder, PacketStore, + Annotator, AnswerFiles, CorpusStore, EntityId, EntityStore, InventoryStore, OllamaEmbedder, OntologyStore, PacketStore, Scorer, Split, SplitScheme, StoreError, lower, pronounce, validate, }; @@ -36,6 +37,9 @@ struct Cli { /// The entity store. #[arg(long, global = true, default_value = "data/entity/entity.cc")] entities: PathBuf, + /// The language ontology store. + #[arg(long, global = true, default_value = "data/ontology/ontology.cc")] + ontology: PathBuf, #[command(subcommand)] command: Command, } @@ -51,6 +55,8 @@ enum Command { #[command(subcommand)] Entity(EntityCommand), #[command(subcommand)] + Ontology(OntologyCommand), + #[command(subcommand)] Packet(PacketCommand), /// Lower a stored packet to English text. Lower { @@ -81,7 +87,7 @@ enum Command { #[derive(Subcommand)] enum LoopCommand { - /// Pin the global --corpus, --inventory and --entities as a run at ROOT. + /// Pin the global --corpus, --inventory, --entities and --ontology as a run at ROOT. Open { root: PathBuf, #[arg(long)] @@ -105,8 +111,8 @@ enum LoopCommand { panel: Vec, }, /// Run one round of the run at ROOT. The stores are read only from the - /// run's record: the global --corpus, --inventory and --entities are not - /// read. Encode prompts are written first; once they are answered a rerun + /// run's record: the global --corpus, --inventory, --entities and --ontology + /// are not read. Encode prompts are written first; once they are answered a rerun /// writes the lowering prompts. The command exits 3 each time until every /// prompt has an answer. Round { @@ -195,6 +201,28 @@ enum EntityCommand { Dump, } +#[derive(Subcommand)] +enum OntologyCommand { + /// Admit a model's ontology draft (a JSON file) for one language as a + /// batch, numbered from the language's latest plus one and annotated as + /// the model that drafted it. + Admit { + answer: PathBuf, + #[arg(long)] + language: String, + /// The inventory revision the draft's conditions are read at. + #[arg(long)] + inventory_rev: u32, + #[arg(long)] + model_family: ModelFamilyArg, + /// The attestation string naming the drafting model. + #[arg(long)] + generator: String, + }, + /// Text projection of the ontology for review; not a store. + Dump, +} + #[derive(Args)] struct PutArgs { /// `entity:`. @@ -648,6 +676,48 @@ fn main() -> Result<()> { println!("{} {:?}\n forms: {:?}", e.id, e.origin, e.forms); } } + Command::Ontology(OntologyCommand::Admit { + answer, + language, + inventory_rev, + model_family, + generator, + }) => { + let text = std::fs::read_to_string(&answer).context("the draft would not read")?; + let draft = OntologyDraft::parse(&text)?; + let mut store = OntologyStore::open(&cli.ontology)?; + let first = store + .all()? + .iter() + .filter(|e| e.language == language) + .map(|e| e.n) + .max() + .unwrap_or(0) + + 1; + let annotator = Annotator::Model { + family: model_family.into(), + generator, + }; + let entries = draft.into_entries(&language, inventory_rev, first, &annotator); + let count = entries.len(); + let inventory = InventoryStore::open(&cli.inventory)?; + let outcome = store.admit(entries, &inventory).map_err(refused)?; + println!("{outcome:?} {count} entries of {language}"); + } + Command::Ontology(OntologyCommand::Dump) => { + for e in OntologyStore::open(&cli.ontology)?.all()? { + println!( + "{} {} rev:{} {:?}\n when: {:?}\n {}\n e.g. {:?}", + e.key(), + e.layer.name(), + e.inventory_rev, + e.annotator, + e.when, + e.choice, + e.examples + ); + } + } Command::Inventory(InventoryCommand::Seed) => { let (rev, entries) = seed(); let count = entries.len(); @@ -779,6 +849,7 @@ fn main() -> Result<()> { corpus: &cli.corpus, inventory: &cli.inventory, entities: &cli.entities, + ontology: &cli.ontology, }, &answerer, &embed_model, diff --git a/crates/weksa/src/ontology.rs b/crates/weksa/src/ontology.rs new file mode 100644 index 0000000..4bc38fb --- /dev/null +++ b/crates/weksa/src/ontology.rs @@ -0,0 +1,164 @@ +//! The language ontology: per target language, the choices its culture makes +//! about word, grammar, discourse, sound and script for a speaker in a stance +//! (ruling `persona-drives-ontology`). An entry says when it applies (feature +//! conditions over the lowering view's stance and discourse) and what it +//! chooses. `select` is the only reader that decides which entries a lowering +//! prompt carries, and it runs only inside `Prompt::lower`; +//! `OntologyStore::admit` is the only writer. Entries are immutable. + +use cultcache_rs::DatabaseEntry; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::inventory::{InvId, Slot}; +use crate::packet::{FeatValue, FeatureValue, Grade}; +use crate::speaker::Annotator; +use crate::validate::LoweringInput; + +pub const MAX_CONDITIONS: usize = 4; +pub const MAX_EXAMPLES: usize = 3; +pub const MAX_CHOICE_BYTES: usize = 200; +pub const MAX_EXAMPLE_BYTES: usize = 120; +pub const MAX_ENTRIES_PER_LANGUAGE: usize = 64; + +/// Which part of the language an entry shapes. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub enum Layer { + Lexicon, + Grammar, + Discourse, + Phonology, + Orthography, +} + +impl Layer { + pub fn name(self) -> &'static str { + match self { + Layer::Lexicon => "lexicon", + Layer::Grammar => "grammar", + Layer::Discourse => "discourse", + Layer::Phonology => "phonology", + Layer::Orthography => "orthography", + } + } +} + +/// What a feature's value must be for a condition to hold. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub enum Test { + AtLeast(Grade), + AtMost(Grade), + Is(FeatValue), +} + +/// One feature of one slot of the view, tested. Only `Stance` and `Discourse` +/// reach a lowering view today. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct Condition { + pub slot: Slot, + pub feature: InvId, + pub test: Test, +} + +impl Condition { + fn holds(&self, features: &[FeatureValue]) -> bool { + let Some(found) = features.iter().find(|f| f.feature == self.feature) else { + return false; + }; + match (&self.test, &found.value) { + (Test::AtLeast(g), FeatValue::Grade(v)) => v >= g, + (Test::AtMost(g), FeatValue::Grade(v)) => v <= g, + (Test::Is(want), v) => want == v, + _ => false, + } + } + + /// The feature value this condition names, as the inventory check reads it. + pub(crate) fn as_value(&self) -> FeatureValue { + FeatureValue { + feature: self.feature, + value: match &self.test { + Test::AtLeast(g) | Test::AtMost(g) => FeatValue::Grade(*g), + Test::Is(v) => v.clone(), + }, + } + } +} + +/// One choice of one language. Key `onto::`. +#[derive(Clone, Debug, PartialEq, Eq, JsonSchema, DatabaseEntry)] +#[cultcache(type = "weksa.ontology_entry")] +pub struct OntologyEntry { + #[cultcache(key = 0)] + pub language: String, + /// The entry's number within its language, from 1. + #[cultcache(key = 1)] + pub n: u32, + /// The inventory revision the conditions read against. + #[cultcache(key = 2)] + pub inventory_rev: u32, + #[cultcache(key = 3)] + pub layer: Layer, + /// Every condition must hold; none means the entry always applies. + #[cultcache(key = 4)] + pub when: Vec, + #[cultcache(key = 5)] + pub choice: String, + #[cultcache(key = 6)] + pub examples: Vec, + #[cultcache(key = 7)] + pub annotator: Annotator, +} + +/// `onto::`. +pub fn ontology_key(language: &str, n: u32) -> String { + format!("onto:{language}:{n}") +} + +impl OntologyEntry { + pub fn key(&self) -> String { + ontology_key(&self.language, self.n) + } +} + +/// Every entry of the run's ontology store, by language then number. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Ontology { + entries: Vec, +} + +impl Ontology { + pub fn empty() -> Self { + Self::default() + } + + pub(crate) fn of(mut entries: Vec) -> Self { + entries.sort_by(|a, b| (&a.language, a.n).cmp(&(&b.language, b.n))); + Self { entries } + } + + pub fn entries(&self) -> &[OntologyEntry] { + &self.entries + } +} + +/// The entries of exactly `language` whose every condition holds against the +/// view's stance and discourse features, in `n` order. An absent feature meets +/// no condition. Nothing else of the view is read. +pub fn select<'a>( + o: &'a Ontology, + language: &str, + input: &LoweringInput, +) -> Vec<&'a OntologyEntry> { + o.entries + .iter() + .filter(|e| e.language == language) + .filter(|e| { + e.when.iter().all(|c| match c.slot { + Slot::Stance => c.holds(&input.stance().features), + Slot::Discourse => c.holds(&input.discourse().features), + _ => false, + }) + }) + .collect() +} diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index a1b879d..0e63af4 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -47,7 +47,8 @@ use crate::packet::{Packet, Stance}; use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate}; use crate::split::{Split, SplitScheme, fnv1a64, fnv1a64_of, hex16}; use crate::store::{ - CorpusStore, EntityStore, InventoryStore, PacketStore, StoreError, open_cache_over, + CorpusStore, EntityStore, InventoryStore, OntologyStore, PacketStore, StoreError, + check_ontology_entry, open_cache_over, }; use crate::validate::LoweringInput; @@ -55,7 +56,9 @@ use crate::validate::LoweringInput; /// whose scheme and digest function decide which lines the prompt holds as /// Train and name every prompt, then `lower.rs`, whose `EntityForms` port and /// `entity()` the lowering prompt reads, then `packet.rs`, whose serde -/// shapes the lowering prompt prints as the packet. Each file enters the digest +/// shapes the lowering prompt prints as the packet, then `ontology.rs`, whose +/// entry shape and `select` decide which ontology lines the lowering prompt +/// prints. Each file enters the digest /// behind its length, so the parts cannot be re-cut into other parts with the /// same digest. Any edit to any of them, comments included, ends a run; code /// outside those files is not covered. @@ -64,6 +67,7 @@ pub const PROMPT_SOURCE_DIGEST: u64 = fnv1a64_of(&[ include_bytes!("split.rs"), include_bytes!("lower.rs"), include_bytes!("packet.rs"), + include_bytes!("ontology.rs"), ]); /// The least composite gain an edit must show over item noise to be kept, and @@ -125,6 +129,13 @@ pub struct RunRecord { /// language is never offered, minted from, probed or scored. #[cultcache(key = 14)] pub panel: Vec, + /// Canonical absolute path of the ontology store: read-only for the run, + /// pinned by `ontology_digest`. + #[cultcache(key = 15)] + pub ontology: String, + /// 16 hex of the ontology file's bytes. + #[cultcache(key = 16)] + pub ontology_digest: String, } /// How a run's lowering prompts reach the model that answers them (rulings @@ -429,6 +440,7 @@ impl RoundRecord { pub enum DriftOf { Corpus, Entities, + Ontology, Prompts, Model, Embedder, @@ -464,6 +476,9 @@ pub enum RunError { /// The panel is empty, repeats or misspells a language, or names one with no /// live Train or Validation item in the pinned corpus. Panel, + /// An ontology entry of a panel language does not read against the run's + /// head inventory at its revision. + Ontology, } impl fmt::Display for RunError { @@ -473,7 +488,7 @@ impl fmt::Display for RunError { RunError::NoRun => f.write_str("the root holds no run"), RunError::Malformed(what) => write!(f, "malformed {what}"), RunError::Drift(DriftOf::Prompts) => f.write_str( - "drift in Prompts (the prompt source digest covers context.rs, split.rs, lower.rs and packet.rs)", + "drift in Prompts (the prompt source digest covers context.rs, split.rs, lower.rs, packet.rs and ontology.rs)", ), RunError::Drift(of) => write!(f, "drift in {of:?}"), RunError::NoInventory => f.write_str("the inventory holds no revision"), @@ -485,6 +500,9 @@ impl fmt::Display for RunError { RunError::Embed => f.write_str("the embedder failed"), RunError::NoFloor => f.write_str("the run has no noise floor: record two rounds first"), RunError::NoValidation => f.write_str("a reading has no validation item"), + RunError::Ontology => f.write_str( + "an ontology entry of a panel language does not read against the run's inventory", + ), RunError::Panel => f.write_str( "the panel is empty, repeats or misspells a language, or names one with no live Train or Validation item", ), @@ -500,12 +518,13 @@ impl fmt::Debug for RunError { impl std::error::Error for RunError {} -/// The three stores a run pins. +/// The four stores a run pins. #[derive(Clone, Copy)] pub struct RunPaths<'a> { pub corpus: &'a Path, pub inventory: &'a Path, pub entities: &'a Path, + pub ontology: &'a Path, } /// A run's open stores, reopened only from its record. The one public producer @@ -517,6 +536,7 @@ pub struct RunStores { corpus: CorpusStore, inventory: InventoryStore, entities: EntityStore, + ontology: OntologyStore, } /// The run's own copy of the inventory, made at `create`. @@ -530,6 +550,9 @@ fn check_pins(cache: &CultCache, root: &Path, record: &RunRecord) -> Result<(), if file_digest(&record.entities).as_ref() != Some(&record.entity_digest) { return Err(RunError::Drift(DriftOf::Entities)); } + if file_digest(&record.ontology).as_ref() != Some(&record.ontology_digest) { + return Err(RunError::Drift(DriftOf::Ontology)); + } let (path, digest, _) = head_of(cache, root, record)?; if file_digest_at(&path).as_ref() != Some(&digest) { return Err(RunError::Drift(DriftOf::Inventory)); @@ -663,6 +686,8 @@ impl RunStores { corpus: canonical(paths.corpus)?, inventory: canonical(paths.inventory)?, entities: canonical(paths.entities)?, + ontology: canonical(paths.ontology)?, + ontology_digest: String::new(), corpus_digest: String::new(), entity_digest: String::new(), prompt_digest: hex16(PROMPT_SOURCE_DIGEST), @@ -684,11 +709,13 @@ impl RunStores { corpus_digest: file_digest(&record.corpus).ok_or(RunError::Store)?, entity_digest: file_digest(&record.entities).ok_or(RunError::Store)?, inventory_digest: file_digest_at(&head).ok_or(RunError::Store)?, + ontology_digest: file_digest(&record.ontology).ok_or(RunError::Store)?, ..record }; between(); let mut run = Self::with(root, cache, record)?; run.check_panel()?; + run.check_ontology()?; // The stores were read between the digests: unless they still match // the pins, what was opened is not what was pinned. Nothing is // recorded for a run that did not pin. @@ -739,6 +766,7 @@ impl RunStores { corpus: CorpusStore::open(Path::new(&record.corpus)).map_err(store)?, inventory: InventoryStore::open(&head).map_err(store)?, entities: EntityStore::open(Path::new(&record.entities)).map_err(store)?, + ontology: OntologyStore::open(Path::new(&record.ontology)).map_err(store)?, cache, record, }) @@ -748,6 +776,18 @@ impl RunStores { &self.record } + /// Every ontology entry of a panel language reads against the run's head + /// inventory at its own revision: the check the store applied at admission, + /// applied again to this inventory. + fn check_ontology(&self) -> Result<(), RunError> { + let inventory = self.inventory.inventory().map_err(|_| RunError::Store)?; + let entries = self.ontology.all().map_err(|_| RunError::Store)?; + entries + .iter() + .filter(|e| self.record.panel.contains(&e.language)) + .try_for_each(|e| check_ontology_entry(e, &inventory).map_err(|_| RunError::Ontology)) + } + /// Every panel language holds a live Train and a live Validation item. fn check_panel(&self) -> Result<(), RunError> { let items = self.corpus.all().map_err(|_| RunError::Store)?; @@ -1171,7 +1211,9 @@ impl RunStores { /// Train and Validation packet of plain round `n`, per panel language, asks the lowerer again /// (`model`, the replay), with the packet's stance emptied (drop) and with /// the next item's stance in its split (permute), and reads how far the - /// lowering moves against the round's own answer (`recorded`). Test and + /// lowering moves against the round's own answer (`recorded`). The variant + /// stance also changes which ontology entries are selected, so the figure is + /// the stance's whole effect on the lowering, packet and ontology together. Test and /// withdrawn items are never asked. Records nothing and writes nothing in /// `round-`; the variant stores are rebuilt each call. An answer that /// is not a usable lowering drops its pair and is counted in `unusable`. @@ -1190,6 +1232,7 @@ impl RunStores { .find(|r| r.n() == n && r.cycle().is_none()) .ok_or(RunError::Malformed("round"))?; let inventory = self.inventory.inventory().map_err(store)?; + let ontology = self.ontology.ontology().map_err(store)?; let dir = self.ablation_dir(n); std::fs::create_dir_all(&dir).map_err(|e| RunError::Io(e.kind()))?; @@ -1242,7 +1285,14 @@ impl RunStores { let view = stores[slot] .admit(packet, &self.inventory, &self.corpus, &self.entities) .map_err(|_| RunError::Store)?; - Prompt::lower(&view, &packet_item, &item.language, &inventory, &self.entities) + Prompt::lower( + &view, + &packet_item, + &item.language, + &inventory, + &self.entities, + &ontology, + ) .map_err(|_| RunError::Store) }; let base = prompt_of(0, packet.clone())?; @@ -1444,6 +1494,7 @@ fn score_round( ) -> Result { let store = |_| RunError::Store; let inventory = inventory_store.inventory().map_err(store)?; + let ontology = run.ontology.ontology().map_err(store)?; let rev = inventory.latest().ok_or(RunError::NoInventory)?.n; let train = run.train().map_err(store)?; @@ -1524,7 +1575,7 @@ fn score_round( // Each item is lowered in its own language: the only reference the // scorer has is the item's own text. let language = &items[*at].language; - let Ok(prompt) = Prompt::lower(view, id, language, &inventory, &run.entities) else { + let Ok(prompt) = Prompt::lower(view, id, language, &inventory, &run.entities, &ontology) else { fates[*at] = Fate::Refused(Stage::Lower, Some(packet_key.clone())); continue; }; @@ -2725,6 +2776,8 @@ mod tests { lowerer_reach: LowererReach::ToolLessRelay, answerer_family: ModelFamily::Unrecorded, panel: vec!["en-US".into()], + ontology: String::new(), + ontology_digest: String::new(), } } @@ -2915,6 +2968,8 @@ mod tests { copy("inventory.cc", "i.cc"), copy("entity.cc", "e.cc"), ); + let ontology = dir.path().join("o.cc"); + fs::write(&ontology, [0x90]).unwrap(); let root = dir.path().join("run"); let mut run = RunStores::create( &root, @@ -2923,6 +2978,7 @@ mod tests { corpus: &corpus, inventory: &inventory, entities: &entities, + ontology: &ontology, }, "fake-model", "bag", diff --git a/crates/weksa/src/schema.rs b/crates/weksa/src/schema.rs index 21add8f..4fea40d 100644 --- a/crates/weksa/src/schema.rs +++ b/crates/weksa/src/schema.rs @@ -4,6 +4,7 @@ use crate::corpus::CorpusItem; use crate::entity::Entity; use crate::inventory::{InventoryEntry, InventoryRevision}; +use crate::ontology::OntologyEntry; use crate::packet::Packet; use crate::run::{Decision, RoundRecord, RunRecord}; use crate::speaker::{Scene, Speaker}; @@ -25,6 +26,7 @@ pub fn published() -> Vec<(String, String)> { published_for::(), published_for::(), published_for::(), + published_for::(), published_for::(), published_for::(), published_for::(), diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index 16b7ccd..9183f2f 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -19,7 +19,11 @@ use crate::form::{ use crate::inventory::{ Change, InvId, Inventory, InventoryEntry, InventoryRevision, Slot, revision_key, }; -use crate::packet::{EntityId, Packet}; +use crate::ontology::{ + MAX_CONDITIONS, MAX_CHOICE_BYTES, MAX_ENTRIES_PER_LANGUAGE, MAX_EXAMPLE_BYTES, MAX_EXAMPLES, + Ontology, OntologyEntry, Test, +}; +use crate::packet::{EntityId, FeatValue, Packet}; use crate::speaker::{Annotator, MAX_PARTICIPANTS, Party, Scene, Speaker, scene_key, speaker_key}; use crate::validate::{LoweringInput, Refusal, feature_refusals, validate}; @@ -1207,3 +1211,131 @@ fn check_next(key: &str, latest: u32, got: u32) -> Result<(), StoreError> { } Ok(()) } + +/// The language ontology: immutable entries admitted in all-or-nothing +/// batches. The only writer of `weksa.ontology_entry`. +pub struct OntologyStore { + cache: CultCache, +} + +impl OntologyStore { + pub fn open(path: &Path) -> anyhow::Result { + Ok(Self { + cache: open_cache(path, |c| c.register_entry_type::())?, + }) + } + + /// Every entry, by language then number. + pub fn all(&self) -> anyhow::Result> { + Ok(self.ontology()?.entries().to_vec()) + } + + pub fn ontology(&self) -> anyhow::Result { + Ok(Ontology::of(self.cache.get_all::()?)) + } + + /// Admits a batch whole or not at all; an identical replay of stored + /// entries is `Unchanged`. Per language the batch's numbers run the + /// language's latest plus one, consecutively, to at most + /// `MAX_ENTRIES_PER_LANGUAGE` entries in all. Every entry passes + /// [`check_ontology_entry`] against the inventory before any is put. No + /// message echoes an entry's text. + pub fn admit( + &mut self, + batch: Vec, + inventory: &InventoryStore, + ) -> Result { + let all = self.cache.get_all::()?; + if batch + .iter() + .all(|e| all.iter().any(|old| old == e)) + { + return Ok(AddOutcome::Unchanged); + } + let inv = inventory.inventory()?; + let mut counted: std::collections::BTreeMap<&str, u32> = std::collections::BTreeMap::new(); + for e in &batch { + let key = e.key(); + if !is_language_tag(&e.language) { + return Err(malformed(&key, "an ontology language is a language tag")); + } + let latest = match counted.get(e.language.as_str()) { + Some(n) => *n, + None => all + .iter() + .filter(|o| o.language == e.language) + .map(|o| o.n) + .max() + .unwrap_or(0), + }; + check_next(&key, latest, e.n)?; + if e.n as usize > MAX_ENTRIES_PER_LANGUAGE { + return Err(malformed(&key, "a language holds at most 64 entries")); + } + counted.insert(e.language.as_str(), e.n); + check_ontology_entry(e, &inv)?; + } + let mut staged = Vec::new(); + for e in &batch { + staged.push(self.cache.prepare_entry_named(e.key(), e)?.0); + } + self.cache.put_prepared_batch(staged)?; + Ok(AddOutcome::Added) + } +} + +/// The rules of one ontology entry against an inventory: the annotator, the +/// bounded and normalized text of the choice and examples, the condition count +/// and each condition's feature (slot `Stance` or `Discourse`, a grade feature +/// for `AtLeast` and `AtMost`, an option or flag value for `Is`) read at the +/// entry's inventory revision. The store applies it on admission and a run +/// applies it again against its own head inventory. +pub(crate) fn check_ontology_entry( + e: &OntologyEntry, + inventory: &Inventory, +) -> Result<(), StoreError> { + let key = e.key(); + check_annotator(&key, &e.annotator)?; + let text_ok = |t: &str, max: usize| { + (1..=max).contains(&t.len()) && !t.chars().any(char::is_control) && is_nfc(t) + }; + if !text_ok(&e.choice, MAX_CHOICE_BYTES) { + return Err(malformed( + &key, + "a choice is 1 to 200 normalized bytes with no control character", + )); + } + if e.examples.len() > MAX_EXAMPLES + || !e.examples.iter().all(|x| text_ok(x, MAX_EXAMPLE_BYTES)) + { + return Err(malformed( + &key, + "an entry has at most 3 examples of 1 to 120 normalized bytes with no control character", + )); + } + if e.when.len() > MAX_CONDITIONS { + return Err(malformed(&key, "an entry has at most 4 conditions")); + } + let mut out = Vec::new(); + for c in &e.when { + if !matches!(c.slot, Slot::Stance | Slot::Discourse) { + return Err(malformed(&key, "a condition reads slot Stance or Discourse")); + } + if matches!(c.test, Test::Is(FeatValue::Grade(_))) { + return Err(malformed(&key, "Is names an option or a flag")); + } + out.extend(feature_refusals( + inventory, + e.inventory_rev, + &[c.as_value()], + c.slot, + )); + } + if e.when.is_empty() && !inventory.has_revision(e.inventory_rev) { + out.extend(feature_refusals(inventory, e.inventory_rev, &[], Slot::Stance)); + } + if !out.is_empty() { + return Err(StoreError::Invalid(out)); + } + Ok(()) +} diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 5f75500..605d6a7 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -12,6 +12,8 @@ fn weksa(dir: &std::path::Path, args: &[&str]) -> Output { "./i.cc", "--entities", "./e.cc", + "--ontology", + "./o.cc", ]) .args(args) .output() @@ -316,6 +318,7 @@ fn schema_writes_the_published_files() { "weksa.loop_decision.schema.json", "weksa.loop_round.schema.json", "weksa.loop_run.schema.json", + "weksa.ontology_entry.schema.json", "weksa.scene.schema.json", "weksa.speaker.schema.json" ] @@ -1337,6 +1340,7 @@ fn dao_stores(dir: &std::path::Path) { ] { std::fs::copy(src.join(from), dir.join(to)).unwrap(); } + std::fs::write(dir.join("o.cc"), [0x90]).unwrap(); } #[test] diff --git a/crates/weksa/tests/common/mod.rs b/crates/weksa/tests/common/mod.rs index b7b7c7e..8350428 100644 --- a/crates/weksa/tests/common/mod.rs +++ b/crates/weksa/tests/common/mod.rs @@ -114,6 +114,15 @@ pub fn entities_in(dir: &std::path::Path) -> EntityStore { store } +/// An empty ontology store at `/ontology.cc`, made if absent. +pub fn ontology_in(dir: &std::path::Path) -> std::path::PathBuf { + let path = dir.join("ontology.cc"); + if !path.exists() { + std::fs::write(&path, [0x90]).unwrap(); + } + path +} + /// A run over the stores `world()` and `corpus()` keep in `dir`: `c.cc`, /// `i.cc` and `entities.cc`. Read at the call, so create it after the items. /// The run's en-US panel needs a live Train and a live Validation item: a @@ -125,6 +134,7 @@ pub fn run_over(dir: &std::path::Path) -> weksa::run::RunStores { } } let entities = entities_in(dir); + ontology_in(dir); let mut corpus = CorpusStore::open(&dir.join("c.cc")).unwrap(); for (n, split) in [(1, weksa::Split::Train), (2, weksa::Split::Validation)] { let held = corpus @@ -147,6 +157,7 @@ pub fn run_over(dir: &std::path::Path) -> weksa::run::RunStores { corpus: &dir.join("c.cc"), inventory: &dir.join("i.cc"), entities: &dir.join("entities.cc"), + ontology: &dir.join("ontology.cc"), }, "fake-model", "bag", @@ -230,6 +241,7 @@ pub fn world() -> World { let dir = tempfile::tempdir().unwrap(); let mut corpus = CorpusStore::open(&dir.path().join("c.cc")).unwrap(); let entities = entities_in(dir.path()); + ontology_in(dir.path()); let train = group_in_bucket(10); let mut first = item_in("item:nibu:1", "Nibu says mind the hatch.", &train, cast()); first.names = vec![NameSpan { diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs index 0103149..61e86f7 100644 --- a/crates/weksa/tests/context.rs +++ b/crates/weksa/tests/context.rs @@ -602,7 +602,7 @@ mod lowering { fn lower_prompt_names_its_language() { let (view, inv, entities, item) = dao(2); for tag in ["en-US", "ja-JP", "pt-BR"] { - let prompt = Prompt::lower(&view, &item, tag, &inv, &entities).unwrap(); + let prompt = Prompt::lower(&view, &item, tag, &inv, &entities, &weksa::Ontology::empty()).unwrap(); let want = format!("{LOWER}\n\nlanguage: {tag}\n\npacket: "); assert!(prompt.text().starts_with(&want), "{tag}"); } @@ -614,10 +614,10 @@ mod lowering { #[test] fn lower_prompt_text_is_pinned() { let (view, inv, entities, item) = dao(2); - let prompt = Prompt::lower(&view, &item, "en-US", &inv, &entities).unwrap(); + let prompt = Prompt::lower(&view, &item, "en-US", &inv, &entities, &weksa::Ontology::empty()).unwrap(); let (head, listing) = prompt.text().split_once("inventory rev:3\n").unwrap(); let expected = format!( - "{LOWER}\n\nlanguage: en-US\n\npacket: {}\n\nentities: {{}}\n\n", + "{LOWER}\n\nlanguage: en-US\n\npacket: {}\n\nentities: {{}}\n\nontology: none\n\n", concat!( r#"{"context":{"setting":null,"medium":null,"participants":[],"audience":[]},"#, r#""discourse":{"speech_act":"speech_act:1","addressee":null,"addressee_consent":null,"features":[]},"#, @@ -676,7 +676,7 @@ mod lowering { let mut packet = packets().remove(2); packet.stance = stance.clone(); let view = admitted(packet, &store, "daodejing/corpus.cc", &entities); - let prompt = Prompt::lower(&view, &item, "en-US", &inv, &entities).unwrap(); + let prompt = Prompt::lower(&view, &item, "en-US", &inv, &entities, &weksa::Ontology::empty()).unwrap(); (stance, prompt) }; let three = [prompt_of(own), prompt_of(angry), prompt_of(Stance::default())]; @@ -713,7 +713,7 @@ mod lowering { ("sa-Latn", &["Nibuwa"]), ("en-US", &["Nibu", "Nibu's"]), ])]); - let prompt = Prompt::lower(&view, &item, "en-US", &cast.inventory().unwrap(), &held).unwrap(); + let prompt = Prompt::lower(&view, &item, "en-US", &cast.inventory().unwrap(), &held, &weksa::Ontology::empty()).unwrap(); assert!( prompt .text() @@ -743,7 +743,7 @@ mod lowering { ), (Held(vec![]), LowerError::UnknownEntity { entity: id }), ] { - assert_eq!(Prompt::lower(&view, &item, "en-US", &inv, &held).err(), Some(want)); + assert_eq!(Prompt::lower(&view, &item, "en-US", &inv, &held, &weksa::Ontology::empty()).err(), Some(want)); } } } diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 73f1df2..38ae231 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -24,8 +24,13 @@ fn rewrite_run(root: &Path, change: impl FnOnce(&mut RunRecord)) { cache.put(format!("run:{}", record.label), &record).unwrap(); } -fn paths(dir: &Path) -> (PathBuf, PathBuf, PathBuf) { - (dir.join("c.cc"), dir.join("i.cc"), dir.join("entities.cc")) +fn paths(dir: &Path) -> (PathBuf, PathBuf, PathBuf, PathBuf) { + ( + dir.join("c.cc"), + dir.join("i.cc"), + dir.join("entities.cc"), + common::ontology_in(dir), + ) } fn hex(path: &Path) -> String { @@ -37,7 +42,7 @@ fn a_run_pins_its_stores_and_refuses_drift() { let mut w = world(); let dir = w.dir.path().to_path_buf(); let root = dir.join("run"); - let (c, i, e) = paths(&dir); + let (c, i, e, o) = paths(&dir); let at = |label: &str, model: &str| { RunStores::create( &root, @@ -46,6 +51,7 @@ fn a_run_pins_its_stores_and_refuses_drift() { corpus: &c, inventory: &i, entities: &e, + ontology: &o, }, model, "bag", @@ -141,7 +147,7 @@ fn a_run_works_on_its_own_copy_of_the_inventory() { let w = world(); let dir = w.dir.path().to_path_buf(); let root = dir.join("run"); - let (c, i, e) = paths(&dir); + let (c, i, e, o) = paths(&dir); let source_bytes = std::fs::read(&i).unwrap(); let mut run = RunStores::create( &root, @@ -150,6 +156,7 @@ fn a_run_works_on_its_own_copy_of_the_inventory() { corpus: &c, inventory: &i, entities: &e, + ontology: &o, }, "fake-model", "bag", @@ -228,6 +235,7 @@ fn create_refuses_a_root_that_holds_an_inventory_and_never_truncates_it() { corpus: &dao.corpus, inventory, entities: &dao.entities, + ontology: &dao.ontology, }, "fake-model", "bag", @@ -276,6 +284,7 @@ fn a_create_that_fails_leaves_no_inventory_and_the_retry_succeeds() { corpus: &dao.corpus, inventory, entities, + ontology: &dao.ontology, }, "fake-model", "bag", @@ -388,10 +397,12 @@ fn a_store_changed_between_its_digest_and_its_open_is_drift() { assert!(dir.join(name).exists(), "{name}"); } let (c, i, e) = (dir.join("c.cc"), dir.join("i.cc"), dir.join("entities.cc")); + let o = dir.join("ontology.cc"); let paths = || weksa::run::RunPaths { corpus: &c, inventory: &i, entities: &e, + ontology: &o, }; assert_eq!( RunStores::create_between( @@ -440,10 +451,12 @@ fn window( let dir = w.dir.path().to_path_buf(); let root = dir.join("run"); let (c, i, e) = (dir.join("c.cc"), dir.join("i.cc"), dir.join("entities.cc")); + let o = dir.join("ontology.cc"); let paths = weksa::run::RunPaths { corpus: &c, inventory: &i, entities: &e, + ontology: &o, }; let created = RunStores::create_between( &root, @@ -501,12 +514,13 @@ fn source_digests_cover_their_files() { let digest = |name: &str| fnv1a64(&read(name)); // The prompt's partition and names come from split.rs, and the lowering // prompt reads lower.rs's target language and entity port and prints - // packet.rs's serde shapes: all four are covered. - let (context, split, lower, packet) = ( + // packet.rs's serde shapes and ontology.rs's selection: all five are covered. + let (context, split, lower, packet, ontology) = ( read("context.rs"), read("split.rs"), read("lower.rs"), read("packet.rs"), + read("ontology.rs"), ); // Each file enters behind its length, so the parts have one framing. let framed = [ @@ -518,12 +532,14 @@ fn source_digests_cover_their_files() { &lower, (packet.len() as u64).to_le_bytes().as_slice(), &packet, + (ontology.len() as u64).to_le_bytes().as_slice(), + &ontology, ] .concat(); assert_eq!(weksa::run::PROMPT_SOURCE_DIGEST, fnv1a64(&framed)); assert_ne!( weksa::run::PROMPT_SOURCE_DIGEST, - fnv1a64(&[context.clone(), split.clone(), lower.clone(), packet.clone()].concat()), + fnv1a64(&[context.clone(), split.clone(), lower.clone(), packet.clone(), ontology.clone()].concat()), "bare concatenation is not the digest" ); // Moving the boundary between the parts moves the digest. @@ -532,6 +548,7 @@ fn source_digests_cover_their_files() { weksa::split::fnv1a64_of(&[b"a", b"bc"]) ); assert_ne!(weksa::run::PROMPT_SOURCE_DIGEST, digest("context.rs")); + assert_ne!(weksa::run::PROMPT_SOURCE_DIGEST, digest("ontology.rs")); } /// The encode prompt of `item:nibu:1` over `world()` at revision 4. @@ -837,6 +854,7 @@ struct Dao { corpus: PathBuf, inventory: PathBuf, entities: PathBuf, + ontology: PathBuf, run: RunStores, } @@ -866,6 +884,7 @@ fn dao_panel(edit: impl FnOnce(&Path, &Path), panel: &[&str]) -> Dao { copy("entity.cc", "e.cc"), ); edit(&corpus, &entities); + let ontology = common::ontology_in(dir.path()); let root = dir.path().join("run"); let run = RunStores::create( &root, @@ -874,6 +893,7 @@ fn dao_panel(edit: impl FnOnce(&Path, &Path), panel: &[&str]) -> Dao { corpus: &corpus, inventory: &inventory, entities: &entities, + ontology: &ontology, }, "fake-model", "bag", @@ -888,6 +908,7 @@ fn dao_panel(edit: impl FnOnce(&Path, &Path), panel: &[&str]) -> Dao { corpus, inventory, entities, + ontology, run, } } @@ -1055,7 +1076,7 @@ fn lowering_prompts(dao: &Dao, inventory: &InventoryStore) -> Vec<(String, Promp let view = store .admit(packet, inventory, &corpus, dao.run.entities()) .unwrap(); - let prompt = Prompt::lower(&view, &key, "en-US", &inv, dao.run.entities()).unwrap(); + let prompt = Prompt::lower(&view, &key, "en-US", &inv, dao.run.entities(), &weksa::Ontology::empty()).unwrap(); (key.to_string(), prompt) }) .collect() @@ -1527,6 +1548,7 @@ fn errors_echo_no_input() { corpus: &missing, inventory: &missing, entities: &missing, + ontology: &missing, }; let e = RunStores::create( &root, @@ -1595,6 +1617,7 @@ fn errors_echo_no_input() { corpus: &dao.corpus, inventory: &dao.inventory, entities: &dao.entities, + ontology: &dao.ontology, }, "fake-model", "bag", @@ -2516,6 +2539,7 @@ fn a_changed_head_is_drift() { corpus: &dao.corpus, inventory: &dao.inventory, entities: &dao.entities, + ontology: &dao.ontology, }, "fake-model", "bag", @@ -2854,6 +2878,7 @@ fn a_summary_line_says_how_lowerings_were_answered() { corpus: &dao.corpus, inventory: &dao.inventory, entities: &dao.entities, + ontology: &dao.ontology, }; RunStores::create( &root, @@ -3577,6 +3602,7 @@ fn mixed_family_round(family: ModelFamily) -> u32 { corpus: &dir.join("c.cc"), inventory: &dir.join("i.cc"), entities: &dir.join("entities.cc"), + ontology: &dir.join("ontology.cc"), }, "not-a-draft", "bag", @@ -4363,7 +4389,7 @@ fn open_refuses_a_bad_panel() { let created = RunStores::create( &root, "p", - RunPaths { corpus, inventory, entities }, + RunPaths { corpus, inventory, entities, ontology: &dao.ontology }, "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 9213939..2194a28 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -274,6 +274,8 @@ fn the_recall_probe_covers_every_live_original_item_and_names_no_work() { tmp.path().join(f) }; let (c, i, e) = (path("corpus.cc"), path("inventory.cc"), path("entity.cc")); + let o = tmp.path().join("ontology.cc"); + std::fs::write(&o, [0x90]).unwrap(); let run = RunStores::create( &tmp.path().join("run"), "orig", @@ -281,6 +283,7 @@ fn the_recall_probe_covers_every_live_original_item_and_names_no_work() { corpus: &c, inventory: &i, entities: &e, + ontology: &o, }, "m", "bag", @@ -350,6 +353,7 @@ fn original_round(family: ModelFamily) -> (weksa::run::RunStores, weksa::run::Ro for f in ["corpus.cc", "entity.cc", "inventory.cc"] { std::fs::copy(src.join(f), tmp.join(f)).unwrap(); } + std::fs::write(tmp.join("ontology.cc"), [0x90]).unwrap(); let mut run = RunStores::create( &tmp.join("run"), "orig", @@ -357,6 +361,7 @@ fn original_round(family: ModelFamily) -> (weksa::run::RunStores, weksa::run::Ro corpus: &tmp.join("corpus.cc"), inventory: &tmp.join("inventory.cc"), entities: &tmp.join("entity.cc"), + ontology: &tmp.join("ontology.cc"), }, "nope", "bag", @@ -506,6 +511,7 @@ fn no_test_item_is_offered_to_a_round_or_recall_or_recorded() { for f in ["corpus.cc", "entity.cc", "inventory.cc"] { std::fs::copy(src.join(f), tmp.path().join(f)).unwrap(); } + std::fs::write(tmp.path().join("ontology.cc"), [0x90]).unwrap(); let p = |f: &str| tmp.path().join(f); let mut run = RunStores::create( &p("run"), @@ -514,6 +520,7 @@ fn no_test_item_is_offered_to_a_round_or_recall_or_recorded() { corpus: &p("corpus.cc"), inventory: &p("inventory.cc"), entities: &p("entity.cc"), + ontology: &p("ontology.cc"), }, "nope", "bag", diff --git a/schemas/cultnet/weksa.loop_run.schema.json b/schemas/cultnet/weksa.loop_run.schema.json index bb63827..e5b144b 100644 --- a/schemas/cultnet/weksa.loop_run.schema.json +++ b/schemas/cultnet/weksa.loop_run.schema.json @@ -49,6 +49,14 @@ "description": "The answering model.", "type": "string" }, + "ontology": { + "description": "Canonical absolute path of the ontology store: read-only for the run,\npinned by `ontology_digest`.", + "type": "string" + }, + "ontology_digest": { + "description": "16 hex of the ontology file's bytes.", + "type": "string" + }, "panel": { "description": "The languages the run offers, reads and decides on: BCP 47-form tags,\nsorted, unique, at least one, set once at open. An item of any other\nlanguage is never offered, minted from, probed or scored.", "type": "array", @@ -73,7 +81,9 @@ "chrf_version", "inventory_digest", "lowerer_reach", - "panel" + "panel", + "ontology", + "ontology_digest" ], "$defs": { "LowererReach": { diff --git a/schemas/cultnet/weksa.ontology_entry.schema.json b/schemas/cultnet/weksa.ontology_entry.schema.json new file mode 100644 index 0000000..99ba82d --- /dev/null +++ b/schemas/cultnet/weksa.ontology_entry.schema.json @@ -0,0 +1,264 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "OntologyEntry", + "description": "One choice of one language. Key `onto::`.", + "type": "object", + "properties": { + "annotator": { + "$ref": "#/$defs/Annotator" + }, + "choice": { + "type": "string" + }, + "examples": { + "type": "array", + "items": { + "type": "string" + } + }, + "inventory_rev": { + "description": "The inventory revision the conditions read against.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "language": { + "type": "string" + }, + "layer": { + "$ref": "#/$defs/Layer" + }, + "n": { + "description": "The entry's number within its language, from 1.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "when": { + "description": "Every condition must hold; none means the entry always applies.", + "type": "array", + "items": { + "$ref": "#/$defs/Condition" + } + } + }, + "required": [ + "language", + "n", + "inventory_rev", + "layer", + "when", + "choice", + "examples", + "annotator" + ], + "$defs": { + "Annotator": { + "description": "Who annotated a document. A model's generator is an attestation string, held\nto the same form as every provenance string.", + "oneOf": [ + { + "type": "string", + "enum": [ + "Operator" + ] + }, + { + "type": "object", + "properties": { + "Model": { + "type": "object", + "properties": { + "family": { + "$ref": "#/$defs/ModelFamily" + }, + "generator": { + "type": "string" + } + }, + "required": [ + "family", + "generator" + ] + } + }, + "additionalProperties": false, + "required": [ + "Model" + ] + } + ] + }, + "Condition": { + "description": "One feature of one slot of the view, tested. Only `Stance` and `Discourse`\nreach a lowering view today.", + "type": "object", + "properties": { + "feature": { + "$ref": "#/$defs/InvId" + }, + "slot": { + "$ref": "#/$defs/Slot" + }, + "test": { + "$ref": "#/$defs/Test" + } + }, + "required": [ + "slot", + "feature", + "test" + ] + }, + "FeatValue": { + "oneOf": [ + { + "type": "object", + "properties": { + "Grade": { + "$ref": "#/$defs/Grade" + } + }, + "additionalProperties": false, + "required": [ + "Grade" + ] + }, + { + "description": "A label the feature's inventory entry declares.", + "type": "object", + "properties": { + "Option": { + "$ref": "#/$defs/OptionLabel" + } + }, + "additionalProperties": false, + "required": [ + "Option" + ] + }, + { + "type": "object", + "properties": { + "Flag": { + "type": "boolean" + } + }, + "additionalProperties": false, + "required": [ + "Flag" + ] + } + ] + }, + "Grade": { + "type": "integer", + "maximum": 1000, + "minimum": 0 + }, + "InvId": { + "type": "string", + "pattern": "^(concept|frame|role|relation|speech_act|feature|referent_kind|extension):[1-9][0-9]*$" + }, + "Layer": { + "description": "Which part of the language an entry shapes.", + "type": "string", + "enum": [ + "Lexicon", + "Grammar", + "Discourse", + "Phonology", + "Orthography" + ] + }, + "ModelFamily": { + "description": "The family of the model that wrote a line.", + "oneOf": [ + { + "type": "string", + "enum": [ + "Anthropic", + "OpenAi", + "Google", + "Alibaba", + "Meta", + "Mistral" + ] + }, + { + "description": "Nothing records which family wrote the line.", + "type": "string", + "const": "Unrecorded" + } + ] + }, + "OptionLabel": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]{0,23}$" + }, + "Slot": { + "description": "Where a feature may sit in a packet.", + "oneOf": [ + { + "type": "string", + "enum": [ + "Discourse", + "Referent", + "Predication", + "Relation", + "Stance" + ] + }, + { + "description": "The speaker's relation to one party of the scene, held by `weksa.scene`\ndocuments; see `LoweringInput::with_scene`.", + "type": "string", + "const": "Scene" + }, + { + "description": "The speaker's stable axes, held by `weksa.speaker` documents.", + "type": "string", + "const": "Speaker" + } + ] + }, + "Test": { + "description": "What a feature's value must be for a condition to hold.", + "oneOf": [ + { + "type": "object", + "properties": { + "AtLeast": { + "$ref": "#/$defs/Grade" + } + }, + "additionalProperties": false, + "required": [ + "AtLeast" + ] + }, + { + "type": "object", + "properties": { + "AtMost": { + "$ref": "#/$defs/Grade" + } + }, + "additionalProperties": false, + "required": [ + "AtMost" + ] + }, + { + "type": "object", + "properties": { + "Is": { + "$ref": "#/$defs/FeatValue" + } + }, + "additionalProperties": false, + "required": [ + "Is" + ] + } + ] + } + } +} From cd435e2a48a0f3451048711eadab2ad4ecdb14cf Mon Sep 17 00:00:00 2001 From: Weksa Date: Thu, 12 Nov 2026 14:14:44 +0000 Subject: [PATCH 06/57] Ontology tests: select, admit rules, prompt section, run pin, round prompts, draft door, CLI Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/encode.rs | 24 ++++ crates/weksa/tests/cli.rs | 44 +++++++ crates/weksa/tests/context.rs | 148 +++++++++++++++++++++++- crates/weksa/tests/golf_loop.rs | 195 ++++++++++++++++++++++++++++++++ crates/weksa/tests/stores.rs | 166 +++++++++++++++++++++++++++ 5 files changed, 575 insertions(+), 2 deletions(-) diff --git a/crates/weksa/src/encode.rs b/crates/weksa/src/encode.rs index d616d3d..5ae008a 100644 --- a/crates/weksa/src/encode.rs +++ b/crates/weksa/src/encode.rs @@ -681,4 +681,28 @@ mod tests { } assert!(EditDraft::parse("3").is_err() && EditDraft::parse("\"x\"").is_err()); } + + #[test] + fn ontology_draft_refuses_unknown_fields_and_numbers_from_first_n() { + for bad in [ + r#"{"entries": [], "extra": 1}"#, + r#"{"entries": [{"layer": "Lexicon", "choice": "c", "language": "ja-JP"}]}"#, + "```json\n{\"entries\": []}\n```", + "[]", + ] { + assert!(OntologyDraft::parse(bad).is_err(), "{bad}"); + } + let draft = OntologyDraft::parse( + r#"{"entries": [{"layer": "Grammar", "choice": "a"}, {"layer": "Lexicon", "choice": "b", "examples": ["x"]}]}"#, + ) + .unwrap(); + let by = Annotator::Operator; + let entries = draft.into_entries("ja-JP", 3, 5, &by); + assert_eq!( + entries.iter().map(|e| (e.language.as_str(), e.n, e.inventory_rev)).collect::>(), + [("ja-JP", 5, 3), ("ja-JP", 6, 3)] + ); + assert!(entries.iter().all(|e| e.annotator == by)); + assert_eq!(entries[1].examples, ["x"]); + } } diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 605d6a7..d46db9f 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -1767,3 +1767,47 @@ fn panel_and_language_are_required() { assert!(!o.status.success()); assert!(out(&o).contains("--language"), "{}", out(&o)); } + +#[test] +fn ontology_admit_and_dump() { + let d = tempfile::tempdir().unwrap(); + dao_stores(d.path()); + let draft = |name: &str, body: &str| { + std::fs::write(d.path().join(name), body).unwrap(); + }; + draft( + "good.json", + r#"{"entries": [{"layer": "Discourse", "when": [{"slot": "Stance", "feature": "feature:18", "test": {"AtLeast": 600}}], "choice": "SOFTCHOICE", "examples": ["EXAMPLETEXT"]}, {"layer": "Lexicon", "choice": "plain"}]}"#, + ); + let admit = |file: &str| { + weksa( + d.path(), + &[ + "ontology", "admit", file, "--language", "ja-JP", "--inventory-rev", "1", + "--model-family", "anthropic", "--generator", "a drafting model", + ], + ) + }; + let o = admit("good.json"); + assert!(o.status.success(), "{}", out(&o)); + let o = weksa(d.path(), &["ontology", "dump"]); + let text = out(&o); + assert_eq!(text.matches("onto:ja-JP:").count(), 2, "{text}"); + for want in ["onto:ja-JP:1", "onto:ja-JP:2", "Anthropic", "a drafting model", "SOFTCHOICE", "EXAMPLETEXT", "Stance"] { + assert!(text.contains(want), "{want}: {text}"); + } + // A second draft numbers on from the language's latest. + let o = admit("good.json"); + assert!(o.status.success(), "{}", out(&o)); + assert!(out(&weksa(d.path(), &["ontology", "dump"])).contains("onto:ja-JP:4")); + // A refused draft writes nothing and echoes no choice text. + draft( + "bad.json", + r#"{"entries": [{"layer": "Discourse", "when": [{"slot": "Stance", "feature": "feature:9999", "test": {"AtLeast": 600}}], "choice": "LEAKYCHOICE"}]}"#, + ); + let before = out(&weksa(d.path(), &["ontology", "dump"])); + let o = admit("bad.json"); + assert!(!o.status.success()); + assert!(!out(&o).contains("LEAKYCHOICE"), "{}", out(&o)); + assert_eq!(out(&weksa(d.path(), &["ontology", "dump"])), before); +} diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs index 61e86f7..fdad261 100644 --- a/crates/weksa/tests/context.rs +++ b/crates/weksa/tests/context.rs @@ -537,9 +537,10 @@ mod lowering { use weksa::daodejing::packets; use weksa::entity::{Entity, EntityOrigin, LanguageForms}; use weksa::packet::{EntityId, Packet, proof_packet}; + use weksa::packet::Grade; use weksa::{ - CorpusItemId, CorpusStore, EntityForms, EntityStore, Inventory, InventoryStore, - LowerError, LoweringInput, PacketStore, + Annotator, Condition, CorpusItemId, CorpusStore, EntityForms, EntityStore, Inventory, + InventoryStore, Layer, LowerError, LoweringInput, OntologyEntry, OntologyStore, PacketStore, Test, select, }; fn data(sub: &str) -> PathBuf { @@ -746,6 +747,149 @@ mod lowering { assert_eq!(Prompt::lower(&view, &item, "en-US", &inv, &held, &weksa::Ontology::empty()).err(), Some(want)); } } + + fn feature(inv: &Inventory, gloss: &str) -> weksa::InvId { + inv.entries().find(|e| e.gloss == gloss).unwrap().id + } + + fn entry(language: &str, n: u32, when: Vec, choice: &str) -> OntologyEntry { + OntologyEntry { + language: language.into(), + n, + inventory_rev: 1, + layer: Layer::Discourse, + when, + choice: choice.into(), + examples: vec![], + annotator: Annotator::Operator, + } + } + + /// The ontology the store holds after admitting `entries` as one batch. + fn ontology_of(cast: &InventoryStore, entries: Vec) -> weksa::Ontology { + let dir = tempfile::tempdir().unwrap(); + let mut store = OntologyStore::open(&dir.path().join("o.cc")).unwrap(); + store.admit(entries, cast).unwrap(); + store.ontology().unwrap() + } + + fn at_least(inv: &Inventory, gloss: &str, g: u16) -> Condition { + Condition { + slot: weksa::Slot::Stance, + feature: feature(inv, gloss), + test: Test::AtLeast(Grade::new(g).unwrap()), + } + } + + /// The proof packet's view with the given stance restraint and discourse irony. + fn view_of( + restraint: Option, + irony: Option, + cast: &InventoryStore, + entities: &EntityStore, + ) -> LoweringInput { + use weksa::packet::{FeatValue, FeatureValue}; + let inv = cast.inventory().unwrap(); + let mut packet = proof_packet(); + packet.stance.features = restraint + .map(|v| FeatureValue { + feature: feature(&inv, "restraint"), + value: FeatValue::Grade(Grade::new(v).unwrap()), + }) + .into_iter() + .collect(); + packet.discourse.features = irony + .map(|v| FeatureValue { + feature: feature(&inv, "irony"), + value: FeatValue::Flag(v), + }) + .into_iter() + .collect(); + admitted(packet, cast, "corpus/corpus.cc", entities) + } + + #[test] + fn select_reads_stance_and_discourse_of_its_language() { + use weksa::packet::FeatValue; + let cast = InventoryStore::open(&data("interlingua/inventory.cc")).unwrap(); + let entities = EntityStore::open(&data("entity/entity.cc")).unwrap(); + let inv = cast.inventory().unwrap(); + let ontology = ontology_of(&cast, vec![ + entry("ja-JP", 1, vec![at_least(&inv, "restraint", 600)], "high"), + entry( + "ja-JP", + 2, + vec![Condition { + slot: weksa::Slot::Stance, + feature: feature(&inv, "restraint"), + test: Test::AtMost(Grade::new(300).unwrap()), + }], + "low", + ), + entry( + "ja-JP", + 3, + vec![Condition { + slot: weksa::Slot::Discourse, + feature: feature(&inv, "irony"), + test: Test::Is(FeatValue::Flag(true)), + }], + "ironic", + ), + entry("ja-JP", 4, vec![], "always"), + entry("en-US", 1, vec![], "other language"), + ]); + let keys = |view: &LoweringInput, language: &str| -> Vec { + select(&ontology, language, view).iter().map(|e| e.key()).collect() + }; + let both = view_of(Some(700), Some(true), &cast, &entities); + assert_eq!(keys(&both, "ja-JP"), ["onto:ja-JP:1", "onto:ja-JP:3", "onto:ja-JP:4"]); + assert_eq!(keys(&both, "en-US"), ["onto:en-US:1"]); + let low = view_of(Some(200), Some(false), &cast, &entities); + assert_eq!(keys(&low, "ja-JP"), ["onto:ja-JP:2", "onto:ja-JP:4"]); + // An absent feature meets no condition, however lax. + let bare = view_of(None, None, &cast, &entities); + assert_eq!(keys(&bare, "ja-JP"), ["onto:ja-JP:4"]); + } + + #[test] + fn lowering_prompt_prints_only_selected_entries() { + let cast = InventoryStore::open(&data("interlingua/inventory.cc")).unwrap(); + let cast_entities = EntityStore::open(&data("entity/entity.cc")).unwrap(); + let inv = cast.inventory().unwrap(); + let mut hit = entry("ja-JP", 1, vec![at_least(&inv, "restraint", 600)], "HITCHOICE"); + hit.examples = vec!["HITEXAMPLE".into()]; + let ontology = ontology_of(&cast, vec![ + hit, + entry("ja-JP", 2, vec![at_least(&inv, "restraint", 900)], "MISSCHOICE"), + entry("en-US", 1, vec![], "ENGLISHCHOICE"), + ]); + let held = Held(vec![nibu(&[("ja-JP", &["ニブ"]), ("en-US", &["Nibu"])])]); + let item = CorpusItemId::parse("item:nibu:1").unwrap(); + let view = view_of(Some(700), None, &cast, &cast_entities); + let lowered = |view: &LoweringInput, o: &weksa::Ontology| { + Prompt::lower(view, &item, "ja-JP", &inv, &held, o).unwrap() + }; + let prompt = lowered(&view, &ontology); + let text = prompt.text(); + assert!(text.contains("\n\nontology:\nonto:ja-JP:1 [discourse] HITCHOICE | e.g. HITEXAMPLE\n\n"), "{text}"); + for absent in ["MISSCHOICE", "ENGLISHCHOICE", "onto:ja-JP:2", "onto:en-US:1"] { + assert!(!text.contains(absent), "{absent}"); + } + // The same view with no entries: the section reads none. + let none = lowered(&view, &weksa::Ontology::empty()); + assert!(none.text().contains("\n\nontology: none\n\n")); + // Entries move the prompt and nothing else of the view does. + assert_ne!(prompt.digest(), none.digest()); + // The group is still private; the speaker's entity appears only because + // the packet cites it on a referent. + let held_item = CorpusStore::open(&data("corpus/corpus.cc")) + .unwrap() + .get(item.as_str()) + .unwrap() + .unwrap(); + assert!(!text.contains(held_item.group.as_str())); + } } /// First real rounds: five of six encoders wrote bare integers (`"concept": 7`) diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 38ae231..c1f83ca 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -871,6 +871,16 @@ fn dao_over(edit: impl FnOnce(&Path, &Path)) -> Dao { /// `dao_over` with the run opened on `panel`. fn dao_panel(edit: impl FnOnce(&Path, &Path), panel: &[&str]) -> Dao { + dao_build(edit, panel, |_, _| {}) +} + +/// `dao_panel` with `seed_ontology` (given the ontology and inventory paths) +/// run before the run pins the ontology. +fn dao_build( + edit: impl FnOnce(&Path, &Path), + panel: &[&str], + seed_ontology: impl FnOnce(&Path, &Path), +) -> Dao { let panel: Vec = panel.iter().map(|t| t.to_string()).collect(); let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/daodejing"); let dir = tempfile::tempdir().unwrap(); @@ -885,6 +895,7 @@ fn dao_panel(edit: impl FnOnce(&Path, &Path), panel: &[&str]) -> Dao { ); edit(&corpus, &entities); let ontology = common::ontology_in(dir.path()); + seed_ontology(&ontology, &inventory); let root = dir.path().join("run"); let run = RunStores::create( &root, @@ -2650,6 +2661,7 @@ fn source_tripwires() { assert!(!signature.contains(banned), "Prompt::lower takes {banned}"); } assert!(signature.contains("&LoweringInput") && signature.contains("&CorpusItemId")); + assert!(signature.contains("&Ontology"), "the whole ontology goes in; select chooses"); assert_eq!(count("admit_revision("), 1, "one door call"); assert_eq!( count("std::fs::copy"), @@ -4583,3 +4595,186 @@ fn ablation_reads_each_language_apart() { [("en-US", Train, 1), ("en-US", Validation, 2), ("ja-JP", Train, 1), ("ja-JP", Validation, 2)] ); } + +/// An en-US Dao ontology: entry 1 for a stance with affection, entry 2 for one +/// with almost no restraint (which no Dao packet has). +fn dao_ontology(path: &Path, inventory: &Path) { + use weksa::packet::Grade; + use weksa::{Condition, Layer, OntologyEntry, OntologyStore, Slot, Test}; + let inv = InventoryStore::open(inventory).unwrap(); + let feature = |gloss: &str| { + inv.inventory() + .unwrap() + .entries() + .find(|e| e.gloss == gloss) + .unwrap() + .id + }; + let entry = |n, gloss: &str, test, choice: &str| OntologyEntry { + language: "en-US".into(), + n, + inventory_rev: 1, + layer: Layer::Discourse, + when: vec![Condition { + slot: Slot::Stance, + feature: feature(gloss), + test, + }], + choice: choice.into(), + examples: vec![], + annotator: weksa::Annotator::Operator, + }; + OntologyStore::open(path) + .unwrap() + .admit( + vec![ + entry(1, "affection", Test::AtLeast(Grade::new(200).unwrap()), "warm"), + entry(2, "restraint", Test::AtMost(Grade::new(100).unwrap()), "blunt"), + ], + &inv_store(inventory), + ) + .unwrap(); +} + +fn inv_store(path: &Path) -> InventoryStore { + InventoryStore::open(path).unwrap() +} + +#[test] +fn a_run_pins_its_ontology_and_refuses_drift() { + let dao = dao_build(|_, _| {}, &["en-US"], dao_ontology); + let r = dao.run.record(); + assert_eq!(r.ontology, std::fs::canonicalize(&dao.ontology).unwrap().to_str().unwrap()); + assert_eq!(r.ontology_digest, hex(&dao.ontology)); + let root = dao.root.clone(); + assert_eq!(RunStores::open(&root).unwrap().record(), dao.run.record()); + // Another entry after the pin is another ontology. + let entry = weksa::OntologyStore::open(&dao.ontology).unwrap().all().unwrap().remove(0); + let mut store = weksa::OntologyStore::open(&dao.ontology).unwrap(); + store + .admit( + vec![weksa::OntologyEntry { n: 3, ..entry }], + &inv_store(&dao.inventory), + ) + .unwrap(); + assert_eq!( + RunStores::open(&root).err(), + Some(RunError::Drift(DriftOf::Ontology)) + ); +} + +#[test] +fn create_refuses_an_entry_its_inventory_cannot_read() { + use weksa::packet::Grade; + use weksa::{Condition, Layer, OntologyEntry, OntologyStore, Slot, Test}; + // The entry reads feature 18 at revision 1; the run's head inventory is + // another file where that feature is retired or absent, so the entry is + // admitted against a richer inventory and refused against the run's. + let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/daodejing"); + let make = |language: &str, dir: &Path| { + for (from, to) in [("corpus.cc", "c.cc"), ("inventory.cc", "i.cc"), ("entity.cc", "e.cc")] { + std::fs::copy(src.join(from), dir.join(to)).unwrap(); + } + let ontology = dir.join("o.cc"); + // A richer inventory the entry is admitted against: the Dao one plus a feature. + let rich = dir.join("rich.cc"); + std::fs::copy(src.join("inventory.cc"), &rich).unwrap(); + let mut store = InventoryStore::open(&rich).unwrap(); + let inv = store.inventory().unwrap(); + let head = inv.latest().unwrap().n; + let n = head + 1; + let next = std::num::NonZeroU32::new( + inv.entries().filter(|e| e.id.family() == weksa::Family::Feature).map(|e| e.id.n().get()).max().unwrap() + 1, + ) + .unwrap(); + let id = weksa::Family::Feature.id(next); + store + .admit_revision( + weksa::InventoryRevision { + n, + parent: Some(head), + changes: vec![weksa::Change::Add(id)], + frozen: false, + }, + vec![weksa::InventoryEntry { + id, + gloss: "pride".into(), + value_kind: Some(weksa::ValueKind::Grade), + applies_to: vec![Slot::Stance], + introduced_rev: n, + retired_rev: None, + }], + ) + .unwrap(); + OntologyStore::open(&ontology) + .unwrap() + .admit( + vec![OntologyEntry { + language: language.into(), + n: 1, + inventory_rev: n, + layer: Layer::Lexicon, + when: vec![Condition { + slot: Slot::Stance, + feature: id, + test: Test::AtLeast(Grade::new(1).unwrap()), + }], + choice: "boast".into(), + examples: vec![], + annotator: weksa::Annotator::Operator, + }], + &store, + ) + .unwrap(); + ontology + }; + let create = |dir: &Path, ontology: &Path| { + RunStores::create( + &dir.join("run"), + "x", + RunPaths { + corpus: &dir.join("c.cc"), + inventory: &dir.join("i.cc"), + entities: &dir.join("e.cc"), + ontology, + }, + "fake-model", + "bag", + weksa::run::LowererReach::ToolLessRelay, + ModelFamily::Unrecorded, + &["en-US".to_string()], + ) + }; + let panel_dir = tempfile::tempdir().unwrap(); + let o = make("en-US", panel_dir.path()); + assert_eq!(create(panel_dir.path(), &o).err(), Some(RunError::Ontology)); + assert!(!panel_dir.path().join("run/run.cc").exists()); + assert!(!panel_dir.path().join("run/inventory.cc").exists()); + // The same entry under a language outside the panel does not refuse. + let other_dir = tempfile::tempdir().unwrap(); + let o = make("ja-JP", other_dir.path()); + assert!(create(other_dir.path(), &o).is_ok()); +} + +#[test] +fn round_lowering_prompts_carry_selected_entries() { + let mut dao = dao_build(|_, _| {}, &["en-US"], dao_ontology); + let mut model = fake(&good_answers()); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + let mut lowerings = 0; + for ((_, text), call) in model.seen.iter().zip(r.calls()) { + if !text.starts_with(LOWER_INSTRUCTIONS) { + continue; + } + lowerings += 1; + let item = call.items[0].as_str(); + // Only the third Dao packet carries affection. + if item == "item:daodejing:3" { + assert!(text.contains("\nonto:en-US:1 [discourse] warm\n"), "{text}"); + } else { + assert!(text.contains("\n\nontology: none\n\n"), "{item}: {text}"); + } + assert!(!text.contains("onto:en-US:2") && !text.contains("blunt")); + } + assert_eq!(lowerings, 3); +} diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs index fa51e47..6baa1ff 100644 --- a/crates/weksa/tests/stores.rs +++ b/crates/weksa/tests/stores.rs @@ -1628,3 +1628,169 @@ fn a_public_domain_source_without_an_edition_year_is_refused() { .add(item_in(key, "line", "scene-1", source(1891)), &ents(&d)) .unwrap(); } + +mod ontology { + use super::*; + use weksa::packet::{FeatValue, Grade}; + use weksa::speaker::Annotator; + use weksa::{Condition, Layer, OntologyEntry, OntologyStore, Test}; + + fn feature(gloss: &str) -> weksa::InvId { + seed().1.into_iter().find(|e| e.gloss == gloss).unwrap().id + } + + fn when(slot: Slot, gloss: &str, test: Test) -> Condition { + Condition { + slot, + feature: feature(gloss), + test, + } + } + + fn entry(language: &str, n: u32) -> OntologyEntry { + OntologyEntry { + language: language.into(), + n, + inventory_rev: 1, + layer: Layer::Lexicon, + when: vec![when( + Slot::Stance, + "restraint", + Test::AtLeast(Grade::new(600).unwrap()), + )], + choice: "prefer the humble form".into(), + examples: vec!["example one".into()], + annotator: Annotator::Operator, + } + } + + fn store(d: &TempDir) -> OntologyStore { + OntologyStore::open(&d.path().join("o.cc")).unwrap() + } + + fn refused(r: Result) -> StoreRefusal { + refusal(r) + } + + #[test] + fn ontology_admit_refuses_each_rule() { + let d = dir(); + let w = world(); + let mut s = store(&d); + let long = |n| "x".repeat(n); + let mut cases: Vec<(&str, Vec)> = Vec::new(); + cases.push(("non-tag language", vec![entry("not a tag", 1)])); + cases.push(("n not latest+1", vec![entry("ja-JP", 2)])); + cases.push(( + "65th entry", + (1..=65).map(|n| entry("ja-JP", n)).collect(), + )); + for (why, choice) in [("empty choice", String::new()), ("201-byte choice", long(201))] { + cases.push((why, vec![OntologyEntry { choice, ..entry("ja-JP", 1) }])); + } + cases.push(( + "line break in an example", + vec![OntologyEntry { examples: vec!["a\nb".into()], ..entry("ja-JP", 1) }], + )); + cases.push(( + "121-byte example", + vec![OntologyEntry { examples: vec![long(121)], ..entry("ja-JP", 1) }], + )); + cases.push(( + "four examples", + vec![OntologyEntry { examples: vec!["a".into(); 4], ..entry("ja-JP", 1) }], + )); + let restraint = || when(Slot::Stance, "restraint", Test::AtLeast(Grade::new(1).unwrap())); + cases.push(( + "five conditions", + vec![OntologyEntry { when: (0..5).map(|_| restraint()).collect(), ..entry("ja-JP", 1) }], + )); + for slot in [Slot::Speaker, Slot::Scene, Slot::Predication] { + cases.push(( + "slot not Stance or Discourse", + vec![OntologyEntry { + when: vec![when(slot, "restraint", Test::AtLeast(Grade::new(1).unwrap()))], + ..entry("ja-JP", 1) + }], + )); + } + cases.push(( + "AtLeast on an option feature", + vec![OntologyEntry { + when: vec![when(Slot::Discourse, "irony", Test::AtLeast(Grade::new(1).unwrap()))], + ..entry("ja-JP", 1) + }], + )); + cases.push(( + "Is with a grade", + vec![OntologyEntry { + when: vec![when(Slot::Stance, "restraint", Test::Is(FeatValue::Grade(Grade::new(1).unwrap())))], + ..entry("ja-JP", 1) + }], + )); + cases.push(( + "unknown feature", + vec![OntologyEntry { + when: vec![Condition { + slot: Slot::Stance, + feature: "feature:9999".parse().unwrap(), + test: Test::AtLeast(Grade::new(1).unwrap()), + }], + ..entry("ja-JP", 1) + }], + )); + cases.push(( + "unknown inventory revision", + vec![OntologyEntry { inventory_rev: 99, ..entry("ja-JP", 1) }], + )); + cases.push(( + "model generator that is not an attestation", + vec![OntologyEntry { + annotator: Annotator::Model { + family: ModelFamily::Anthropic, + generator: "--".into(), + }, + ..entry("ja-JP", 1) + }], + )); + for (why, batch) in cases { + assert!( + s.admit(batch, &w.inventory).is_err(), + "{why} must be refused" + ); + assert!(s.all().unwrap().is_empty(), "{why} wrote something"); + } + // The refusal of a bad choice names the key and no text. + let secret = OntologyEntry { choice: "SECRETCHOICE\n".into(), ..entry("ja-JP", 1) }; + let shown = refused(s.admit(vec![secret], &w.inventory)).to_string(); + assert!(!shown.contains("SECRETCHOICE"), "{shown}"); + } + + #[test] + fn ontology_admit_is_all_or_nothing() { + let d = dir(); + let w = world(); + let mut s = store(&d); + let bad = OntologyEntry { choice: String::new(), ..entry("ja-JP", 3) }; + assert!(s.admit(vec![entry("ja-JP", 1), entry("ja-JP", 2), bad], &w.inventory).is_err()); + assert!(s.all().unwrap().is_empty()); + + let batch = vec![entry("ja-JP", 1), entry("en-US", 1), entry("ja-JP", 2), entry("en-US", 2)]; + assert_eq!(s.admit(batch.clone(), &w.inventory).unwrap(), AddOutcome::Added); + assert_eq!(s.all().unwrap().len(), 4); + // An identical replay, from a reopened store, changes nothing. + let mut again = store(&d); + assert_eq!(again.admit(batch, &w.inventory).unwrap(), AddOutcome::Unchanged); + // Each language numbers from its own latest. + assert_eq!( + again + .admit(vec![entry("ja-JP", 3), entry("en-US", 3)], &w.inventory) + .unwrap(), + AddOutcome::Added + ); + // An entry never changes in place. + let changed = OntologyEntry { choice: "something else".into(), ..entry("ja-JP", 1) }; + assert!(again.admit(vec![changed], &w.inventory).is_err()); + assert_eq!(again.all().unwrap().len(), 6); + } +} From 602f7c2cd50b895735e5b4c8c1cdc0b31253be81 Mon Sep 17 00:00:00 2001 From: Weksa Date: Thu, 12 Nov 2026 14:31:35 +0000 Subject: [PATCH 07/57] Ontology fix pass s1: refuse format and separator characters, canonical-case language, entries on retired features; kill six surviving mutants; two-language round test Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/form.rs | 19 +++++ crates/weksa/src/ontology.rs | 5 ++ crates/weksa/src/run.rs | 14 +++- crates/weksa/src/store.rs | 18 +++-- crates/weksa/tests/context.rs | 5 ++ crates/weksa/tests/golf_loop.rs | 78 ++++++++++++++----- crates/weksa/tests/stores.rs | 56 +++++++++++++ .../cultnet/weksa.ontology_entry.schema.json | 2 +- 8 files changed, 169 insertions(+), 28 deletions(-) diff --git a/crates/weksa/src/form.rs b/crates/weksa/src/form.rs index 3626fd1..c530c0e 100644 --- a/crates/weksa/src/form.rs +++ b/crates/weksa/src/form.rs @@ -107,6 +107,25 @@ pub(crate) fn is_language_tag(s: &str) -> bool { .all(|t| (1..=8).contains(&t.len()) && t.bytes().all(|b| b.is_ascii_alphanumeric())) } +/// A language tag in its canonical case: lowercase language, two-letter region +/// in capitals, four-letter script in title case, any other subtag lowercase. +/// An ontology is selected by exact tag, so a case variant would be a dead +/// language. +pub(crate) fn is_canonical_language_tag(s: &str) -> bool { + is_language_tag(s) + && s.split('-').enumerate().all(|(i, t)| { + let upper = |c: char| c.is_ascii_uppercase(); + match (i, t.len()) { + (0, _) => !t.chars().any(upper), + (_, 2) if t.bytes().all(|b| b.is_ascii_alphabetic()) => t.chars().all(upper), + (_, 4) if t.bytes().all(|b| b.is_ascii_alphabetic()) => { + t.chars().next().is_some_and(upper) && !t.chars().skip(1).any(upper) + } + _ => !t.chars().any(upper), + } + }) +} + /// A slug: `[a-z0-9]` and inner `-`, 1 to 48 bytes, no edge `-`. pub(crate) fn is_slug(s: &str) -> bool { let b = s.as_bytes(); diff --git a/crates/weksa/src/ontology.rs b/crates/weksa/src/ontology.rs index 4bc38fb..4194882 100644 --- a/crates/weksa/src/ontology.rs +++ b/crates/weksa/src/ontology.rs @@ -86,6 +86,11 @@ impl Condition { } /// One choice of one language. Key `onto::`. +/// +/// The store judges the form of an entry's text (bounds, normalization, no +/// control or format character) and never its meaning: what may enter an entry +/// is owned by the minting flow (Train-only assembly) and the scorer's leak +/// diagnostic (ruling `leak-owned-by-flow-and-scorer-3`), not by a door here. #[derive(Clone, Debug, PartialEq, Eq, JsonSchema, DatabaseEntry)] #[cultcache(type = "weksa.ontology_entry")] pub struct OntologyEntry { diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 0e63af4..42bc5be 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -43,6 +43,7 @@ use crate::corpus::{CorpusItem, CorpusItemId, ModelFamily, Source}; use crate::encode::{EditDraft, LoweringDraft, PacketDraft, encoder_id}; use crate::form::{is_attestation, is_slug}; use crate::model::{ModelPort, PortError}; +use crate::ontology::OntologyEntry; use crate::packet::{Packet, Stance}; use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate}; use crate::split::{Split, SplitScheme, fnv1a64, fnv1a64_of, hex16}; @@ -782,10 +783,21 @@ impl RunStores { fn check_ontology(&self) -> Result<(), RunError> { let inventory = self.inventory.inventory().map_err(|_| RunError::Store)?; let entries = self.ontology.all().map_err(|_| RunError::Store)?; + let head = inventory.latest().map(|r| r.n); entries .iter() .filter(|e| self.record.panel.contains(&e.language)) - .try_for_each(|e| check_ontology_entry(e, &inventory).map_err(|_| RunError::Ontology)) + .try_for_each(|e| { + // At its own revision, and again at the head: an entry on a + // feature the head retired could never select. + let at_head = OntologyEntry { + inventory_rev: head.ok_or(RunError::Ontology)?, + ..e.clone() + }; + check_ontology_entry(e, &inventory) + .and_then(|()| check_ontology_entry(&at_head, &inventory)) + .map_err(|_| RunError::Ontology) + }) } /// Every panel language holds a live Train and a live Validation item. diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index 9183f2f..97671a0 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -14,7 +14,8 @@ use unicode_segmentation::UnicodeSegmentation; use crate::corpus::{CorpusItem, CorpusItemId, Source, item_key, key_slug}; use crate::entity::{Entity, EntityOrigin, Pronunciation}; use crate::form::{ - is_attestation, is_calendar_date, is_joining_only, is_language_tag, is_refused_format, is_slug, + is_attestation, is_calendar_date, is_canonical_language_tag, is_joining_only, is_language_tag, + is_refused_format, is_slug, }; use crate::inventory::{ Change, InvId, Inventory, InventoryEntry, InventoryRevision, Slot, revision_key, @@ -1256,8 +1257,11 @@ impl OntologyStore { let mut counted: std::collections::BTreeMap<&str, u32> = std::collections::BTreeMap::new(); for e in &batch { let key = e.key(); - if !is_language_tag(&e.language) { - return Err(malformed(&key, "an ontology language is a language tag")); + if !is_canonical_language_tag(&e.language) { + return Err(malformed( + &key, + "an ontology language is a language tag in canonical case", + )); } let latest = match counted.get(e.language.as_str()) { Some(n) => *n, @@ -1297,12 +1301,14 @@ pub(crate) fn check_ontology_entry( let key = e.key(); check_annotator(&key, &e.annotator)?; let text_ok = |t: &str, max: usize| { - (1..=max).contains(&t.len()) && !t.chars().any(char::is_control) && is_nfc(t) + (1..=max).contains(&t.len()) + && !t.chars().any(|c| c.is_control() || is_refused_format(c)) + && is_nfc(t) }; if !text_ok(&e.choice, MAX_CHOICE_BYTES) { return Err(malformed( &key, - "a choice is 1 to 200 normalized bytes with no control character", + "a choice is 1 to 200 normalized bytes with no control or format character", )); } if e.examples.len() > MAX_EXAMPLES @@ -1310,7 +1316,7 @@ pub(crate) fn check_ontology_entry( { return Err(malformed( &key, - "an entry has at most 3 examples of 1 to 120 normalized bytes with no control character", + "an entry has at most 3 examples of 1 to 120 normalized bytes with no control or format character", )); } if e.when.len() > MAX_CONDITIONS { diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs index fdad261..5d44f01 100644 --- a/crates/weksa/tests/context.rs +++ b/crates/weksa/tests/context.rs @@ -847,6 +847,11 @@ mod lowering { assert_eq!(keys(&both, "en-US"), ["onto:en-US:1"]); let low = view_of(Some(200), Some(false), &cast, &entities); assert_eq!(keys(&low, "ja-JP"), ["onto:ja-JP:2", "onto:ja-JP:4"]); + // A grade equal to the bound meets AtLeast and AtMost alike. + let edge = view_of(Some(600), None, &cast, &entities); + assert_eq!(keys(&edge, "ja-JP"), ["onto:ja-JP:1", "onto:ja-JP:4"]); + let edge = view_of(Some(300), None, &cast, &entities); + assert_eq!(keys(&edge, "ja-JP"), ["onto:ja-JP:2", "onto:ja-JP:4"]); // An absent feature meets no condition, however lax. let bare = view_of(None, None, &cast, &entities); assert_eq!(keys(&bare, "ja-JP"), ["onto:ja-JP:4"]); diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index c1f83ca..f799c86 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -4596,8 +4596,9 @@ fn ablation_reads_each_language_apart() { ); } -/// An en-US Dao ontology: entry 1 for a stance with affection, entry 2 for one -/// with almost no restraint (which no Dao packet has). +/// A Dao ontology for en-US and ja-JP: entry 1 of each for a stance with +/// affection, and a ja-JP entry 2 for almost no restraint (which no Dao packet +/// has). fn dao_ontology(path: &Path, inventory: &Path) { use weksa::packet::Grade; use weksa::{Condition, Layer, OntologyEntry, OntologyStore, Slot, Test}; @@ -4610,8 +4611,8 @@ fn dao_ontology(path: &Path, inventory: &Path) { .unwrap() .id }; - let entry = |n, gloss: &str, test, choice: &str| OntologyEntry { - language: "en-US".into(), + let entry = |language: &str, n, gloss: &str, test, choice: &str| OntologyEntry { + language: language.into(), n, inventory_rev: 1, layer: Layer::Discourse, @@ -4624,14 +4625,22 @@ fn dao_ontology(path: &Path, inventory: &Path) { examples: vec![], annotator: weksa::Annotator::Operator, }; + let affection = || Test::AtLeast(Grade::new(200).unwrap()); OntologyStore::open(path) .unwrap() .admit( vec![ - entry(1, "affection", Test::AtLeast(Grade::new(200).unwrap()), "warm"), - entry(2, "restraint", Test::AtMost(Grade::new(100).unwrap()), "blunt"), + entry("en-US", 1, "affection", affection(), "ENWARM"), + entry("ja-JP", 1, "affection", affection(), "JAWARM"), + entry( + "ja-JP", + 2, + "restraint", + Test::AtMost(Grade::new(100).unwrap()), + "JABLUNT", + ), ], - &inv_store(inventory), + &inv, ) .unwrap(); } @@ -4653,7 +4662,7 @@ fn a_run_pins_its_ontology_and_refuses_drift() { let mut store = weksa::OntologyStore::open(&dao.ontology).unwrap(); store .admit( - vec![weksa::OntologyEntry { n: 3, ..entry }], + vec![weksa::OntologyEntry { n: 2, ..entry }], &inv_store(&dao.inventory), ) .unwrap(); @@ -4671,7 +4680,7 @@ fn create_refuses_an_entry_its_inventory_cannot_read() { // another file where that feature is retired or absent, so the entry is // admitted against a richer inventory and refused against the run's. let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/daodejing"); - let make = |language: &str, dir: &Path| { + let make = |language: &str, dir: &Path, retire: bool| { for (from, to) in [("corpus.cc", "c.cc"), ("inventory.cc", "i.cc"), ("entity.cc", "e.cc")] { std::fs::copy(src.join(from), dir.join(to)).unwrap(); } @@ -4726,6 +4735,21 @@ fn create_refuses_an_entry_its_inventory_cannot_read() { &store, ) .unwrap(); + if retire { + // The run's head inventory retires the feature after the entry's revision. + store + .admit_revision( + weksa::InventoryRevision { + n: n + 1, + parent: Some(n), + changes: vec![weksa::Change::Retire(id)], + frozen: false, + }, + vec![], + ) + .unwrap(); + std::fs::copy(&rich, dir.join("i.cc")).unwrap(); + } ontology }; let create = |dir: &Path, ontology: &Path| { @@ -4746,35 +4770,49 @@ fn create_refuses_an_entry_its_inventory_cannot_read() { ) }; let panel_dir = tempfile::tempdir().unwrap(); - let o = make("en-US", panel_dir.path()); + let o = make("en-US", panel_dir.path(), false); assert_eq!(create(panel_dir.path(), &o).err(), Some(RunError::Ontology)); assert!(!panel_dir.path().join("run/run.cc").exists()); assert!(!panel_dir.path().join("run/inventory.cc").exists()); // The same entry under a language outside the panel does not refuse. let other_dir = tempfile::tempdir().unwrap(); - let o = make("ja-JP", other_dir.path()); + let o = make("ja-JP", other_dir.path(), false); assert!(create(other_dir.path(), &o).is_ok()); + // An entry on a feature the head inventory has retired can never select. + let retired_dir = tempfile::tempdir().unwrap(); + let o = make("en-US", retired_dir.path(), true); + assert_eq!(create(retired_dir.path(), &o).err(), Some(RunError::Ontology)); + assert!(!retired_dir.path().join("run/run.cc").exists()); } #[test] fn round_lowering_prompts_carry_selected_entries() { - let mut dao = dao_build(|_, _| {}, &["en-US"], dao_ontology); - let mut model = fake(&good_answers()); + let mut dao = dao_build( + |corpus, entities| add_languages(corpus, entities, LAOZI_JA), + &["ja-JP", "en-US"], + dao_ontology, + ); + let mut model = bilingual_fake(); let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); - let mut lowerings = 0; + let mut seen = std::collections::BTreeSet::new(); for ((_, text), call) in model.seen.iter().zip(r.calls()) { if !text.starts_with(LOWER_INSTRUCTIONS) { continue; } - lowerings += 1; let item = call.items[0].as_str(); - // Only the third Dao packet carries affection. - if item == "item:daodejing:3" { - assert!(text.contains("\nonto:en-US:1 [discourse] warm\n"), "{text}"); + let (language, warm, other) = if text.contains("\n\nlanguage: ja-JP\n\n") { + ("ja-JP", "JAWARM", "ENWARM") + } else { + ("en-US", "ENWARM", "JAWARM") + }; + // Only the third packet of each language carries affection. + if item.ends_with(":3") || item.ends_with(":13") { + assert!(text.contains(&format!("\nonto:{language}:1 [discourse] {warm}\n")), "{text}"); } else { assert!(text.contains("\n\nontology: none\n\n"), "{item}: {text}"); } - assert!(!text.contains("onto:en-US:2") && !text.contains("blunt")); + assert!(!text.contains(other) && !text.contains("JABLUNT"), "{item}: {text}"); + seen.insert(item.to_string()); } - assert_eq!(lowerings, 3); + assert_eq!(seen.len(), 6, "three items in each language"); } diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs index 6baa1ff..4cbe3d3 100644 --- a/crates/weksa/tests/stores.rs +++ b/crates/weksa/tests/stores.rs @@ -1673,6 +1673,7 @@ mod ontology { } #[test] + #[allow(clippy::vec_init_then_push)] fn ontology_admit_refuses_each_rule() { let d = dir(); let w = world(); @@ -1681,6 +1682,40 @@ mod ontology { let mut cases: Vec<(&str, Vec)> = Vec::new(); cases.push(("non-tag language", vec![entry("not a tag", 1)])); cases.push(("n not latest+1", vec![entry("ja-JP", 2)])); + cases.push(("case-variant language", vec![entry("ja-jp", 1)])); + cases.push(( + "decomposed choice", + vec![OntologyEntry { choice: "e\u{301}".into(), ..entry("ja-JP", 1) }], + )); + cases.push(( + "decomposed example", + vec![OntologyEntry { examples: vec!["e\u{301}".into()], ..entry("ja-JP", 1) }], + )); + for sep in ["\u{2028}", "\u{2029}", "\u{202e}", "\u{200b}"] { + cases.push(( + "line separator or format character in a choice", + vec![OntologyEntry { choice: format!("a{sep}b"), ..entry("ja-JP", 1) }], + )); + cases.push(( + "line separator or format character in an example", + vec![OntologyEntry { examples: vec![format!("a{sep}b")], ..entry("ja-JP", 1) }], + )); + } + cases.push(( + "unconditioned entry at an unknown revision", + vec![OntologyEntry { when: vec![], inventory_rev: 99, ..entry("ja-JP", 1) }], + )); + cases.push(( + "Predication slot on a feature that sits there", + vec![OntologyEntry { + when: vec![when( + Slot::Predication, + "modality", + Test::Is(FeatValue::Option(lab("actual"))), + )], + ..entry("ja-JP", 1) + }], + )); cases.push(( "65th entry", (1..=65).map(|n| entry("ja-JP", n)).collect(), @@ -1766,6 +1801,23 @@ mod ontology { assert!(!shown.contains("SECRETCHOICE"), "{shown}"); } + #[test] + fn ontology_admit_accepts_the_bounds_and_joiners() { + let d = dir(); + let w = world(); + let mut s = store(&d); + // Exactly 64 entries in one language; exactly the byte bounds; a ZWNJ. + let mut batch: Vec = (1..=64).map(|n| entry("ja-JP", n)).collect(); + batch[0].choice = "x".repeat(200); + batch[0].examples = vec!["y".repeat(120); 3]; + batch[1].choice = "a\u{200c}b".into(); + batch[2].when = (0..4) + .map(|_| when(Slot::Stance, "restraint", Test::AtMost(Grade::new(1000).unwrap()))) + .collect(); + assert_eq!(s.admit(batch, &w.inventory).unwrap(), AddOutcome::Added); + assert_eq!(s.all().unwrap().len(), 64); + } + #[test] fn ontology_admit_is_all_or_nothing() { let d = dir(); @@ -1781,6 +1833,10 @@ mod ontology { // An identical replay, from a reopened store, changes nothing. let mut again = store(&d); assert_eq!(again.admit(batch, &w.inventory).unwrap(), AddOutcome::Unchanged); + // A batch of one stored entry and one new one is not a replay. + let mixed = vec![entry("ja-JP", 1), entry("ja-JP", 5)]; + assert!(again.admit(mixed, &w.inventory).is_err()); + assert_eq!(again.all().unwrap().len(), 4); // Each language numbers from its own latest. assert_eq!( again diff --git a/schemas/cultnet/weksa.ontology_entry.schema.json b/schemas/cultnet/weksa.ontology_entry.schema.json index 99ba82d..e976470 100644 --- a/schemas/cultnet/weksa.ontology_entry.schema.json +++ b/schemas/cultnet/weksa.ontology_entry.schema.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "OntologyEntry", - "description": "One choice of one language. Key `onto::`.", + "description": "One choice of one language. Key `onto::`.\n\nThe store judges the form of an entry's text (bounds, normalization, no\ncontrol or format character) and never its meaning: what may enter an entry\nis owned by the minting flow (Train-only assembly) and the scorer's leak\ndiagnostic (ruling `leak-owned-by-flow-and-scorer-3`), not by a door here.", "type": "object", "properties": { "annotator": { From e46f9332689377a14402edf84f852924f7390d25 Mon Sep 17 00:00:00 2001 From: Weksa Date: Thu, 12 Nov 2026 14:51:31 +0000 Subject: [PATCH 08/57] One language-tag owner: is_language_tag is canonical-case; tests for the case arms and Hangul fillers; leak-owner doc Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/form.rs | 76 ++++++++++++------- crates/weksa/src/ontology.rs | 7 +- crates/weksa/src/run.rs | 7 +- crates/weksa/src/store.rs | 10 +-- crates/weksa/tests/golf_loop.rs | 1 + crates/weksa/tests/stores.rs | 2 +- .../cultnet/weksa.ontology_entry.schema.json | 2 +- 7 files changed, 65 insertions(+), 40 deletions(-) diff --git a/crates/weksa/src/form.rs b/crates/weksa/src/form.rs index c530c0e..2819ea6 100644 --- a/crates/weksa/src/form.rs +++ b/crates/weksa/src/form.rs @@ -91,39 +91,45 @@ pub fn is_calendar_date(s: &str) -> bool { (1..=days).contains(&day) } -/// A BCP 47-form language tag: 1 to 35 bytes of ASCII alphanumerics in -/// hyphen-separated subtags of 1 to 8 bytes; the first is 2 to 8 letters, or the -/// single letter `x` of a private-use tag. Form only: no registry is consulted. +/// A BCP 47 language tag in its canonical case (RFC 5646 section 2.1.1): 1 to 35 +/// bytes of ASCII alphanumerics in hyphen-separated subtags of 1 to 8 bytes; the +/// first is 2 to 8 letters, or the single letter `x` of a private-use tag. The +/// language subtag is lowercase, a four-letter script subtag is Titlecase, a +/// two-letter region subtag is UPPERCASE, and every other subtag, and every +/// subtag after a singleton such as `x`, is lowercase. Form only: no registry is +/// consulted. Selection and comparison are by exact tag, so a case variant would +/// be a different, dead language; this is the one tag check of corpus items, +/// entities, ontology entries and the run panel. pub(crate) fn is_language_tag(s: &str) -> bool { if s.len() > 35 { return false; } let mut subtags = s.split('-'); let first = subtags.next().unwrap_or(""); - let first_ok = first.bytes().all(|b| b.is_ascii_alphabetic()) + let first_ok = first.bytes().all(|b| b.is_ascii_lowercase()) && ((2..=8).contains(&first.len()) || first == "x"); - first_ok - && subtags - .all(|t| (1..=8).contains(&t.len()) && t.bytes().all(|b| b.is_ascii_alphanumeric())) -} - -/// A language tag in its canonical case: lowercase language, two-letter region -/// in capitals, four-letter script in title case, any other subtag lowercase. -/// An ontology is selected by exact tag, so a case variant would be a dead -/// language. -pub(crate) fn is_canonical_language_tag(s: &str) -> bool { - is_language_tag(s) - && s.split('-').enumerate().all(|(i, t)| { - let upper = |c: char| c.is_ascii_uppercase(); - match (i, t.len()) { - (0, _) => !t.chars().any(upper), - (_, 2) if t.bytes().all(|b| b.is_ascii_alphabetic()) => t.chars().all(upper), - (_, 4) if t.bytes().all(|b| b.is_ascii_alphabetic()) => { - t.chars().next().is_some_and(upper) && !t.chars().skip(1).any(upper) - } - _ => !t.chars().any(upper), - } - }) + if !first_ok { + return false; + } + let mut singleton = first == "x"; + subtags.all(|t| { + if !(1..=8).contains(&t.len()) || !t.bytes().all(|b| b.is_ascii_alphanumeric()) { + return false; + } + let alpha = t.bytes().all(|b| b.is_ascii_alphabetic()); + let canonical = if singleton { + t.bytes().all(|b| !b.is_ascii_uppercase()) + } else if alpha && t.len() == 2 { + t.bytes().all(|b| b.is_ascii_uppercase()) + } else if alpha && t.len() == 4 { + t.as_bytes()[0].is_ascii_uppercase() + && t.bytes().skip(1).all(|b| b.is_ascii_lowercase()) + } else { + t.bytes().all(|b| !b.is_ascii_uppercase()) + }; + singleton |= t.len() == 1; + canonical + }) } /// A slug: `[a-z0-9]` and inner `-`, 1 to 48 bytes, no edge `-`. @@ -233,6 +239,24 @@ mod tests { assert!(!is_format('\u{200c}') && !is_format('\u{200d}')); } + #[test] + fn a_language_tag_is_in_canonical_case() { + for good in [ + "en", "en-US", "ja-JP", "zh-Hant-TW", "sr-Latn", "es-419", "de-1996", "en-CA-x-ca", + "x-private", "en-u-nu-latn", "ja-JP-x-a1", + ] { + assert!(is_language_tag(good), "{good}"); + } + for bad in [ + "", "e", "EN", "En", "en-us", "en-Us", "en-uS", "ja-jp", "JA-JP", "zh-hant-TW", + "zh-HANT-TW", "zh-hAnt-TW", "zh-Hant-tw", "en-CA-x-CA", "en-CA-x-Ca", "de-1996-X", + "es-419-LATN", "en--US", "en-", "-en", "en_US", "abcdefghi", "en-abcdefghi", + "xx-\u{e9}", + ] { + assert!(!is_language_tag(bad), "{bad}"); + } + } + #[test] fn an_attestation_is_bounded_readable_text() { assert!(is_attestation("a")); diff --git a/crates/weksa/src/ontology.rs b/crates/weksa/src/ontology.rs index 4194882..30bc5dd 100644 --- a/crates/weksa/src/ontology.rs +++ b/crates/weksa/src/ontology.rs @@ -88,9 +88,10 @@ impl Condition { /// One choice of one language. Key `onto::`. /// /// The store judges the form of an entry's text (bounds, normalization, no -/// control or format character) and never its meaning: what may enter an entry -/// is owned by the minting flow (Train-only assembly) and the scorer's leak -/// diagnostic (ruling `leak-owned-by-flow-and-scorer-3`), not by a door here. +/// control or format character) and never its meaning: no door judges text +/// (ruling `leak-owned-by-flow-and-scorer-3`). No Train-only ontology mint +/// exists yet, so a seed or a drafted entry is checked against held-out text by +/// nothing today; follow-up `leak-diagnostic-in-round` owns that diagnostic. #[derive(Clone, Debug, PartialEq, Eq, JsonSchema, DatabaseEntry)] #[cultcache(type = "weksa.ontology_entry")] pub struct OntologyEntry { diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 42bc5be..770d9b1 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -41,7 +41,7 @@ use unicode_segmentation::UnicodeSegmentation; use crate::context::{EditFeedback, EncoderOffer, Prompt, TrainItems}; use crate::corpus::{CorpusItem, CorpusItemId, ModelFamily, Source}; use crate::encode::{EditDraft, LoweringDraft, PacketDraft, encoder_id}; -use crate::form::{is_attestation, is_slug}; +use crate::form::{is_attestation, is_language_tag, is_slug}; use crate::model::{ModelPort, PortError}; use crate::ontology::OntologyEntry; use crate::packet::{Packet, Stance}; @@ -675,7 +675,10 @@ impl RunStores { panel.sort(); // A tag no item carries (misspelt, wrong case, not a tag) is refused by // `check_panel` once the corpus is open. - if panel.is_empty() || panel.windows(2).any(|w| w[0] == w[1]) { + if panel.is_empty() + || panel.windows(2).any(|w| w[0] == w[1]) + || !panel.iter().all(|t| is_language_tag(t)) + { return Err(RunError::Panel); } let cache = run_cache(root)?; diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index 97671a0..5afb025 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -14,8 +14,7 @@ use unicode_segmentation::UnicodeSegmentation; use crate::corpus::{CorpusItem, CorpusItemId, Source, item_key, key_slug}; use crate::entity::{Entity, EntityOrigin, Pronunciation}; use crate::form::{ - is_attestation, is_calendar_date, is_canonical_language_tag, is_joining_only, is_language_tag, - is_refused_format, is_slug, + is_attestation, is_calendar_date, is_joining_only, is_language_tag, is_refused_format, is_slug, }; use crate::inventory::{ Change, InvId, Inventory, InventoryEntry, InventoryRevision, Slot, revision_key, @@ -1257,11 +1256,8 @@ impl OntologyStore { let mut counted: std::collections::BTreeMap<&str, u32> = std::collections::BTreeMap::new(); for e in &batch { let key = e.key(); - if !is_canonical_language_tag(&e.language) { - return Err(malformed( - &key, - "an ontology language is a language tag in canonical case", - )); + if !is_language_tag(&e.language) { + return Err(malformed(&key, "an ontology language is a language tag")); } let latest = match counted.get(e.language.as_str()) { Some(n) => *n, diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index f799c86..0b22f4c 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -4393,6 +4393,7 @@ fn open_refuses_a_bad_panel() { vec![], vec![tag("en-US"), tag("en-US")], vec![tag("en-US"), tag("not a tag!")], + vec![tag("en-us")], // No live item at all, and a Train item without a Validation one. vec![tag("en-US"), tag("de-DE")], vec![tag("en-US"), tag("fr-FR")], diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs index 4cbe3d3..f027611 100644 --- a/crates/weksa/tests/stores.rs +++ b/crates/weksa/tests/stores.rs @@ -1691,7 +1691,7 @@ mod ontology { "decomposed example", vec![OntologyEntry { examples: vec!["e\u{301}".into()], ..entry("ja-JP", 1) }], )); - for sep in ["\u{2028}", "\u{2029}", "\u{202e}", "\u{200b}"] { + for sep in ["\u{2028}", "\u{2029}", "\u{202e}", "\u{200b}", "\u{3164}", "\u{115f}", "\u{ffa0}"] { cases.push(( "line separator or format character in a choice", vec![OntologyEntry { choice: format!("a{sep}b"), ..entry("ja-JP", 1) }], diff --git a/schemas/cultnet/weksa.ontology_entry.schema.json b/schemas/cultnet/weksa.ontology_entry.schema.json index e976470..080275b 100644 --- a/schemas/cultnet/weksa.ontology_entry.schema.json +++ b/schemas/cultnet/weksa.ontology_entry.schema.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "OntologyEntry", - "description": "One choice of one language. Key `onto::`.\n\nThe store judges the form of an entry's text (bounds, normalization, no\ncontrol or format character) and never its meaning: what may enter an entry\nis owned by the minting flow (Train-only assembly) and the scorer's leak\ndiagnostic (ruling `leak-owned-by-flow-and-scorer-3`), not by a door here.", + "description": "One choice of one language. Key `onto::`.\n\nThe store judges the form of an entry's text (bounds, normalization, no\ncontrol or format character) and never its meaning: no door judges text\n(ruling `leak-owned-by-flow-and-scorer-3`). No Train-only ontology mint\nexists yet, so a seed or a drafted entry is checked against held-out text by\nnothing today; follow-up `leak-diagnostic-in-round` owns that diagnostic.", "type": "object", "properties": { "annotator": { From 6f14b00090f77a0888a46e41a350486b6122a845 Mon Sep 17 00:00:00 2001 From: Weksa Date: Thu, 12 Nov 2026 15:17:36 +0000 Subject: [PATCH 09/57] language-ontology-seed: model-drafted en-US and ja-JP seed entries; the target-language YAML, its typed copy and their schema prose deleted Entries are Annotator::Model (Anthropic), inventory revision 1, drafted from the YAML and standard references, not from any corpus text. pt-BR profiles are deleted unconverted. Citations name data/ontology/ontology.cc. Co-Authored-By: Claude Sonnet 5.5 --- .weksa/cultural-ontology/README.md | 27 ---- .weksa/cultural-ontology/starter-targets.cc | Bin 13426 -> 0 bytes README.md | 7 +- crates/weksa/tests/original_corpus.rs | 63 ++++++++ data/ontology/ontology.cc | Bin 0 -> 9181 bytes data/target-language-ontology/README.md | 39 ----- data/target-language-ontology/en-US.yaml | 135 ------------------ data/target-language-ontology/ja-JP.yaml | 70 --------- .../pt-BR-overlays.yaml | 76 ---------- .../pt-BR-umbanda.yaml | 72 ---------- data/target-language-ontology/pt-BR.yaml | 74 ---------- docs/multilingual-utterance-lowering.md | 22 ++- examples/cultcache-lowering/README.md | 9 +- .../nibu-proof-001/README.md | 4 - .../nibu-proof-001/en-transatlantic.yaml | 3 - .../ja-JP-kanto-contemporary.yaml | 3 - .../pt-BR-vanilla-brazilian.yaml | 3 - .../nibu-proof-001/realizations.yaml | 6 +- prompts/nibu-cultcache-target-critic.md | 2 - prompts/nibu-cultcache-target-lowering.md | 24 +--- schemas/README.md | 3 - schemas/cultural-ontology-profile.md | 4 +- schemas/target-language-cultural-ontology.md | 81 ----------- 23 files changed, 86 insertions(+), 641 deletions(-) delete mode 100644 .weksa/cultural-ontology/README.md delete mode 100644 .weksa/cultural-ontology/starter-targets.cc create mode 100644 data/ontology/ontology.cc delete mode 100644 data/target-language-ontology/README.md delete mode 100644 data/target-language-ontology/en-US.yaml delete mode 100644 data/target-language-ontology/ja-JP.yaml delete mode 100644 data/target-language-ontology/pt-BR-overlays.yaml delete mode 100644 data/target-language-ontology/pt-BR-umbanda.yaml delete mode 100644 data/target-language-ontology/pt-BR.yaml delete mode 100644 schemas/target-language-cultural-ontology.md diff --git a/.weksa/cultural-ontology/README.md b/.weksa/cultural-ontology/README.md deleted file mode 100644 index a424e82..0000000 --- a/.weksa/cultural-ontology/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# Cultural Ontology CultCache Seeds - -This directory stores Weksa cultural ontology as CultCache `.cc` witnesses. - -## Starter Database - -- `starter-targets.cc` - -This store contains one catalog entry and three draft guarded profile entries: - -- `weksa.cultural_ontology.pt-BR.vanilla_brazilian.v0` -- `weksa.cultural_ontology.en.transatlantic.v0` -- `weksa.cultural_ontology.ja-JP.kanto.contemporary.v0` - -The catalog entry is: - -- `weksa.cultural_ontology.catalog.seed_targets.v0` - -The `.cc` file is the typed state witness. This README is only an inspection -guide; it does not own cultural ontology truth. - -The profiles follow `schemas/cultural-ontology-profile.md`: salience axes and -linguistic affordances are nested typed state with feature IDs, activation -gates, constraints, indexicality, evidence, uncertainty, and trace labels. - -Qdrant retrieval may later propose evidence bundles for these profiles. It does -not own accepted profile state. diff --git a/.weksa/cultural-ontology/starter-targets.cc b/.weksa/cultural-ontology/starter-targets.cc deleted file mode 100644 index 680b06ca60a0efc878ed82d17209886f5bb14c45..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13426 zcmd^FYmXd96-DAR2>wCE2?Uaz*>#W&*aFLTkibsBtZgEKP^+f9W~RN}U6Zcr-5n4T z4j}Oqp53+0hX5kMHhK85_yH!^k+Ap?CGpMV!N1_#dUVh1BfDOS4=bW5ne9h))va^y zJ?GY4{&QSqI_7a-w3U{nXwUCmy7x2lq7ku0>#WN+Axx#sw>)qS&PivjSJ zEypOozeizOIsZD(#hwFO4Y_B^KmILIs#F$Rzkx~MrCj74ys_uNcfR|B(N>c>x4CHs zrR=7e*mMMce{<9L2R_U+VT|pX#y|fErfDXzZPWNy&aJGN>8-Y136Vsamp!3Xdwyn9 zV|kq2dS>RC{jC}N`^8gx_Z`@`|G?hQzxdCOKK`3ctr+YfU z__F|i=4XC2DS1~%y^5EKNG2w^|G>U|2WDnoTsF^aW{ZJt9hq&<^CHbM9(79oy);XC zK{I}Ovv)+%)+H|#*BLK#8q>p{+w9>pymjnEdkzoEc8s};d>~6+4)M~D-X6aY*=M3N ze83B##EwS1Ju$CH6um4}{YP5K>N;EO#CXf*;7Px6M-jJjk|4nRap_!OZ6G4xaLYJwS=NZfSkOAZasRUa{HB6<1nqr+w zGu?;XSo1xMg_jhQ3kBnGECyO6j6h#wsBOgIEW?YX5Hu>q48=HRnTRY!B;s0RL)IxJ zPngA7K;1!1sfWOZ8OeE3W_+kvUMbBwSf?0pdb1v}z4Ay~u&E>Z8BSolcEpeW^j{zO z#OAK(-+feI-$@IlQ(bAh)`%9Zuy>PAci~l0Toy%ZW@hgnpX0iigdzKigW+hC%eV40 zX&v^y6dpcjZXdQjpU%UQ37Y_Rc*yPnO~-;Ng>z+D+Dki`U~wsxYDUHC0SzE8NhxJk zqRmeAQ^oLaAvNoX0^p0;;hg@*i>ZmLN6J*gjiwkc5_^NQSmq)b0x7VDjIr4kV1;fk zt(Yei&r*w8z6dmJjCZq#$C}m#yV`N_SSL+@_Ev?N_ONJWo&dUO@fjSX!j~{Z-o=B7 zb)8bg_^IBhorIy=mo|l2wU|xj>XFdNHB{V!9s7WfQCz_QhvdXUJrh(CFi#9xjb7Nqcbefe>-u zd|}pmgbxf3IYRdfY!>iV&t{#~`DsFAf$*}>7X?!>Fo?H#iB-vR3Nxu4Z;D3Z6}H8O z%3Nrkfcs$`a}HmVa|~-*8JVZuvLf&OkrHGQr2Wn?3ZWNH{ZfThEtx01A4J`Z&&x9M zNbnrNzl^QEol9V$97cGC+)qe79Ei!7uyh;nQX=xZ5aqm_0}{+*0e1Ff9+#L*m~a$l zJk4Leo{2?xVHR~b5^OA1)?T;_Qh@8`^x=7#BBbk5BE$^Y>60yMP+05qare^?1+>N9 zdR2iKub6iVkGaN_sUBUtYokXL@_UqFk2~N6dg$&naizIq4xhBPt=%X~=O}k*2@K>s zEeM~)-|0$6>ON*`ec<$QJivi8wPdEAm53OcA`^Kd#SS%Ib{Mf*LUi06A?NXUF9VS*^gk|vEA;hhiTo)R z5QTCHCq$Zo_357#`p<=>?YWToOd{iryo_RXi^Cm;DphTyHyZUa<*_PXf`jfmNqn>m zcvht#QfU*nVi_g^(Id-2rCJ?Iy$Mlds15)Y2Hg%g9y|bwo>+$>@BoF6CGd7o2+N>v zKJTx=JHs`Po7ds5F$jbbS6ytdkzAC2ZhSCW;3Y8&Gy8)3p==lc6LzB^!GTv}Z5pmZ zel_OZjjs}WK}uo@F~0|5ftBhZR{B)%sPLIOR&=teu+xvMe~a?Ynx#=bLB1cEgU>qL zYFWj7>zN79c@H-At2oF~?W6^W&rnwITn3ia6J?tqXOnZpGt5s#&FpIoj?9x4WOSh; z5W^fR!5z1e$YG_3(j4N((v#hdx^02!!Gm+l|2TK<{AT{;t}k7)sw(1DV#)F)1RtI) z-x#y54XroqV=-4cB4)<3n-l2-TP%3FZ1#s8MlOXCoZxI;e-Q2SR6a!Ows~%Q?*(BH zUbLA78BX=tJm%f0GLrP>+kP~F4)KaI*_cECA@o^tigm%jZ7PD|%)NFXN(uc^^eo*2 zbiq;xo~Rh;_Wv{xonQ501VO>mg4apAL|8|1)dwf{*J}M&Db83b83Je-FrTdKsCKb4 z)%(fTc2)0ADl~WNL=u=m1R3$MQS+tv3jUV879w?25|ccTw|$2{rEOPk4%<=U6l!SsGCQ}#g6^-boewB zAXw8tf;fPs1^QnL7ggapX=nHuJ0<6avW3{iUG)QLp2$qCuQ%4wh>-O{of5(f6i*xJ z3~CE?W0_c(*8j6+UbcKbr`wcZJPI15=AmejKV6}U@^pe~XuTB} z*eY%xYR&$MQNvJf!^SoUFrb>81j%+{w=?Utb*uh_L*zjaLr_F2rC15aRMl}Hg``DZ z1hGP{#b7psL%P`zH})wnZb4`lP9t^_0)6PS#Zu=((>5Wj|F+MyO(Bk6_qub+FJWq8 zO1Q7A^t5D^B&^!4V);+xlLL^;mHJmh+02WR!N*b@3EhKr#}vt|zhko~l%r5uGUnORY3hxNyO?94CyMe77vTMLW~&#WytzV<|(Ja3Z4_KB*o>|Lh{~K!wT?86g+CZ z7A62wAO+i3cvomtVH|g5Jny8vN>*z7dhHDxQ&GQBYGv&A*iUmRcv%y7ZMhigiYk*9 z+0$;Ap!zX?Xcbo>>$FOs2Aqo3DT%G zK%ib%P3PeMuKdMfl+LJ_<%DkiIf^BY?Jt>6z0e1vs&83Gw`TKYMC>Ce%`11LszdE7P=L$2o zL0^jqHR`!LZit%2SScftk@=j%2o>Y{DdIsH_igDd>~|n_D#9dFvbQXDw}yEH6hX?pDd^j-x@?Vz!f9E>r21FGi*@KU3*XpwjJCzXAd!)R zeUZ^IzdB9V(WgriJ5=~HsVHvNakjmD%T7rUnq|j8O;s(fxOSwMyVS z`(xnGV)1AG&$Cwuk)!)zaH_zJhj1s{M&{GXhF=Viq7z|$fy!!82VcuYR0rc-jkym& z5Y=ez)d0)WMMyArq!J~f%R%&SJx>=~?jq_U_M(T@o5;m|i3m3653l;yT)pnb952qK z6fQYD7Xz3f#V>or1F1U6uIvNMzA;U2;IVpGqIQ)qFW{_y2rK0N#TcYk{4-TQy|;I*H; zfA8AEvu{2;dwm1EE#2`F$yO7P859PNj-9yUh-?i)e+WM^pJ~=WF%Z!Y;6p_;7D4IR zAUrR2H2^;xVGkbcxL&hz$tIS|E{j~(I1#@xUfAmyHhgx= zDvWO?nlaao*p88nMrruJi&T~``ZJHosN9rTg6y6mb*<3t0(OQz2WS*mSfQeGg|qh| zZn*voCz*Tz##5xs$xYlLA{-$Yh3xqMT!0x}cAI2o+?GvL9X8y*P_Q;fj*gewsQbXS z16dF-vMXx2P7Z`4Ce}?MV5z&vccGXsSk=_jO=$mIQxUZe_)Ts#``gq>Z2JHeI$X`i zfvFWdqDEPLq!hBTYZ!3fp?K-)Ak)n!Y6$JtipU#{lv)}49hwF{m_6APoPYCNu#t^UhV_3J1XkjAM@Lf&jH$)Fg6{A<) zak%OPqEk1Fcr4v*oH=xNtU;M?61mn8dDO=9Rj;NH&#XvC>s5Zj{@##Ot!o{jG*v6l zdgwX}GT)NbNJcm$`f1T5CCUXf5sfQIKNVl+ppxrNS3`{~RzlmzpD2+N!p@J@OG5kP z$UG$~9EYJR62;R%2t$;rn+y;n5sY>y5<%t!swTK{ zI`7aiSk0M%LE9T^#bBVnwL>)8QpqlQ=t5pcS2yA%GVEGi=s2BiTc@KFCx8i#yGjrK E7fJF~WB>pF diff --git a/README.md b/README.md index 3133190..ef509cb 100644 --- a/README.md +++ b/README.md @@ -82,9 +82,10 @@ English, Brazilian Portuguese, Japanese, and later target-language dialogue. [Flavored English Lowering](./docs/flavored-english-lowering.md) remains the target-specific English receipt. -See [Target-Language Cultural Ontology Profiles](./data/target-language-ontology/README.md) -for the reviewable address, register, politeness, idiom, metaphor, and script -distinctions that target lowerers must inspect before producing output. +The language ontology is `data/ontology/ontology.cc` (read it with +`weksa ontology dump`): per language, the address, register, politeness and script +choices a lowerer makes for a speaker in a stance. Its en-US and ja-JP entries are +model-drafted seeds, never Test evidence; accepted rounds grow them. For the first AquaSynth speech-training handoff, see [Tiny PanPhon v0.1 Speech Curriculum](./examples/speech-training/tiny-panphon-v0.1/README.md). diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 2194a28..4e1dce1 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -251,6 +251,69 @@ fn the_original_inventory_is_the_dao_revision_three() { assert!(InventoryStore::open(©).is_ok()); } +/// The committed seed ontology: every entry model-drafted, exactly the two +/// panel languages, each with a Stance-conditioned entry, and every committed +/// corpus opens a run over it (ruling `ja-ontology-owner-answer`: a model-drafted +/// entry is labelled so and is never Test evidence). +#[test] +fn committed_ontology_is_model_drafted_and_reads_every_inventory() { + use weksa::OntologyStore; + use weksa::ontology::Condition; + use weksa::run::{LowererReach, RunPaths, RunStores}; + use weksa::speaker::Annotator; + let data = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data"); + let tmp = tempfile::tempdir().unwrap(); + let onto = tmp.path().join("ontology.cc"); + std::fs::copy(data.join("ontology/ontology.cc"), &onto).unwrap(); + let entries = OntologyStore::open(&onto).unwrap().all().unwrap(); + for e in &entries { + match &e.annotator { + Annotator::Model { family, generator } => { + assert_eq!(*family, ModelFamily::Anthropic, "{}", e.key()); + assert!(generator.contains("claude"), "{}", e.key()); + } + other => panic!("{} is {other:?}", e.key()), + } + } + let mut langs: Vec<&str> = entries.iter().map(|e| e.language.as_str()).collect(); + langs.sort(); + langs.dedup(); + assert_eq!(langs, PANEL); + for lang in PANEL { + let stance = |c: &Condition| c.slot == weksa::inventory::Slot::Stance; + assert!( + entries + .iter() + .any(|e| e.language == lang && e.when.iter().any(stance)), + "{lang} has no Stance-conditioned entry" + ); + } + for (corpus, panel) in [("original", &PANEL[..]), ("daodejing", &PANEL[..1])] { + let dir = data.join(corpus); + let copy = |f: &str| { + std::fs::copy(dir.join(f), tmp.path().join(format!("{corpus}-{f}"))).unwrap(); + tmp.path().join(format!("{corpus}-{f}")) + }; + let (c, i, e) = (copy("corpus.cc"), copy("inventory.cc"), copy("entity.cc")); + RunStores::create( + &tmp.path().join(format!("run-{corpus}")), + corpus, + RunPaths { + corpus: &c, + inventory: &i, + entities: &e, + ontology: &onto, + }, + "m", + "bag", + LowererReach::BriefedSubagent, + ModelFamily::Unrecorded, + &panel.iter().map(|l| l.to_string()).collect::>(), + ) + .unwrap_or_else(|e| panic!("{corpus}: {e:?}")); + } +} + /// The probe over the real original corpus: every item is offered, none names a /// work, and a model that answers with unrelated words reads at the noise floor. #[test] diff --git a/data/ontology/ontology.cc b/data/ontology/ontology.cc new file mode 100644 index 0000000000000000000000000000000000000000..94eeb19bb785bae535198a500c282f330f069d9b GIT binary patch literal 9181 zcmc&)ZEqY`8O1H7Erl;21pRC!1P8+TL{1?BO8~B~a z|NVuBPhFxXsfn<%XR0>%PrJN!&DxEX@$t&|+TG(DYUAr`>(>43pCjz_Idj#>$c4?m za%2)KalChY!gXVz6w{tyfv>a(Ma*137LE?uY}ON@IiwsMU?HB>oZ8yNnnFa`p-2eF zW8Bfq4>f*`F(nyivpf#{u)&n(ZB=;BLl^I2@q8kb?jN=9t8-VkiuM?5%cRP-nZJi= z$y%b?Ik#Q9BIwT7dCL#l7bZgO#WM1p3k~|4IV#@!ffMk=6%{4JQ0U5r$_Dnmy;Og; z72m4HSc`5^absTB!evU}N6w7Zr`|3?yJlbsmcjO07TYh6ENl}8eMg3KhtH}&>iH?n zLq{ys1=mR|F52I<@r`FSp1!j%p|=UH^uO-e*V{KyuLE`Pq(<1afM#9BEgk@?1|bo% zSzmh$ch@ms#)){Mx*r3@tT2Quz|etUek&qO{28IDbJq-{nm6&eX81k;EStRj#E>y9 z#n*3GOdl%nbtP0EU5e_SDG#KrWjIv;3b--G0y$f0iZ&y}JmEHkJC=dVnet?;8IO!y zFkt8bcHt+7#8Pa5fWSq@%&c{KO^jA-THv=Z>QPUE zfYGv5K=u8-+=qP30@NqiPh|ojat*}8X6m{!!0xudezm~V6DzF$HA|q)E?Cs&Tr)1@ znqkR3KcTlvD8|U~-u|Zqx1s<-Ge^XlqZI8Ln~r4@%AIjJLl|g>p=i+;d%J_602h!J9*FV_M4985=8%EKUPV%|S+Y=S&@oYLM#R%rn8}}@^C_|o=%^V>U1F_ID&(pffeAFVjJS+A0*7 zDg&J{ML~sPYL`?)e1_sE3D*UymaQ^;U^*}S$;It*MpW63p~JtS4EzsU`0wk(ztT#u za28G8{%01^OF|c55XMD9krN#sn9wts-Mvb2bkwQTjAo&v|&i5GB?-q8y@_` zB!Ke6CaDW&)4twhfb;mTc${HLI3^@fPTLG5R$UD3;TKXblV5{RS5M^+_wY zb?ygMMr@`2XwA!z@qDZdo|`OLA1H;V6@fP%w^a^vK69{y`p#gk=SDxNgE*Sd$>2v0)PX|LU?CL+aS)QImgGU>ur=g`UY2{N? zfdr{X57Qsfsbelw6#X9A05R4LV~*}&$n5UiJ4RqMHs?W^~B%O z&e2a^Kb>}7NIS2lo$G1mwe(Q;TH5(z+BumX>fCtwy|iz}@%JSw>`0E|n3D zSeWlTBf4FQkrDmSrWplzGdD%?gpDi=qahm-Arei`i@9EP?t`1B{t6^-oyX5^+C6F@ zJ(G5U<*>mk6$bs-f>#WK2Aki-h;4j{2&^!Z53p;`<#vtILhh0Z)EK$n0F!bZIa1%E z<@UEy_d_T(@(mhNU`Dc0TVgUXwaBd;wMfj@hRH9codsfc=WS5>KZoY=Z^%q8HDG_! z7UTa4l6Ef!NyYRi%jrlz5nu4d+z67AM@kZ^Ekd`b**F?XmxJGF=SbRl6EJ@Khf6m; zI++7|Y$*KBCc8`=N+J4?MYI@M%eFN2>pNQN08a9fdwK1lxT*1~++{hgInBIYY;=TB+p_oTAJ zP>CXnWupp%{bE!xSR$66k3YOKg(%F&`#*C&&c-=MB5cLP2AEL;&&LEpQE0|tjK(%` z#u?;6B#fVjO2vNNchCYanR*Um$Ts*F(QNXTgq4f2 zD~fhVsJEqr{Zk3%4Ihw_@gv{5Zo*DToO0*%|99%k+xx6j;|OOWqV{!D;kc;N&vkWO+X>5nj3c?js7%*JV)G%&t< zqz5RI$gyuR - Bounded English dialogue lowering profile for current Weksa fixtures. This is - not a claim about all English-speaking cultures. -authority: - owns: - - address directness choices - - casual versus formal register - - technical metaphor legibility - - understatement or sarcasm risk - does_not_own: - - source meaning - - speaker identity - - whole-culture description -salience_axes: - address: - relevant_distinctions: - - direct second person is acceptable when scene context names the addressee - - imperative force can be softened or sharpened through wording - lowering_questions: - - Does direct address preserve the speech act? - - Does softening erase urgency? - register: - relevant_distinctions: - - casual - - technical - - formal - - intimate - - institutional - lowering_questions: - - Is technical language character pressure or exposition sludge? - - Does sarcasm obscure the actual warning? - agency: - relevant_distinctions: - - explicit agent responsibility is usually available - - passive phrasing can hide authority or blame - lowering_questions: - - Should the speaker own the intervention directly? - - Does the line preserve who is at risk? -linguistic_affordances: - grammar_features: - - feature_id: en_us_direct_imperative_warning - label: Direct imperative warning - status: draft - domain: - layer: grammar - category: clause_mood - subcategory: imperative - description: > - Broad en-US allows direct imperatives for urgent safety warnings when - the speaker has standing to intervene and the scene rewards clarity - over mitigation. - formal_conditions: - syntactic_environment: - - main_clause - - direct_address_optional - semantic_inputs: - - warning - - addressee_action_to_stop_or_avoid - - consequence_or_risk_available - required_interlingua_fields: - - speech_act - - addressee - - prohibited_action - realization: - function: stop_or_prevent_action - surface_strategy: bare_imperative_or_do_not_imperative - examples: - - "Don't touch the damaged control." - activation: - allowed_when: - - scene_has_immediate_risk - - speaker_has_contextual_authority_or_care_relation - forbidden_when: - - politeness_softening_is_required_by_target_profile - - imperative_would_falsely_imply_hierarchy - requires_profile_authority: false - minimum_density: light - maximum_density: normal - constraints: - grammar_contexts: - - avoid_if_source_requires_indirect_request - register_contexts: - - casual - - technical - - emergency - speaker_contexts: - - known_addressee - - operator_warning - incompatibilities: - - generic_assistant_over_politeness - indexicality: - social_meanings: - - urgency - - clarity - - responsibility - in_group_meanings: [] - out_group_risks: - - may sound abrupt if the scene is not urgent - stereotypes_to_avoid: [] - evidence: - source_refs: - - schemas/cultural-ontology-profile.md - qdrant_source_ids: [] - cached_source_refs: [] - source_hashes: [] - reviewer_refs: [] - confidence: medium - uncertainty: - open_questions: - - Should a later en-US regional overlay override imperative force? - known_limits: - - Base en-US does not model regional, racialized, or contact-language - dialect grammar. - trace: - trace_label: direct_imperative_warning - must_report_when_used: true - must_report_when_rejected: true - phonology_features: [] - lexicon_features: [] - discourse_patterns: [] - orthography_features: [] - code_switching: - allowed: false - constraints: - - Base en-US profile does not authorize contact-language switching. - patterns: [] -forbidden_flattenings: - - Do not treat English as the neutral source for other target languages. - - Do not use generic assistant politeness when a character voice is required. - - Do not let sarcasm replace the actual predicate being lowered. diff --git a/data/target-language-ontology/ja-JP.yaml b/data/target-language-ontology/ja-JP.yaml deleted file mode 100644 index aa23492..0000000 --- a/data/target-language-ontology/ja-JP.yaml +++ /dev/null @@ -1,70 +0,0 @@ -profile_id: weksa.target_cultural_ontology.ja-JP.v0 -language: - code: ja - locale: ja-JP - script: Jpan -scope: > - Bounded Japanese dialogue lowering profile for Weksa fixtures. This profile - records realization distinctions that must be inspected; it is not a total - description of Japanese culture. -authority: - owns: - - politeness and honorific pressure - - in-group/out-group stance when context supplies it - - subject and object omission choices - - script mix and technical register - - sentence-final force - does_not_own: - - source meaning - - character identity - - exhaustive Japanese grammar -salience_axes: - politeness: - relevant_distinctions: - - plain - - polite - - honorific - - humble - - rough - lowering_questions: - - Does politeness preserve the relationship, or does it lie about it? - - Is roughness character-specific or generic stylization? - social_position: - relevant_distinctions: - - uchi_soto_pressure - - hierarchy - - familiarity - - emergency_override - lowering_questions: - - Does urgency override ordinary deference? - - Does the line imply a relationship the scene has not earned? - ellipsis: - relevant_distinctions: - - subject omission - - object omission - - explicit pronoun for emphasis or contrast - lowering_questions: - - Is the addressee recoverable without an explicit pronoun? - - Would explicit pronouns sound unnatural or too pointed? - script_and_register: - relevant_distinctions: - - kanji for technical compounds - - kana for softness or readability - - katakana for foreignness, machine texture, or emphasis - lowering_questions: - - Does the script choice help the speaker voice? - - Does a technical compound preserve precision without becoming stiff? - sentence_final_force: - relevant_distinctions: - - direct request - - warning consequence - - command - - threat-like phrasing - lowering_questions: - - Does the ending carry warning force without cartoon aggression? - - Does the line preserve the hazard predicate? -forbidden_flattenings: - - Do not treat politeness as decoration. - - Do not use explicit pronouns just because English has them. - - Do not make every sharp line rough or anime-coded. - - Do not erase the difference between technical precision and formal politeness. diff --git a/data/target-language-ontology/pt-BR-overlays.yaml b/data/target-language-ontology/pt-BR-overlays.yaml deleted file mode 100644 index d7da0d9..0000000 --- a/data/target-language-ontology/pt-BR-overlays.yaml +++ /dev/null @@ -1,76 +0,0 @@ -catalog_id: weksa.target_cultural_ontology.pt-BR.overlays.v0 -base_profile: data/target-language-ontology/pt-BR.yaml -scope: > - Catalog of optional Brazilian Portuguese regional and cultural overlay slots. - An overlay slot is not permission to imitate a community. It is a named place - where reviewed, source-grounded distinctions can live. -activation_rule: > - Use the broad pt-BR profile by default. Add an overlay only when source - context, speaker state, setting documentation, or authoring metadata names the - relevant region, community, institution, religion, or register. -overlay_slots: - - overlay_id: broad_brazilian_default - status: active_base_mode - profile_ref: data/target-language-ontology/pt-BR.yaml - owns: - - broadly legible Brazilian Portuguese dialogue - does_not_own: - - regional specificity - - religious-cultural specificity - - overlay_id: rio_urban - status: planned_profile - profile_ref: - owns_when_defined: - - Rio de Janeiro urban speech/register pressures for specific characters or settings - blocked_until: - - reviewed source context - - human-approved examples - - overlay_id: sao_paulo_urban - status: planned_profile - profile_ref: - owns_when_defined: - - Sao Paulo urban speech/register pressures for specific characters or settings - blocked_until: - - reviewed source context - - human-approved examples - - overlay_id: northeast_regional - status: planned_profile - profile_ref: - owns_when_defined: - - Northeast Brazilian regional speech/register pressures for specific characters or settings - blocked_until: - - specific region named - - reviewed examples - - human approval - - overlay_id: south_regional - status: planned_profile - profile_ref: - owns_when_defined: - - South Brazilian regional speech/register pressures for specific characters or settings - blocked_until: - - specific region named - - reviewed examples - - human approval - - overlay_id: amazonian_regional - status: planned_profile - profile_ref: - owns_when_defined: - - Amazonian regional speech/register pressures for specific characters or settings - blocked_until: - - specific region or community named - - reviewed examples - - human approval - - overlay_id: umbanda - status: active_guarded_overlay - profile_ref: data/target-language-ontology/pt-BR-umbanda.yaml - owns: - - Umbanda-related register gating - - sacred address caution - - anti-caricature constraints - blocked_until: - - source context or authoring metadata explicitly activates Umbanda context -forbidden_flattenings: - - Do not use a planned overlay as if it were defined. - - Do not treat regional slots as accent generators. - - Do not use religious-cultural overlays for ambience. - - Do not collapse race, region, religion, class, and slang into one Brazilian flavor. diff --git a/data/target-language-ontology/pt-BR-umbanda.yaml b/data/target-language-ontology/pt-BR-umbanda.yaml deleted file mode 100644 index c65289d..0000000 --- a/data/target-language-ontology/pt-BR-umbanda.yaml +++ /dev/null @@ -1,72 +0,0 @@ -profile_id: weksa.target_cultural_ontology.pt-BR.umbanda.v0 -language: - code: pt - locale: pt-BR - script: Latn -overlay: - base_profile: data/target-language-ontology/pt-BR.yaml - kind: religious_cultural_register -scope: > - Optional Brazilian Portuguese cultural ontology overlay for Umbanda-related - scenes, speakers, ritual contexts, or referents. This profile is a consent and - context gate, not a decorative vocabulary pack. -authority: - owns: - - whether Umbanda-specific register is allowed for a realization - - sacred or ritual address caution - - house/tradition specificity warnings - - syncretic and entity-reference sensitivity - - anti-caricature constraints - does_not_own: - - Umbanda doctrine as a whole - - authority to invent ritual speech - - generic Brazilian Portuguese - - speaker identity unless the speaker context explicitly supplies it -activation_requirements: - any_required: - - source context names an Umbanda scene, terreiro, practitioner, entity, ritual, or discourse community - - speaker profile explicitly owns an Umbanda-informed voice or memory - - authoring metadata requests this overlay for reviewed cultural grounding - never_activate_for: - - generic Brazilian flavor - - jokes, insults, mystic ambience, or unexplained exotic texture - - non-Umbanda religious scenes unless the source context makes a syncretic relation explicit -salience_axes: - sacred_address: - relevant_distinctions: - - ordinary dialogue - - respectful reference - - ritual address - - entity_or_guide_reference - lowering_questions: - - Is the line ordinary speech or ritualized speech? - - Does the speaker have standing to use sacred address? - - Should a term be left general until a human reviewer approves specificity? - house_specificity: - relevant_distinctions: - - practices and terms can vary by house, region, and lineage - - generic flattening can become false authority - lowering_questions: - - Is the output pretending one practice represents all Umbanda? - - Should the trace mark the choice as house-specific or uncertain? - syncretic_reference: - relevant_distinctions: - - Catholic, African, Indigenous, Spiritist, and local references may coexist differently by context - - entity names and roles are not interchangeable decoration - lowering_questions: - - Does the source actually contain the syncretic relation? - - Does the line preserve respect without overexplaining? - anti_caricature: - relevant_distinctions: - - religious register - - folkloric stereotype - - occult ambience - lowering_questions: - - Is the output using Umbanda as culture, faith, and practice rather than spooky texture? - - Would a practitioner recognize this as respectful enough for review? -forbidden_flattenings: - - Do not use Umbanda terms as generic magic flavor. - - Do not invent ritual formulas. - - Do not collapse Umbanda into Candomblé, Spiritism, Catholicism, or generic Afro-Brazilian religion. - - Do not imply all Brazilian Portuguese religious language is Umbanda-coded. - - Do not activate this overlay unless source context or authoring metadata explicitly asks for it. diff --git a/data/target-language-ontology/pt-BR.yaml b/data/target-language-ontology/pt-BR.yaml deleted file mode 100644 index 7fe8dbc..0000000 --- a/data/target-language-ontology/pt-BR.yaml +++ /dev/null @@ -1,74 +0,0 @@ -profile_id: weksa.target_cultural_ontology.pt-BR.v0 -language: - code: pt - locale: pt-BR - script: Latn -scope: > - Bounded Brazilian Portuguese dialogue lowering profile. This profile is the - base Brazilian Portuguese target; regional, class, religious, and community - registers should be added as explicit overlays rather than silently folded - into generic pt-BR. -authority: - owns: - - Brazilian Portuguese lexical preference - - treatment and address choices - - directness versus mitigation - - regional_or_cultural_overlay_selection - - idiom naturalness for reviewed fixture prose - does_not_own: - - source meaning - - Brazilian culture as a whole - - regional or religious-community authority without an explicit overlay - - target speaker identity -salience_axes: - treatment: - relevant_distinctions: - - voce - - tu_regionally_marked - - o_senhor_a_senhora - - name_or_title_address - lowering_questions: - - Is the relationship close, hostile, hierarchical, unknown, or sacred? - - Does `voce` preserve broad Brazilian naturalness better than `tu`? - - Is `tu` regionally grounded or just imported flavor? - directness: - relevant_distinctions: - - bare imperative - - softened imperative - - warning framed as consequence - - formal warning - lowering_questions: - - Does urgency justify blunt imperative force? - - Would mitigation make the warning dishonest? - lexical_locale: - relevant_distinctions: - - Brazilian Portuguese terms should be preferred in reviewed pt-BR fixtures - - European Portuguese defaults must be marked if intentionally used - lowering_questions: - - Is a term recognizably Brazilian Portuguese in this context? - - Is a literal technical word natural enough for dialogue? - regional_and_cultural_overlay: - relevant_distinctions: - - broad_brazilian_default - - regional_speech_pattern - - class_or_institutional_register - - religious_or_ritual_register - - community_specific_lexicon - lowering_questions: - - Does the source context justify an overlay? - - Is the overlay owned by a named profile? - - Would applying the overlay stereotype the speaker or setting? - metaphor: - relevant_distinctions: - - technical metaphors can preserve character ontology - - over-literal metaphors may sound translated rather than spoken - lowering_questions: - - Does the metaphor still sound like the speaker? - - Does it preserve the warning instead of becoming decorative? -forbidden_flattenings: - - Do not label output as Portuguese without a locale in reviewed fixtures. - - Do not default to European Portuguese wording when the target is pt-BR. - - Do not treat Brazilian Portuguese as one culture, region, class, or religion. - - Do not add regional or religious markers without a cited overlay profile and source-context reason. - - Do not soften a warning merely because directness feels impolite in English review. -overlay_catalog: data/target-language-ontology/pt-BR-overlays.yaml diff --git a/docs/multilingual-utterance-lowering.md b/docs/multilingual-utterance-lowering.md index fb1f22a..6d01bbb 100644 --- a/docs/multilingual-utterance-lowering.md +++ b/docs/multilingual-utterance-lowering.md @@ -76,8 +76,8 @@ English remains a lowering target, not the pivot language. Existing flavored English fixtures continue to be useful, but new multilingual fixtures should show English as one realization in a set. -The initial cultural ontology profile is -`data/target-language-ontology/en-US.yaml`. +The en-US choices are the `en-US` entries of `data/ontology/ontology.cc` +(model-drafted seeds; read with `weksa ontology dump`). ### Brazilian Portuguese @@ -94,16 +94,10 @@ The target lowerer should record: defaults - optional regional or cultural overlays when the source context earns them -The initial cultural ontology profile is -`data/target-language-ontology/pt-BR.yaml`. - -Umbanda-related scenes, speakers, ritual contexts, or referents may add the -`data/target-language-ontology/pt-BR-umbanda.yaml` overlay. That overlay must -not activate for generic Brazilian flavor. - -Regional or cultural variants should be selected through the overlay catalog at -`data/target-language-ontology/pt-BR-overlays.yaml`. Planned overlay slots are -not active profiles until reviewed examples and source context exist. +No pt-BR ontology entry exists yet; the cut that adds pt-BR writes them to +`data/ontology/ontology.cc`. Umbanda-related scenes, speakers, ritual contexts, +or referents may later earn a regional or cultural variant, which must not +activate for generic Brazilian flavor. ### Japanese @@ -119,8 +113,8 @@ The target lowerer should record: - whether subject omission preserves or harms clarity - whether katakana, kanji, kana, or mixed script choices carry useful pressure -The initial cultural ontology profile is -`data/target-language-ontology/ja-JP.yaml`. +The ja-JP choices are the `ja-JP` entries of `data/ontology/ontology.cc` +(model-drafted seeds; read with `weksa ontology dump`). ## Packet Shape diff --git a/examples/cultcache-lowering/README.md b/examples/cultcache-lowering/README.md index 4a786a0..ed4e037 100644 --- a/examples/cultcache-lowering/README.md +++ b/examples/cultcache-lowering/README.md @@ -1,11 +1,8 @@ # CultCache Lowering Examples -These examples lower interlingua packets through CultCache cultural ontology -profiles rather than the older YAML-only target-language profiles. - -The starter database is: - -- `.weksa/cultural-ontology/starter-targets.cc` +These examples lower interlingua packets for separate target languages. The +language ontology that owns en-US and ja-JP choices is `data/ontology/ontology.cc` +(`weksa ontology dump`); its entries are model-drafted seeds. Each target file is produced as a separate agent run so differences in target ontology remain inspectable instead of being smoothed into one multilingual diff --git a/examples/cultcache-lowering/nibu-proof-001/README.md b/examples/cultcache-lowering/nibu-proof-001/README.md index 4bb85be..67661ca 100644 --- a/examples/cultcache-lowering/nibu-proof-001/README.md +++ b/examples/cultcache-lowering/nibu-proof-001/README.md @@ -8,10 +8,6 @@ Prompt: - `prompts/nibu-cultcache-target-lowering.md` -CultCache profile store: - -- `.weksa/cultural-ontology/starter-targets.cc` - Expected target outputs: - `pt-BR-vanilla-brazilian.yaml` diff --git a/examples/cultcache-lowering/nibu-proof-001/en-transatlantic.yaml b/examples/cultcache-lowering/nibu-proof-001/en-transatlantic.yaml index 8d4667e..3484adf 100644 --- a/examples/cultcache-lowering/nibu-proof-001/en-transatlantic.yaml +++ b/examples/cultcache-lowering/nibu-proof-001/en-transatlantic.yaml @@ -1,7 +1,6 @@ schema_version: weksa.utterance_lowering.agent_run.v0 source_packet_id: nibu-proof-001 source_packet_ref: examples/english-lowering/nibu-proof-001/interlingua.yaml -target_profile_ref: .weksa/cultural-ontology/starter-targets.cc target_profile_id: weksa.cultural_ontology.en.transatlantic.v0 target_agent_id: nibu realization: @@ -11,8 +10,6 @@ realization: locale: en-001 script: Latn register: cultivated transatlantic English; cross-Atlantic intelligibility; measured technical warning - cultural_ontology_refs: - - .weksa/cultural-ontology/starter-targets.cc cultural_ontology_profile_ids: - weksa.cultural_ontology.en.transatlantic.v0 outputs: diff --git a/examples/cultcache-lowering/nibu-proof-001/ja-JP-kanto-contemporary.yaml b/examples/cultcache-lowering/nibu-proof-001/ja-JP-kanto-contemporary.yaml index 10a706e..4e88905 100644 --- a/examples/cultcache-lowering/nibu-proof-001/ja-JP-kanto-contemporary.yaml +++ b/examples/cultcache-lowering/nibu-proof-001/ja-JP-kanto-contemporary.yaml @@ -1,7 +1,6 @@ schema_version: weksa.utterance_lowering.agent_run.v0 source_packet_id: nibu-proof-001 source_packet_ref: examples/english-lowering/nibu-proof-001/interlingua.yaml -target_profile_ref: .weksa/cultural-ontology/starter-targets.cc target_profile_id: weksa.cultural_ontology.ja-JP.kanto.contemporary.v0 target_agent_id: nibu realization: @@ -11,8 +10,6 @@ realization: locale: ja-JP script: Jpan register: contemporary Kanto metropolitan clipped hazard warning - cultural_ontology_refs: - - .weksa/cultural-ontology/starter-targets.cc cultural_ontology_profile_ids: - weksa.cultural_ontology.ja-JP.kanto.contemporary.v0 outputs: diff --git a/examples/cultcache-lowering/nibu-proof-001/pt-BR-vanilla-brazilian.yaml b/examples/cultcache-lowering/nibu-proof-001/pt-BR-vanilla-brazilian.yaml index 7c0788f..bc05339 100644 --- a/examples/cultcache-lowering/nibu-proof-001/pt-BR-vanilla-brazilian.yaml +++ b/examples/cultcache-lowering/nibu-proof-001/pt-BR-vanilla-brazilian.yaml @@ -1,7 +1,6 @@ schema_version: weksa.utterance_lowering.agent_run.v0 source_packet_id: nibu-proof-001 source_packet_ref: examples/english-lowering/nibu-proof-001/interlingua.yaml -target_profile_ref: .weksa/cultural-ontology/starter-targets.cc target_profile_id: weksa.cultural_ontology.pt-BR.vanilla_brazilian.v0 target_agent_id: nibu realization: @@ -11,8 +10,6 @@ realization: locale: pt-BR script: Latn register: casual_clear - cultural_ontology_refs: - - .weksa/cultural-ontology/starter-targets.cc cultural_ontology_profile_ids: - weksa.cultural_ontology.pt-BR.vanilla_brazilian.v0 outputs: diff --git a/examples/multilingual-lowering/nibu-proof-001/realizations.yaml b/examples/multilingual-lowering/nibu-proof-001/realizations.yaml index 7ac410d..e8c8080 100644 --- a/examples/multilingual-lowering/nibu-proof-001/realizations.yaml +++ b/examples/multilingual-lowering/nibu-proof-001/realizations.yaml @@ -22,7 +22,7 @@ realizations: script: Latn register: sharp technical casual cultural_ontology_refs: - - data/target-language-ontology/en-US.yaml + - data/ontology/ontology.cc outputs: visible_action: The cockpit lights snap from amber to surgical white around the damaged control. spoken_text: "Take your hand off that control unless you want your nervous system to become my least useful circuit diagram." @@ -52,8 +52,6 @@ realizations: locale: pt-BR script: Latn register: direct technical casual - cultural_ontology_refs: - - data/target-language-ontology/pt-BR.yaml outputs: visible_action: As luzes do cockpit passam de âmbar para branco cirúrgico ao redor do controle danificado. spoken_text: "Tira a mão desse controle, a não ser que você queira transformar o seu sistema nervoso no meu diagrama de circuitos menos útil." @@ -90,7 +88,7 @@ realizations: script: Jpan register: sharp technical plain cultural_ontology_refs: - - data/target-language-ontology/ja-JP.yaml + - data/ontology/ontology.cc outputs: visible_action: 損傷した制御盤の周囲で、操縦室の灯りが琥珀色から外科灯のような白へ切り替わる。 spoken_text: "その制御盤から手を離して。神経系を、私のいちばん役に立たない回路図にしたくないなら。" diff --git a/prompts/nibu-cultcache-target-critic.md b/prompts/nibu-cultcache-target-critic.md index c339739..698d38b 100644 --- a/prompts/nibu-cultcache-target-critic.md +++ b/prompts/nibu-cultcache-target-critic.md @@ -10,8 +10,6 @@ You are reviewing one target lowering produced from the Nibu interlingua packet. `examples/english-lowering/nibu-proof-001/projected-context.yaml` - Lowering prompt: `prompts/nibu-cultcache-target-lowering.md` -- CultCache profile store: - `.weksa/cultural-ontology/starter-targets.cc` The source meaning is: diff --git a/prompts/nibu-cultcache-target-lowering.md b/prompts/nibu-cultcache-target-lowering.md index d044169..512d257 100644 --- a/prompts/nibu-cultcache-target-lowering.md +++ b/prompts/nibu-cultcache-target-lowering.md @@ -24,20 +24,10 @@ text. Lower from the interlingua packet and projected context. ## Cultural Ontology Authority -Use the CultCache store: - -- `.weksa/cultural-ontology/starter-targets.cc` - -It contains: - -- `weksa.cultural_ontology.pt-BR.vanilla_brazilian.v0` -- `weksa.cultural_ontology.en.transatlantic.v0` -- `weksa.cultural_ontology.ja-JP.kanto.contemporary.v0` - -Qdrant-style retrieval and old YAML profiles may be useful context, but the -CultCache profile named in your task is the target authority. Treat the target -profile as draft guarded typed state: preserve its activation gates, forbidden -flattenings, and nested linguistic affordance feature IDs in your trace. +The language ontology is `data/ontology/ontology.cc` (`weksa ontology dump`). +It holds model-drafted seed entries for en-US and ja-JP only; there is no +pt-BR entry yet. Treat an entry as a draft: say in your trace which entries you +used, and when no entry covers the target, say so rather than inventing one. ## Output Contract @@ -50,8 +40,6 @@ The file must use this shape: schema_version: weksa.utterance_lowering.agent_run.v0 source_packet_id: nibu-proof-001 source_packet_ref: examples/english-lowering/nibu-proof-001/interlingua.yaml -target_profile_ref: .weksa/cultural-ontology/starter-targets.cc -target_profile_id: target_agent_id: nibu realization: realization_id: @@ -60,10 +48,6 @@ realization: locale: script: register: - cultural_ontology_refs: - - .weksa/cultural-ontology/starter-targets.cc - cultural_ontology_profile_ids: - - outputs: visible_action: spoken_text: diff --git a/schemas/README.md b/schemas/README.md index 6de8179..7e0350a 100644 --- a/schemas/README.md +++ b/schemas/README.md @@ -23,9 +23,6 @@ them. - [utterance-lowering-packet.md](./utterance-lowering-packet.md): shared packet contract for lowering interlingua and projected speaker context into one or more target-language utterance realizations. -- [target-language-cultural-ontology.md](./target-language-cultural-ontology.md): - contract for the address, politeness, register, idiom, metaphor, script, and - forbidden-flattening profiles consumed by target lowerers. - [cultural-ontology-profile.md](./cultural-ontology-profile.md): nested CultCache profile contract for cultural ontology, including typed salience axes, linguistic affordance feature records, evidence, activation gates, and diff --git a/schemas/cultural-ontology-profile.md b/schemas/cultural-ontology-profile.md index 84b1ed3..f805119 100644 --- a/schemas/cultural-ontology-profile.md +++ b/schemas/cultural-ontology-profile.md @@ -5,8 +5,8 @@ This is the human-facing schema receipt for reviewed cultural ontology used during utterance lowering. Executable schema code does not exist yet. When it does, it must reconcile with -this receipt, the meta-analysis paper, and the profiles under -`data/target-language-ontology/`. +this receipt, the meta-analysis paper, and the language ontology in +`data/ontology/ontology.cc`. ## Authority diff --git a/schemas/target-language-cultural-ontology.md b/schemas/target-language-cultural-ontology.md deleted file mode 100644 index ef34301..0000000 --- a/schemas/target-language-cultural-ontology.md +++ /dev/null @@ -1,81 +0,0 @@ -# Target-Language Cultural Ontology Contract - -This is the human-facing schema receipt for target-language cultural ontology -profiles consumed by Weksa utterance lowering. - -Executable schema code does not exist yet. When it does, it must reconcile with -this document and the profiles under `data/target-language-ontology/`. - -## Authority - -A target-language cultural ontology profile owns the realization distinctions -that must be inspected for a human target language before surface text is -chosen. - -It owns: - -- address and treatment distinctions -- politeness and register pressure -- social stance and directness defaults -- idiom, metaphor, and lexical-locale risks -- script or orthographic choices that carry meaning -- forbidden flattenings for reviewed output - -It does not own: - -- source meaning -- speaker identity -- private state -- exhaustive grammar -- whole-culture description -- final wording without an interlingua packet - -## Required Fields - -- `profile_id`: stable profile identifier. -- `language`: target code, locale, and script. -- `scope`: bounded statement of what the profile is and is not. -- `authority`: `owns` and `does_not_own` lists. -- `salience_axes`: named axes the lowerer must inspect. -- `linguistic_affordances`: optional nested typed feature records, following - `cultural-ontology-profile.md`, when the profile owns dialect, register, - phonological, lexical, discourse, orthographic, or code-switching pressure. -- `forbidden_flattenings`: mistakes the target lowerer must avoid. - -## Salience Axes - -Each salience axis should name: - -- `relevant_distinctions`: choices the target language or locale makes visible. -- `lowering_questions`: questions reviewers and lowerers should answer before - accepting an output. - -Examples: - -- English directness and sarcasm. -- Brazilian Portuguese treatment choice, lexical locale, and regional or - religious-cultural overlays. -- Japanese politeness, ellipsis, and sentence-final force. - -## Fixture Contract - -Reviewed multilingual fixtures should cite the profile used by each -realization: - -```yaml -target_language: - code: pt - locale: pt-BR - script: Latn - register: direct technical casual - cultural_ontology_refs: - - data/target-language-ontology/pt-BR.yaml -``` - -The realization trace should name the cultural ontology axes that shaped the -line. - -For dialectal, regional, historical, ritual, and contact-language profiles, use -the nested `weksa.cultural_ontology_profile.v0` contract in -`cultural-ontology-profile.md`. Flat string lists are only summaries; typed -feature records own accepted affordance detail. From fc99652f1307b7a049b94efab997122b4c498e15 Mon Sep 17 00:00:00 2001 From: Weksa Date: Thu, 12 Nov 2026 15:29:11 +0000 Subject: [PATCH 10/57] language-ontology-seed fix s1: rebuild the seed as 9+9 entries that read only stance and discourse and make true claims; pin the generator exactly; drop deleted store paths from the tex and schema; README says nothing grows the seed yet ontology.cc is rebuilt from nothing (the branch is unmerged); no entry carries over unchanged in number or text. Co-Authored-By: Claude Sonnet 5.5 --- README.md | 3 ++- crates/weksa/tests/original_corpus.rs | 6 +++++- data/ontology/ontology.cc | Bin 9181 -> 6557 bytes docs/cultural-ontology-meta-analysis.tex | 13 +++++-------- schemas/cultural-ontology-profile.md | 13 +++++-------- 5 files changed, 17 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index ef509cb..b9f81e4 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,8 @@ target-specific English receipt. The language ontology is `data/ontology/ontology.cc` (read it with `weksa ontology dump`): per language, the address, register, politeness and script choices a lowerer makes for a speaker in a stance. Its en-US and ja-JP entries are -model-drafted seeds, never Test evidence; accepted rounds grow them. +model-drafted seeds, never Test evidence. Nothing yet grows them from accepted +rounds; until that mint exists they change only by a new `weksa ontology admit`. For the first AquaSynth speech-training handoff, see [Tiny PanPhon v0.1 Speech Curriculum](./examples/speech-training/tiny-panphon-v0.1/README.md). diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 4e1dce1..2a48196 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -270,7 +270,11 @@ fn committed_ontology_is_model_drafted_and_reads_every_inventory() { match &e.annotator { Annotator::Model { family, generator } => { assert_eq!(*family, ModelFamily::Anthropic, "{}", e.key()); - assert!(generator.contains("claude"), "{}", e.key()); + assert_eq!( + generator, "claude-sonnet-5-5 (Weksa Hands, model-drafted seed ontology)", + "{}", + e.key() + ); } other => panic!("{} is {other:?}", e.key()), } diff --git a/data/ontology/ontology.cc b/data/ontology/ontology.cc index 94eeb19bb785bae535198a500c282f330f069d9b..b43ca6dad1290efd5cf090297f772219233c90f9 100644 GIT binary patch delta 1495 zcmYjROKenC7|t#9J;l;eX)8ldF^!#MW(chjGIi4gm0B@s)%Zxn(|ga%J#y!q@tixY z-AK;_AF-mezXpg9NJwjJpe#xg7Cz#_l?yg*jPVf;)2KUE?$mQ1GsEuQ`~UZUe2@Rv zr!I}3y=(O>SI64Nj^xG;zxIRG3CQYedUqO1)_|3RGuDyGG@5s@Pv(d~zDv+N7Y=0w zBzfK^Od>9jbg?Lz=j`&${F4ky}E@WtL%p;NWxo;<($7S25VoMnA=luDbIg(PQxzC1&i!Q3X9JZ0EJc`k~`#zIgcgBP11 zY~aVfPyMn(&}hj3683S3EFUalSw@5f?8!7LVc}ETGk-FYMhiTRPGT=Lfozv^Neqg} z=ctH%+YR~jC?k)=UDAkgA3ebe9+fU~s35k_h?LotEv+tT^}bf?x&m5V(dxQhtlVmR zRU2A;snrFomR9xm(J1}$~G?+17M;wG@4;JjWU9#5S0keVd~*ok7QS_RhK^3 z>Qne_aILPu{+NlKhVRD*Uw^@HCk&{54wachgi)DKiiQ@roj{L8o(PC_1TTfSMNHMe z&Mt3lSJq7rt!@M$snZ6b0EplB9jb$H7zUEZtRl=NQ*SU#gK`U;BL`1}6{9T-7bWgd zBL@S#A>Z@(ybR;QjEH!?&RTZ)+rQL%y7IBE7*v;ZrJ~ij;8m@|n*%<)vcI9&o_O2) z?U6`5e~cDfWE0^Bn=j-H%n|NbO7oG{_4)++8y}uK_?XQnFEap+v5rA_n~WQrYuT=N@6f(aBJ_#!C14h lYC!1P8+TL{1?BO8~B~a z|NVuBPhFxXsfn<%XR0>%PrJN!&DxEX@$t&|+TG(DYUAr`>(>43pCjz_Idj#>$c4?m za%2)KalChY!gXVz6w{tyfv>a(Ma*137LE?uY}ON@IiwsMU?HB>oZ8yNnnFa`p-2eF zW8Bfq4>f*`F(nyivpf#{u)&n(ZB=;BLl^I2@q8kb?jN=9t8-VkiuM?5%cRP-nZJi= z$y%b?Ik#Q9BIwT7dCL#l7bZgO#WM1p3k~|4IV#@!ffMk=6%{4JQ0U5r$_Dnmy;Og; z72m4HSc`5^absTB!evU}N6w7Zr`|3?yJlbsmcjO07TYh6ENl}8eMg3KhtH}&>iH?n zLq{ys1=mR|F52I<@r`FSp1!j%p|=UH^uO-e*V{KyuLE`Pq(<1afM#9BEgk@?1|bo% zSzmh$ch@ms#)){Mx*r3@tT2Quz|etUek&qO{28IDbJq-{nm6&eX81k;EStRj#E>y9 z#n*3GOdl%nbtP0EU5e_SDG#KrWjIv;3b--G0y$f0iZ&y}JmEHkJC=dVnet?;8IO!y zFkt8bcHt+7#8Pa5fWSq@%&c{KO^jA-THv=Z>QPUE zfYGv5K=u8-+=qP30@NqiPh|ojat*}8X6m{!!0xudezm~V6DzF$HA|q)E?Cs&Tr)1@ znqkR3KcTlvD8|U~-u|Zqx1s<-Ge^XlqZI8Ln~r4@%AIjJLl|g>p=i+;d%J_602h!J9*FV_M4985=8%EKUPV%|S+Y=S&@oYLM#R%rn8}}@^C_|o=%^V>U1F_ID&(pffeAFVjJS+A0*7 zDg&J{ML~sPYL`?)e1_sE3D*UymaQ^;U^*}S$;It*MpW63p~JtS4EzsU`0wk(ztT#u za28G8{%01^OF|c55XMD9krN#sn9wts-Mvb2bkwQTjAo&v|&i5GB?-q8y@_` zB!Ke6CaDW&)4twhfb;mTc${HLI3^@fPTLG5R$UD3;TKXblV5{RS5M^+_wY zb?ygMMr@`2XwA!z@qDZdo|`OLA1H;V6@fP%w^a^vK69{y`p#gk=SDxNgE*Sd$>2v0)PX|LU?CL+aS)QImgGU>ur=g`UY2{N? zfdr{X57Qsfsbelw6#X9A05R4LV~*}&$n5UiJ4RqMHs?W^~B%O z&e2a^Kb>}7NIS2lo$G1mwe(Q;TH5(z+BumX>fCtwy|iz}@%JSw>`0E|n3D zSeWlTBf4FQkrDmSrWplzGdD%?gpDi=qahm-Arei`i@9EP?t`1B{t6^-oyX5^+C6F@ zJ(G5U<*>mk6$bs-f>#WK2Aki-h;4j{2&^!Z53p;`<#vtILhh0Z)EK$n0F!bZIa1%E z<@UEy_d_T(@(mhNU`Dc0TVgUXwaBd;wMfj@hRH9codsfc=WS5>KZoY=Z^%q8HDG_! z7UTa4l6Ef!NyYRi%jrlz5nu4d+z67AM@kZ^Ekd`b**F?XmxJGF=SbRl6EJ@Khf6m; zI++7|Y$*KBCc8`=N+J4?MYI@M%eFN2>pNQN08a9fdwK1lxT*1~++{hgInBIYY;=TB+p_oTAJ zP>CXnWupp%{bE!xSR$66k3YOKg(%F&`#*C&&c-=MB5cLP2AEL;&&LEpQE0|tjK(%` z#u?;6B#fVjO2vNNchCYanR*Um$Ts*F(QNXTgq4f2 zD~fhVsJEqr{Zk3%4Ihw_@gv{5Zo*DToO0*%|99%k+xx6j;|OOWqV{!D;kc;N&vkWO+X>5nj3c?js7%*JV)G%&t< zqz5RI$gyuR Date: Thu, 12 Nov 2026 15:39:48 +0000 Subject: [PATCH 11/57] language-ontology-seed fix s2: neutral ja-JP:1 (no politeness per line), entry text defects, drafter residue out of the seed and score vectors, front-door and schema/tex prose ontology.cc regenerated by weksa ontology admit, 9+9 entries, same generator string. Co-Authored-By: Claude Sonnet 5.5 --- README.md | 7 ++++--- crates/weksa/tests/score.rs | 16 ++++++++-------- data/ontology/ontology.cc | Bin 6557 -> 6574 bytes docs/cultural-ontology-meta-analysis.tex | 12 ++++++------ schemas/cultural-ontology-profile.md | 2 +- 5 files changed, 19 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index b9f81e4..f84b165 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ measured. - [Lowering target](./docs/targets/interlingua-lowering.md) and [its map](./docs/targets/interlingua-lowering-map.md): the design, scoring and cut sequence. -- `data/`: the cultural ontology and target-language profile data. +- `data/`: the committed stores (corpora, entities, inventory, language ontology) and research data; there are no target-language profiles. - `settings/`: setting-specific proving grounds for language projects. **Live versus planned.** Live: docs, fixtures, examples, ontology data and the @@ -83,8 +83,9 @@ English, Brazilian Portuguese, Japanese, and later target-language dialogue. target-specific English receipt. The language ontology is `data/ontology/ontology.cc` (read it with -`weksa ontology dump`): per language, the address, register, politeness and script -choices a lowerer makes for a speaker in a stance. Its en-US and ja-JP entries are +`weksa ontology dump`): per language, the lexicon, grammar and discourse choices a lowerer +reads for a stance and discourse packet. The seed sets no address or politeness level; those +belong to the speaker and scene state. Its en-US and ja-JP entries are model-drafted seeds, never Test evidence. Nothing yet grows them from accepted rounds; until that mint exists they change only by a new `weksa ontology admit`. diff --git a/crates/weksa/tests/score.rs b/crates/weksa/tests/score.rs index cd23eb9..8d2efce 100644 --- a/crates/weksa/tests/score.rs +++ b/crates/weksa/tests/score.rs @@ -45,8 +45,8 @@ fn chrf_matches_sacrebleu_vectors() { #[test] fn chrf_argument_order() { - let short = "mind the hatch"; - let long = "mind the hatch before the storm arrives tonight"; + let short = "shut the gate"; + let long = "shut the gate before the storm arrives tonight"; let (as_hypothesis, as_reference) = (chrf(short, long), chrf(long, short)); assert!( as_hypothesis < as_reference - 1.0, @@ -104,8 +104,8 @@ fn chrf_whitespace_parity() { #[test] fn scorer_argument_order() { - let short = "mind the hatch"; - let long = "mind the hatch before the storm arrives tonight"; + let short = "shut the gate"; + let long = "shut the gate before the storm arrives tonight"; let scorer = Scorer::new(Bag); let got = score_one(&scorer, long, short); assert_eq!(got.chrf, chrf(short, long)); @@ -156,7 +156,7 @@ impl Embedder for Fixed { fn scorer_refuses_bad_vectors() { let run = |vectors: Vec>| { Scorer::new(Fixed(vectors)).score(&[ScoreInput { - reference: "mind the hatch", + reference: "shut the gate", output: "mind the door", }]) }; @@ -671,16 +671,16 @@ fn every_script_scores_a_near_miss_as_sacrebleu_does() { #[test] fn coverage_splits_at_any_non_word_character() { let items = [ - validation("item:v:1", "mind the hatch before the storm"), + validation("item:v:1", "shut the gate before the storm"), train("item:t:1", UNRELATED), ]; - let want = read(&["mind the hatch"], &items); + let want = read(&["shut the gate"], &items); assert_eq!(want, [(5, 3), (5, 2), (4, 1)]); for joiner in [ ":", "\u{b7}", "'", "\u{2019}", "\u{ff3f}", "-", "/", "\u{2014}", "|", "+", "\u{ff1a}", "\u{30fb}", ] { - let glued = format!("mind{joiner}the{joiner}hatch"); + let glued = format!("shut{joiner}the{joiner}gate"); assert_eq!(read(&[&glued], &items), want, "{joiner:?}"); } // a mark, a joiner or a digit is part of its word, not a break diff --git a/data/ontology/ontology.cc b/data/ontology/ontology.cc index b43ca6dad1290efd5cf090297f772219233c90f9..4c610995f6dc83367e12a332732f550eab1a9f8a 100644 GIT binary patch delta 749 zcmZWnO=uHA7-f_G5T#8TjH%$^6A@dRtwJuf6-o}0ViAk6ARY?i?w4fDW+u)|Bk^M6 z!Gj1E2c?3+9t@=iwVKjXz3I7l63+`>dMMr%oF>`k;AMF`^SyoFdv6=*59zI>6-i&7 zPG7q7%^K2Ry-e;bct|9I<8kv;MFf{~F{w$ks8_-RdMb48bcgYEf31nkUd)B8jjSN$ zG7+mkm-NHXdA%K)O72`If)en!Q$qK@4^14&F6!;Dw3NOdJFDmWq8T8}g^EWg1E~aO z1*oF9D`$YXE-((H!aZK70T)bUV)LxiXSQHrW%7pH+CNW_u^QV@t`LrWPOK%_*WWC}{e z;F6q#TSY3tTpPATB@c5jF;Js|Qi6w7^k&)_8?9_x3`&R<3DgYdxp2^^0@Ajd_W0zB zkAA)3*PrOVHullI7wtTey8imZ6EVq0r7f`lOvOI!2$zCD@; zV%Q5l2P(nP1(kLE)-IpeI)1~nY>=RV?WX`qow8b|nkrbJK>)LnU~rE|X4;D#QPcWq zkF;EE^_EUMFc%QYKq|ovw}E0-I>0zcURB6Yn2~0>x$x1eq139k+1HW$@lfAYD}~{% z9r%6srLO%cJ(SxhjL!&T+cAxJ+j>W+{+Vl~pME7pT7kn_u{cS!N#HOp#E zQ|aoeGJJ0GVfk zGCt>sP}B_!lc8gBa-dFv@hNb9&ZP;}?21E3l6CUz3eK@wK&1~J73eC4 zQl5PNIf+-2_)!wSPZo6&H@z!J~(b1(%zDVb-y20LF@pI8YQ^{~r97ws^PyN|xro17QYf;X($#3|-IQ T-r;8{)Nc;9dW=f_*Czh~lL+~Z diff --git a/docs/cultural-ontology-meta-analysis.tex b/docs/cultural-ontology-meta-analysis.tex index ef65805..7dcd72f 100644 --- a/docs/cultural-ontology-meta-analysis.tex +++ b/docs/cultural-ontology-meta-analysis.tex @@ -369,6 +369,12 @@ target. A Qdrant nearest neighbor is a witness candidate, not a cultural fact. \section{Proposed CultCache State Shape} +No profile store exists. The per-language lowering choices live in +\texttt{data/ontology/ontology.cc}; the typed profiles once stored under +\texttt{.weksa/cultural-ontology/} and the \texttt{weksa.target\_cultural\_ontology.*} +base profiles named in the listing and sketches below are deleted. Community and dialect +profiles belong to follow-up \texttt{language-ontology-communities}. + The following shape should become a typed CultCache document family: \texttt{weksa.cultural\_ontology\_profile.v0}. It can represent base languages, dialects, sociolects, regional overlays, ritual registers, and communities of @@ -612,12 +618,6 @@ feature requires speaker authority, whether it can be rendered in text or only speech synthesis, which source hashes support it, and what must be reported if the feature is rejected. -No profile store exists. The per-language lowering choices live in -\texttt{data/ontology/ontology.cc}; the typed profiles once stored under -\texttt{.weksa/cultural-ontology/} and the \texttt{weksa.target\_cultural\_ontology.*} -base profiles named in the sketches below are deleted. Community and dialect -profiles belong to follow-up \texttt{language-ontology-communities}. - \section{Dialect Profile Sketches} \subsection{AAVE/AAL} diff --git a/schemas/cultural-ontology-profile.md b/schemas/cultural-ontology-profile.md index 07fdb53..dc26099 100644 --- a/schemas/cultural-ontology-profile.md +++ b/schemas/cultural-ontology-profile.md @@ -268,7 +268,7 @@ cultural_ontology_trace: No profile store exists. The per-language lowering choices live in `data/ontology/ontology.cc` (`weksa ontology dump`); the typed profiles that once sat under `.weksa/cultural-ontology/`, and the `weksa.target_cultural_ontology.*` -base profiles named in the examples below, are deleted. Community and dialect +base profiles, whose names an example below still carries, are deleted. Community and dialect profiles are owned by follow-up `language-ontology-communities`. Qdrant may store chunks, vectors, and payload filters for retrieval. It does From dc5e649851d2ead5c7688eed6d5971c70aebcdc8 Mon Sep 17 00:00:00 2001 From: Weksa Date: Thu, 12 Nov 2026 15:44:08 +0000 Subject: [PATCH 12/57] language-ontology-seed fix s3: ja-JP:3 and :6 name no politeness level; README claim made true; schema clause dropped Co-Authored-By: Claude Sonnet 5.5 --- README.md | 2 +- data/ontology/ontology.cc | Bin 6574 -> 6647 bytes schemas/cultural-ontology-profile.md | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f84b165..61456bf 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ target-specific English receipt. The language ontology is `data/ontology/ontology.cc` (read it with `weksa ontology dump`): per language, the lexicon, grammar and discourse choices a lowerer -reads for a stance and discourse packet. The seed sets no address or politeness level; those +reads for a stance and discourse packet. No entry fixes an address or politeness level; those belong to the speaker and scene state. Its en-US and ja-JP entries are model-drafted seeds, never Test evidence. Nothing yet grows them from accepted rounds; until that mint exists they change only by a new `weksa ontology admit`. diff --git a/data/ontology/ontology.cc b/data/ontology/ontology.cc index 4c610995f6dc83367e12a332732f550eab1a9f8a..b6cc327d1410de006ebfaf698fad80a95ac884d6 100644 GIT binary patch delta 413 zcmZ2y{M~qiI3ufxv6ZReW*J6j7&C=Q0mkfP?uIcXS^Z(mTsAoKAiFnARFJa>#_Zs7 zgfY)?2g8_ZyrnSa0zL~yPO!yMM;SLwe#l?UcxW;si?VrgPG&(tYKlT}YFctBA<#`H4sp*-;C8<;4n_C6hl0tcUu0iJ%|MQQw7f5sosQ zoGaoXAMKHuo}rMOT2zvlnO9P21$0bWPGX5dN@{XWVo~Dc%OdIq8hNQY3YGbq3P5)# zP_15xN-91rk=-*>z^*}0yFkKZk+kJp=mOQ=t`*N+eCw4mh+1D!#_Zs7 zgfY)?2g8_ZyrnSa0zL~yPO!yMM;Uu2KjbfFoHLn`McKL_CowZmAvG^0GcUc^S|L#( zDX}P3p`a)~BQq(pBr`uxK|?n$QBxtmNCBcWGq)hMD6u57EOqjF0o%!9g6p9kIwj}_ zvrSewmr(#_o6wA)Rgp8y@`_pMOQ*C_%9j+vmi{oAI5wo!OI8$n0IAo diff --git a/schemas/cultural-ontology-profile.md b/schemas/cultural-ontology-profile.md index dc26099..f027f8e 100644 --- a/schemas/cultural-ontology-profile.md +++ b/schemas/cultural-ontology-profile.md @@ -268,7 +268,7 @@ cultural_ontology_trace: No profile store exists. The per-language lowering choices live in `data/ontology/ontology.cc` (`weksa ontology dump`); the typed profiles that once sat under `.weksa/cultural-ontology/`, and the `weksa.target_cultural_ontology.*` -base profiles, whose names an example below still carries, are deleted. Community and dialect +base profiles are deleted. Community and dialect profiles are owned by follow-up `language-ontology-communities`. Qdrant may store chunks, vectors, and payload filters for retrieval. It does From e8ba0f2eae27bc237c10fd325b728002817d298e Mon Sep 17 00:00:00 2001 From: Weksa Date: Thu, 12 Nov 2026 16:09:36 +0000 Subject: [PATCH 13/57] language-ontology-seed fix s4: ja-JP:6 bans ne-softening and hedges only; front door and lowering docs stop naming a target-language profile as register owner Co-Authored-By: Claude Sonnet 5.5 --- README.md | 2 +- data/ontology/ontology.cc | Bin 6647 -> 6632 bytes docs/multilingual-utterance-lowering.md | 2 +- docs/targets/interlingua-lowering.md | 4 ++-- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 61456bf..bfa07f2 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ culture, and constraint, and the planned lowerer that turns an agent's meaning packet into the reader's display language. **What is valuable here.** The interlingua standard and the lowering design: a -meaning packet owns what is meant, a target-language profile owns register, +meaning packet owns what is meant, speaker and scene state (planned; not yet live) own register, address and politeness, and a lowerer owns surface text and leaves a trace. **The problem it solves.** An agent's outward speech today is one English diff --git a/data/ontology/ontology.cc b/data/ontology/ontology.cc index b6cc327d1410de006ebfaf698fad80a95ac884d6..2f9cdf7af6d247d95ebed9420e16bc70bce0d5cb 100644 GIT binary patch delta 269 zcmexv{K9yHC?lJhm4T&|@n&g8CpbHqNgmGbVD5smC0PC7>>M@(`vAKaTwH*&5YBGr za)7hXatFcLs=Os|_Iy5bIQtI&b2$5?pf8*)Bb>u11aYTf)KNyY8B4Pgb-e;6pJ8`o z^qG7|#7H6|H6=Y&Au%sSAunH{I6tispw&t5C4e1Jt0CpbHqNgmGbVD5smC0PC7>>M@(`vAKaTwH*&5YBGr za)7hXatFcLs=Os|_Iy5bIQtI&b2$5?pf8*)Bb>u11aYTf)KNy$8B4Pgb-e;6pJ8`o zjGlZ)#7Mn3KdmG+FEcM)p&+rSBr`cDRY4;!RY##RUsEA5FGV3QUm+tkC4I7*=wXgR^f*@G$}ai~mgS diff --git a/docs/multilingual-utterance-lowering.md b/docs/multilingual-utterance-lowering.md index 6d01bbb..cc3daae 100644 --- a/docs/multilingual-utterance-lowering.md +++ b/docs/multilingual-utterance-lowering.md @@ -58,7 +58,7 @@ The utterance lowerer does not own: - rendered audio - phoneme parity or synth-control truth -The target-language cultural ontology profile owns: +The target-language cultural ontology profile was drafted to own the following; no such profile store exists now (address, politeness and register belong to speaker and scene state, and `data/ontology/ontology.cc` holds per-language lexicon, grammar and discourse choices): - address, politeness, register, and social stance distinctions - idiom and metaphor risks diff --git a/docs/targets/interlingua-lowering.md b/docs/targets/interlingua-lowering.md index 54bf512..a39963d 100644 --- a/docs/targets/interlingua-lowering.md +++ b/docs/targets/interlingua-lowering.md @@ -25,8 +25,8 @@ ground truth is Discord text, which can never feed this loop An agent's outward speech today is English text the model chose, so meaning, register and voice are fused in one string nobody can inspect or re-target. -Weksa's design already separates them: the packet owns meaning, the target -profile owns address, politeness and forbidden flattenings, the lowerer owns +Weksa's design already separates them: the packet owns meaning, speaker and scene state own address and politeness +(the language ontology holds only per-language lexicon, grammar and discourse choices), the lowerer owns surface text and a trace. The campaign makes that design real for a live reader, and gives Weksa's Persona a working body when her mind (`eureka-body:follow_up:weksa-mind`) wakes. From f7f4e58d78089b28d762a9a2fc0895355e4ef7ef Mon Sep 17 00:00:00 2001 From: Weksa Date: Thu, 12 Nov 2026 16:27:13 +0000 Subject: [PATCH 14/57] language-ontology-scene s1-2: the run pins the speaker-state store, assembled is the one scene door, select reads slot Scene, Prompt::lower refuses an unread scene, weksa scene put|dump, speaker_scene_revision(n) Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/examples/show_working.rs | 1 + crates/weksa/src/context.rs | 27 +-- crates/weksa/src/inventory.rs | 8 +- crates/weksa/src/lower.rs | 3 + crates/weksa/src/main.rs | 184 +++++++++++++++++- crates/weksa/src/ontology.rs | 12 +- crates/weksa/src/run.rs | 88 ++++++++- crates/weksa/src/store.rs | 7 +- crates/weksa/tests/cli.rs | 3 + crates/weksa/tests/common/mod.rs | 15 ++ crates/weksa/tests/context.rs | 34 +++- crates/weksa/tests/golf_loop.rs | 25 ++- crates/weksa/tests/original_corpus.rs | 4 + crates/weksa/tests/scene.rs | 4 +- crates/weksa/tests/speaker_state.rs | 4 +- schemas/cultnet/weksa.loop_run.schema.json | 12 +- .../cultnet/weksa.ontology_entry.schema.json | 2 +- 17 files changed, 388 insertions(+), 45 deletions(-) diff --git a/crates/weksa/examples/show_working.rs b/crates/weksa/examples/show_working.rs index 8c23fb2..6c39bdb 100644 --- a/crates/weksa/examples/show_working.rs +++ b/crates/weksa/examples/show_working.rs @@ -259,6 +259,7 @@ fn lower_error(e: &LowerError) -> Value { LowerError::ConsentMissing { predication } => ("ConsentMissing", format!("{predication} is a request the packet does not carry the addressee's consent to")), LowerError::AskerNotAddressee { predication } => ("AskerNotAddressee", format!("the agent of {predication} is not the addressee")), LowerError::EntityRead { entity, .. } => ("EntityRead", format!("the entity store failed reading {entity}")), + LowerError::SceneUnread => ("SceneUnread", "the view's scene was never read".to_string()), }; json!({ "variant": variant, "reason": reason }) } diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index 2b88a08..3dec960 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -37,7 +37,7 @@ use crate::packet::{ use crate::run::LADDER_STEP; use crate::split::{Checkpoint, Split, SplitScheme, fnv1a64, hex16}; use crate::store::{CorpusStore, normalized}; -use crate::validate::LoweringInput; +use crate::validate::{LoweringInput, SceneRead}; /// The instructions a mint prompt starts with: the editor's whole brief. Fixed /// per run by `PROMPT_SOURCE_DIGEST`. @@ -71,7 +71,7 @@ A feature is {feature, value}; value is {\"Grade\": 0 to 1000}, {\"Option\": lab /// Fixed per run by `PROMPT_SOURCE_DIGEST`. pub const LOWER_INSTRUCTIONS: &str = "\ Say the packet below in the language on the language line, as one utterance of its speaker: all of its meaning and nothing it does not carry. -Every id is the inventory's listed below and every feature value is as listed; a grade is permille, 0 to 1000. The stance is the speaker's felt affect and restraint: it shapes how the utterance is said, not what it says. The ontology lines are choices this language makes for this speaker and stance: realize each where the packet allows; they add no meaning. +Every id is the inventory's listed below and every feature value is as listed; a grade is permille, 0 to 1000. The stance is the speaker's felt affect and restraint: it shapes how the utterance is said, not what it says. The ontology lines are choices this language makes for this speaker, stance and scene: realize each where the packet allows; they add no meaning. A referent with an entity is named by one of that entity's forms on the entities line. Answer with one JSON object and nothing else: {\"text\": \"...\"}. No code fence, no commentary."; @@ -89,7 +89,7 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat /// ```no_run /// # let dir = tempfile::tempdir().unwrap(); /// # let p = |n: &str| dir.path().join(n); -/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc"), ontology: &p("o.cc") }; +/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc"), ontology: &p("o.cc"), speaker_state: &p("s.cc") }; /// # let panel = ["en-US".to_string()]; /// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &panel).unwrap(); /// let t = run.train().unwrap(); @@ -104,7 +104,7 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat /// # let dir = tempfile::tempdir().unwrap(); /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc", "o.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } -/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc"), ontology: &p("o.cc") }; +/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc"), ontology: &p("o.cc"), speaker_state: &p("s.cc") }; /// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &["en-US".to_string()]).unwrap(); /// let t = run.train().unwrap(); /// let _u = t.clone(); @@ -115,7 +115,7 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat /// # let dir = tempfile::tempdir().unwrap(); /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc", "o.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } -/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc"), ontology: &p("o.cc") }; +/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc"), ontology: &p("o.cc"), speaker_state: &p("s.cc") }; /// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &["en-US".to_string()]).unwrap(); /// let t = run.train().unwrap(); /// s(&t); @@ -337,7 +337,7 @@ impl EditFeedback { /// ```no_run /// # let dir = tempfile::tempdir().unwrap(); /// # let p = |n: &str| dir.path().join(n); -/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc"), ontology: &p("o.cc") }; +/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc"), ontology: &p("o.cc"), speaker_state: &p("s.cc") }; /// # let panel = ["en-US".to_string()]; /// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &panel).unwrap(); /// let t = run.train().unwrap(); @@ -356,7 +356,7 @@ impl EditFeedback { /// # let dir = tempfile::tempdir().unwrap(); /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc", "o.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } -/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc"), ontology: &p("o.cc") }; +/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc"), ontology: &p("o.cc"), speaker_state: &p("s.cc") }; /// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &["en-US".to_string()]).unwrap(); /// let t = run.train().unwrap(); /// let inv = run.inventory().inventory().unwrap(); @@ -369,7 +369,7 @@ impl EditFeedback { /// # let dir = tempfile::tempdir().unwrap(); /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc", "o.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } -/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc"), ontology: &p("o.cc") }; +/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc"), ontology: &p("o.cc"), speaker_state: &p("s.cc") }; /// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &["en-US".to_string()]).unwrap(); /// let t = run.train().unwrap(); /// let inv = run.inventory().inventory().unwrap(); @@ -528,9 +528,11 @@ impl Prompt { /// `input` (no kind, constraints, scene, speaker or group), the forms in /// `language` of every entity its referents cite, the `ontology:` section /// and the entries of `inventory` live at the view's revision. The section - /// holds the entries `ontology::select` chooses from the view's stance and - /// discourse features and `language`, never the caller; the view's speaker - /// and group are still never read. `item` + /// holds the entries `ontology::select` chooses from the view's stance, + /// discourse and the scene relation `with_scene` read, and `language`, + /// never the caller; the scene's values are never printed, only the entries + /// they select. A view whose scene was never read is refused + /// ([`LowerError::SceneUnread`]). `item` /// goes only to [`Prompt::item_keys`], never into the text. A speaker's /// entity id appears /// only when the encoder cites that entity on a referent, which is naming a @@ -545,6 +547,9 @@ impl Prompt { entities: &dyn EntityForms, ontology: &Ontology, ) -> Result { + if input.scene_read() == &SceneRead::Unread { + return Err(LowerError::SceneUnread); + } let mut forms: BTreeMap<&str, Vec> = BTreeMap::new(); for id in input.referents().iter().filter_map(|r| r.entity.as_ref()) { let found = entity(entities, id)?; diff --git a/crates/weksa/src/inventory.rs b/crates/weksa/src/inventory.rs index ec59b13..3ffd5ad 100644 --- a/crates/weksa/src/inventory.rs +++ b/crates/weksa/src/inventory.rs @@ -551,7 +551,7 @@ pub fn proof_revision() -> (InventoryRevision, Vec) { /// arrive as community ids with cut `language-ontology`. Hand-seeded like /// [`proof_revision`] until the loop's minting round owns the vocabulary; the /// features take the ids after the seed's. -pub fn speaker_scene_revision() -> (InventoryRevision, Vec) { +pub fn speaker_scene_revision(n: u32) -> (InventoryRevision, Vec) { let taken = seed() .1 .iter() @@ -593,15 +593,15 @@ pub fn speaker_scene_revision() -> (InventoryRevision, Vec) { gloss: gloss.to_string(), value_kind: Some(kind), applies_to: vec![slot], - introduced_rev: 3, + introduced_rev: n, retired_rev: None, }); } let changes = entries.iter().map(|e| Change::Add(e.id)).collect(); ( InventoryRevision { - n: 3, - parent: Some(2), + n, + parent: Some(n - 1), changes, frozen: false, }, diff --git a/crates/weksa/src/lower.rs b/crates/weksa/src/lower.rs index 76f3158..28f4ca2 100644 --- a/crates/weksa/src/lower.rs +++ b/crates/weksa/src/lower.rs @@ -182,6 +182,8 @@ pub enum LowerError { AskerNotAddressee { predication: NodeId }, /// The entity port failed to read. EntityRead { entity: EntityId, why: String }, + /// A lowering prompt was asked of a view whose scene was never read. + SceneUnread, } impl fmt::Display for LowerError { @@ -203,6 +205,7 @@ impl fmt::Display for LowerError { LowerError::ConsentMissing { .. } => "request lacks the addressee's consent", LowerError::AskerNotAddressee { .. } => "request agent is not the addressee", LowerError::EntityRead { .. } => "entity port failed to read", + LowerError::SceneUnread => "the view's scene was never read", }) } } diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index b9b094e..efae4ff 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -10,7 +10,8 @@ use unicode_normalization::UnicodeNormalization; use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn}; use weksa::encode::OntologyDraft; use weksa::entity::{Entity, EntityOrigin, LanguageForms, Pronunciation}; -use weksa::inventory::{proof_revision, seed}; +use weksa::inventory::{proof_revision, seed, speaker_scene_revision}; +use weksa::packet::{FeatValue, FeatureValue, Grade}; use weksa::packet::proof_packet; use weksa::model::RecordedAnswers; use weksa::run::{ @@ -18,8 +19,8 @@ use weksa::run::{ }; use weksa::score::ScoreInput; use weksa::{ - Annotator, AnswerFiles, CorpusStore, EntityId, EntityStore, InventoryStore, OllamaEmbedder, OntologyStore, PacketStore, - Scorer, Split, SplitScheme, StoreError, lower, pronounce, validate, + Annotator, AnswerFiles, CorpusStore, EntityId, EntityStore, Family, InventoryStore, OllamaEmbedder, OntologyStore, OptionLabel, + PacketStore, PairRelation, Party, Scene, Slot, SpeakerStateStore, ValueKind, Scorer, Split, SplitScheme, StoreError, lower, pronounce, validate, }; #[derive(Parser)] @@ -40,6 +41,9 @@ struct Cli { /// The language ontology store. #[arg(long, global = true, default_value = "data/ontology/ontology.cc")] ontology: PathBuf, + /// The speaker-state store (scenes). + #[arg(long, global = true, default_value = "data/corpus/speaker-state.cc")] + speaker_state: PathBuf, #[command(subcommand)] command: Command, } @@ -57,6 +61,8 @@ enum Command { #[command(subcommand)] Ontology(OntologyCommand), #[command(subcommand)] + Scene(SceneCommand), + #[command(subcommand)] Packet(PacketCommand), /// Lower a stored packet to English text. Lower { @@ -87,7 +93,7 @@ enum Command { #[derive(Subcommand)] enum LoopCommand { - /// Pin the global --corpus, --inventory, --entities and --ontology as a run at ROOT. + /// Pin the global --corpus, --inventory, --entities, --ontology and --speaker-state as a run at ROOT. Open { root: PathBuf, #[arg(long)] @@ -191,6 +197,38 @@ enum InventoryCommand { Proof, /// Admit revision 3 of the Dao De Jing test bed stores (after seed and proof). Dao, + /// Admit the speaker-state revision (Scene and Speaker features) as the + /// inventory's latest plus one. + SpeakerScene, +} + +#[derive(Subcommand)] +enum SceneCommand { + /// Append a scene revision for one corpus group: its one live speaker's + /// relation to the addressee, as Scene-slot feature values. + Put { + #[arg(long)] + group: String, + /// `entity:` of a named addressee, or `unnamed`. + #[arg(long)] + to: String, + /// The inventory revision the glosses resolve at. + #[arg(long)] + inventory_rev: u32, + /// `gloss=value`, repeated: a grade 0-1000, an option label or true/false. + #[arg(long = "set", value_name = "GLOSS=VALUE")] + set: Vec, + #[arg(long, requires = "generator", conflicts_with = "operator")] + model_family: Option, + /// The attestation string naming the annotating model. + #[arg(long, requires = "model_family")] + generator: Option, + /// Annotated by the operator, not a model. + #[arg(long)] + operator: bool, + }, + /// Text projection of the scenes for review; not a store. + Dump, } #[derive(Subcommand)] @@ -734,6 +772,143 @@ fn main() -> Result<()> { .map_err(refused)?; println!("admitted rev:2 with {count} entries"); } + Command::Inventory(InventoryCommand::SpeakerScene) => { + let store = InventoryStore::open(&cli.inventory)?; + let latest = store + .inventory()? + .latest() + .map(|r| r.n) + .context("the inventory holds no revision")?; + let (rev, entries) = speaker_scene_revision(latest + 1); + let (n, count) = (rev.n, entries.len()); + let mut store = store; + store.admit_revision(rev, entries).map_err(refused)?; + println!("admitted rev:{n} with {count} entries"); + } + Command::Scene(SceneCommand::Put { + group, + to, + inventory_rev, + set, + model_family, + generator, + operator, + }) => { + let annotator = match (model_family, generator, operator) { + (Some(family), Some(generator), false) => Annotator::Model { + family: family.into(), + generator, + }, + (None, None, true) => Annotator::Operator, + _ => bail!("name the annotator: --model-family with --generator, or --operator"), + }; + let inventory_store = InventoryStore::open(&cli.inventory)?; + let inventory = inventory_store.inventory()?; + let corpus = CorpusStore::open(&cli.corpus)?; + let live: Vec = corpus + .all()? + .into_iter() + .filter(|i| i.group == group && !i.withdrawn) + .collect(); + let mut speakers: Vec = live.iter().map(|i| i.speaker.clone()).collect(); + speakers.sort(); + speakers.dedup(); + let [from] = speakers.as_slice() else { + bail!("a scene group holds one live speaker; multi-party scenes wait"); + }; + let to = match to.as_str() { + "unnamed" => Party::Unnamed, + named => Party::Named( + EntityId::parse(named).map_err(|_| anyhow::anyhow!("the addressee is unnamed or an entity id"))?, + ), + }; + let mut features = Vec::new(); + for pair in &set { + let bad = || anyhow::anyhow!("a --set is gloss=value of a live Scene feature"); + let (gloss, value) = pair.split_once('=').ok_or_else(bad)?; + let entry = inventory + .entries() + .find(|e| { + e.family() == Family::Feature + && e.gloss == gloss + && e.live_at(inventory_rev) + && e.applies_to.contains(&Slot::Scene) + }) + .ok_or_else(bad)?; + let value = match entry.value_kind.as_ref().ok_or_else(bad)? { + ValueKind::Grade => FeatValue::Grade( + Grade::new(value.parse().map_err(|_| bad())?).map_err(|_| bad())?, + ), + ValueKind::Options(_) => { + FeatValue::Option(OptionLabel::parse(value).map_err(|_| bad())?) + } + ValueKind::Flag => FeatValue::Flag(value.parse().map_err(|_| bad())?), + }; + features.push(FeatureValue { feature: entry.id.clone(), value }); + } + let mut participants = vec![from.clone()]; + if let Party::Named(a) = &to + && a != from + { + participants.push(a.clone()); + } + let mut state = SpeakerStateStore::open(&cli.speaker_state)?; + let latest = state.scene(&group)?; + let mut scene = Scene { + group, + n: latest.as_ref().map_or(1, |l| l.n + 1), + inventory_rev, + participants, + relations: vec![PairRelation { from: from.clone(), to, features }], + annotator, + }; + if let Some(l) = &latest { + let same = Scene { n: l.n, ..scene.clone() }; + if &same == l { + println!("unchanged"); + return Ok(()); + } + } + scene.n = latest.map_or(1, |l| l.n + 1); + let key = weksa::speaker::scene_key(&scene.group, scene.n); + let outcome = state + .put_scene(scene, &inventory_store, &corpus, &EntityStore::open(&cli.entities)?) + .map_err(refused)?; + println!("{outcome:?} {key}"); + } + Command::Scene(SceneCommand::Dump) => { + let inventory = InventoryStore::open(&cli.inventory)?.inventory()?; + let gloss = |f: &FeatureValue| { + let name = inventory.entry(&f.feature).map_or("?", |e| e.gloss.as_str()); + match &f.value { + FeatValue::Grade(g) => format!("{name}={}", g.get()), + FeatValue::Option(l) => format!("{name}={l}"), + FeatValue::Flag(b) => format!("{name}={b}"), + } + }; + for s in SpeakerStateStore::open(&cli.speaker_state)?.all_scenes()? { + let who = match &s.annotator { + Annotator::Operator => "Operator".to_string(), + Annotator::Model { family, generator } => format!("Model {family:?} {generator}"), + }; + let people: Vec<&str> = s.participants.iter().map(|p| p.as_str()).collect(); + println!( + "scene:{}@{} rev:{} {who} [{}]", + s.group, + s.n, + s.inventory_rev, + people.join(",") + ); + for r in &s.relations { + let to = match &r.to { + Party::Named(e) => e.as_str().to_string(), + Party::Unnamed => "unnamed".to_string(), + }; + let feats: Vec = r.features.iter().map(&gloss).collect(); + println!(" {}->{to} {}", r.from, feats.join(" ")); + } + } + } Command::Inventory(InventoryCommand::Dao) => { let (rev, entries) = weksa::daodejing::revision(); let count = entries.len(); @@ -850,6 +1025,7 @@ fn main() -> Result<()> { inventory: &cli.inventory, entities: &cli.entities, ontology: &cli.ontology, + speaker_state: &cli.speaker_state, }, &answerer, &embed_model, diff --git a/crates/weksa/src/ontology.rs b/crates/weksa/src/ontology.rs index 30bc5dd..b1cc384 100644 --- a/crates/weksa/src/ontology.rs +++ b/crates/weksa/src/ontology.rs @@ -1,7 +1,7 @@ //! The language ontology: per target language, the choices its culture makes //! about word, grammar, discourse, sound and script for a speaker in a stance //! (ruling `persona-drives-ontology`). An entry says when it applies (feature -//! conditions over the lowering view's stance and discourse) and what it +//! conditions over the lowering view's stance, discourse and scene relation) and what it //! chooses. `select` is the only reader that decides which entries a lowering //! prompt carries, and it runs only inside `Prompt::lower`; //! `OntologyStore::admit` is the only writer. Entries are immutable. @@ -51,8 +51,8 @@ pub enum Test { Is(FeatValue), } -/// One feature of one slot of the view, tested. Only `Stance` and `Discourse` -/// reach a lowering view today. +/// One feature of one slot of the view, tested. `Stance`, `Discourse` and +/// `Scene` reach a lowering view; `Speaker` does not yet. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub struct Condition { pub slot: Slot, @@ -149,8 +149,9 @@ impl Ontology { } /// The entries of exactly `language` whose every condition holds against the -/// view's stance and discourse features, in `n` order. An absent feature meets -/// no condition. Nothing else of the view is read. +/// view's stance, discourse and scene-relation features (the scene's only when +/// `with_scene` selected a relation), in `n` order. An absent feature meets no +/// condition. Nothing else of the view is read. pub fn select<'a>( o: &'a Ontology, language: &str, @@ -163,6 +164,7 @@ pub fn select<'a>( e.when.iter().all(|c| match c.slot { Slot::Stance => c.holds(&input.stance().features), Slot::Discourse => c.holds(&input.discourse().features), + Slot::Scene => c.holds(input.scene()), _ => false, }) }) diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 770d9b1..6d598d9 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -48,10 +48,12 @@ use crate::packet::{Packet, Stance}; use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate}; use crate::split::{Split, SplitScheme, fnv1a64, fnv1a64_of, hex16}; use crate::store::{ - CorpusStore, EntityStore, InventoryStore, OntologyStore, PacketStore, StoreError, + CorpusStore, EntityStore, InventoryStore, OntologyStore, PacketStore, SpeakerStateStore, + StoreError, check_ontology_entry, open_cache_over, }; -use crate::validate::LoweringInput; +use crate::inventory::Slot; +use crate::validate::{LoweringInput, feature_refusals}; /// The digest of the prompt builder's source: `context.rs`, then `split.rs`, /// whose scheme and digest function decide which lines the prompt holds as @@ -137,6 +139,13 @@ pub struct RunRecord { /// 16 hex of the ontology file's bytes. #[cultcache(key = 16)] pub ontology_digest: String, + /// Canonical absolute path of the speaker-state store (scenes): read-only + /// for the run, pinned by `speaker_state_digest`. + #[cultcache(key = 17)] + pub speaker_state: String, + /// 16 hex of the speaker-state file's bytes. + #[cultcache(key = 18)] + pub speaker_state_digest: String, } /// How a run's lowering prompts reach the model that answers them (rulings @@ -442,6 +451,7 @@ pub enum DriftOf { Corpus, Entities, Ontology, + SpeakerState, Prompts, Model, Embedder, @@ -480,6 +490,8 @@ pub enum RunError { /// An ontology entry of a panel language does not read against the run's /// head inventory at its revision. Ontology, + /// A scene does not read against the run's inventory. + SpeakerState, } impl fmt::Display for RunError { @@ -504,6 +516,9 @@ impl fmt::Display for RunError { RunError::Ontology => f.write_str( "an ontology entry of a panel language does not read against the run's inventory", ), + RunError::SpeakerState => { + f.write_str("a scene does not read against the run's inventory") + } RunError::Panel => f.write_str( "the panel is empty, repeats or misspells a language, or names one with no live Train or Validation item", ), @@ -519,13 +534,14 @@ impl fmt::Debug for RunError { impl std::error::Error for RunError {} -/// The four stores a run pins. +/// The five stores a run pins. #[derive(Clone, Copy)] pub struct RunPaths<'a> { pub corpus: &'a Path, pub inventory: &'a Path, pub entities: &'a Path, pub ontology: &'a Path, + pub speaker_state: &'a Path, } /// A run's open stores, reopened only from its record. The one public producer @@ -538,6 +554,7 @@ pub struct RunStores { inventory: InventoryStore, entities: EntityStore, ontology: OntologyStore, + speaker_state: SpeakerStateStore, } /// The run's own copy of the inventory, made at `create`. @@ -554,6 +571,9 @@ fn check_pins(cache: &CultCache, root: &Path, record: &RunRecord) -> Result<(), if file_digest(&record.ontology).as_ref() != Some(&record.ontology_digest) { return Err(RunError::Drift(DriftOf::Ontology)); } + if file_digest(&record.speaker_state).as_ref() != Some(&record.speaker_state_digest) { + return Err(RunError::Drift(DriftOf::SpeakerState)); + } let (path, digest, _) = head_of(cache, root, record)?; if file_digest_at(&path).as_ref() != Some(&digest) { return Err(RunError::Drift(DriftOf::Inventory)); @@ -692,6 +712,8 @@ impl RunStores { entities: canonical(paths.entities)?, ontology: canonical(paths.ontology)?, ontology_digest: String::new(), + speaker_state: canonical(paths.speaker_state)?, + speaker_state_digest: String::new(), corpus_digest: String::new(), entity_digest: String::new(), prompt_digest: hex16(PROMPT_SOURCE_DIGEST), @@ -714,12 +736,14 @@ impl RunStores { entity_digest: file_digest(&record.entities).ok_or(RunError::Store)?, inventory_digest: file_digest_at(&head).ok_or(RunError::Store)?, ontology_digest: file_digest(&record.ontology).ok_or(RunError::Store)?, + speaker_state_digest: file_digest(&record.speaker_state).ok_or(RunError::Store)?, ..record }; between(); let mut run = Self::with(root, cache, record)?; run.check_panel()?; run.check_ontology()?; + run.check_scenes()?; // The stores were read between the digests: unless they still match // the pins, what was opened is not what was pinned. Nothing is // recorded for a run that did not pin. @@ -771,6 +795,8 @@ impl RunStores { inventory: InventoryStore::open(&head).map_err(store)?, entities: EntityStore::open(Path::new(&record.entities)).map_err(store)?, ontology: OntologyStore::open(Path::new(&record.ontology)).map_err(store)?, + speaker_state: SpeakerStateStore::open(Path::new(&record.speaker_state)) + .map_err(store)?, cache, record, }) @@ -803,6 +829,40 @@ impl RunStores { }) } + /// Every scene's relation features read against the run's inventory at the + /// scene's own revision, and again at the head: the checks `put_scene` + /// applied, applied again to this inventory. + fn check_scenes(&self) -> Result<(), RunError> { + let inventory = self.inventory.inventory().map_err(|_| RunError::Store)?; + let scenes = self.speaker_state.all_scenes().map_err(|_| RunError::Store)?; + for scene in &scenes { + let head = inventory.latest().map(|r| r.n).ok_or(RunError::SpeakerState)?; + for r in &scene.relations { + for rev in [scene.inventory_rev, head] { + if !feature_refusals(&inventory, rev, &r.features, Slot::Scene).is_empty() { + return Err(RunError::SpeakerState); + } + } + } + if scene.relations.is_empty() + && !(inventory.has_revision(scene.inventory_rev) && inventory.has_revision(head)) + { + return Err(RunError::SpeakerState); + } + } + Ok(()) + } + + /// The one door by which a loop view gets its scene: `with_scene` over the + /// run's pinned speaker-state and corpus stores. + fn assembled( + &self, + view: LoweringInput, + inventory: &InventoryStore, + ) -> Result { + view.with_scene(&inventory.inventory()?, &self.speaker_state, &self.corpus) + } + /// Every panel language holds a live Train and a live Validation item. fn check_panel(&self) -> Result<(), RunError> { let items = self.corpus.all().map_err(|_| RunError::Store)?; @@ -1300,6 +1360,9 @@ impl RunStores { let view = stores[slot] .admit(packet, &self.inventory, &self.corpus, &self.entities) .map_err(|_| RunError::Store)?; + let view = self + .assembled(view, &self.inventory) + .map_err(|_| RunError::Store)?; Prompt::lower( &view, &packet_item, @@ -1586,19 +1649,23 @@ fn score_round( // Phase 2: one lowering prompt per admitted view, answered through the // port. Nothing is kept unless every one is answered. let mut unanswered = Vec::new(); - for (at, id, packet_key, view) in &admitted { + for (at, id, packet_key, view) in admitted { // Each item is lowered in its own language: the only reference the // scorer has is the item's own text. - let language = &items[*at].language; - let Ok(prompt) = Prompt::lower(view, id, language, &inventory, &run.entities, &ontology) else { - fates[*at] = Fate::Refused(Stage::Lower, Some(packet_key.clone())); + let language = &items[at].language; + let Ok(view) = run.assembled(view, inventory_store) else { + fates[at] = Fate::Refused(Stage::Lower, Some(packet_key.clone())); + continue; + }; + let Ok(prompt) = Prompt::lower(&view, &id, language, &inventory, &run.entities, &ontology) else { + fates[at] = Fate::Refused(Stage::Lower, Some(packet_key.clone())); continue; }; calls.push(Call { prompt: prompt.digest(), items: prompt.item_keys().to_vec(), }); - fates[*at] = match model.answer(&prompt) { + fates[at] = match model.answer(&prompt) { Ok(answer) => match LoweringDraft::parse(&answer) { Ok(draft) if draft.usable() => { Fate::Lowered(draft.text, packet_key.clone()) @@ -2793,6 +2860,8 @@ mod tests { panel: vec!["en-US".into()], ontology: String::new(), ontology_digest: String::new(), + speaker_state: String::new(), + speaker_state_digest: String::new(), } } @@ -2985,6 +3054,8 @@ mod tests { ); let ontology = dir.path().join("o.cc"); fs::write(&ontology, [0x90]).unwrap(); + let speaker_state = dir.path().join("s.cc"); + fs::write(&speaker_state, [0x90]).unwrap(); let root = dir.path().join("run"); let mut run = RunStores::create( &root, @@ -2994,6 +3065,7 @@ mod tests { inventory: &inventory, entities: &entities, ontology: &ontology, + speaker_state: &speaker_state, }, "fake-model", "bag", diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index 5afb025..cff3086 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -1320,8 +1320,11 @@ pub(crate) fn check_ontology_entry( } let mut out = Vec::new(); for c in &e.when { - if !matches!(c.slot, Slot::Stance | Slot::Discourse) { - return Err(malformed(&key, "a condition reads slot Stance or Discourse")); + if !matches!(c.slot, Slot::Stance | Slot::Discourse | Slot::Scene) { + return Err(malformed( + &key, + "a condition reads slot Stance, Discourse or Scene", + )); } if matches!(c.test, Test::Is(FeatValue::Grade(_))) { return Err(malformed(&key, "Is names an option or a flag")); diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index d46db9f..e90297b 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -14,6 +14,8 @@ fn weksa(dir: &std::path::Path, args: &[&str]) -> Output { "./e.cc", "--ontology", "./o.cc", + "--speaker-state", + "./s.cc", ]) .args(args) .output() @@ -1341,6 +1343,7 @@ fn dao_stores(dir: &std::path::Path) { std::fs::copy(src.join(from), dir.join(to)).unwrap(); } std::fs::write(dir.join("o.cc"), [0x90]).unwrap(); + std::fs::write(dir.join("s.cc"), [0x90]).unwrap(); } #[test] diff --git a/crates/weksa/tests/common/mod.rs b/crates/weksa/tests/common/mod.rs index 8350428..723a371 100644 --- a/crates/weksa/tests/common/mod.rs +++ b/crates/weksa/tests/common/mod.rs @@ -123,6 +123,20 @@ pub fn ontology_in(dir: &std::path::Path) -> std::path::PathBuf { path } +/// An empty speaker-state store beside `ontology` (same directory), made if absent. +pub fn speaker_state_beside(ontology: &std::path::Path) -> std::path::PathBuf { + let path = ontology.with_file_name("speaker-state.cc"); + if !path.exists() { + std::fs::write(&path, [0x90]).unwrap(); + } + path +} + +/// An empty speaker-state store at `/speaker-state.cc`, made if absent. +pub fn speaker_state_in(dir: &std::path::Path) -> std::path::PathBuf { + speaker_state_beside(&dir.join("ontology.cc")) +} + /// A run over the stores `world()` and `corpus()` keep in `dir`: `c.cc`, /// `i.cc` and `entities.cc`. Read at the call, so create it after the items. /// The run's en-US panel needs a live Train and a live Validation item: a @@ -158,6 +172,7 @@ pub fn run_over(dir: &std::path::Path) -> weksa::run::RunStores { inventory: &dir.join("i.cc"), entities: &dir.join("entities.cc"), ontology: &dir.join("ontology.cc"), + speaker_state: &speaker_state_beside(&dir.join("ontology.cc")), }, "fake-model", "bag", diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs index 5d44f01..82cf5b9 100644 --- a/crates/weksa/tests/context.rs +++ b/crates/weksa/tests/context.rs @@ -547,7 +547,8 @@ mod lowering { Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data").join(sub) } - fn admitted( + /// The admitted view, its scene never read. + fn admitted_unread( packet: Packet, inv: &InventoryStore, corpus: &str, @@ -561,6 +562,37 @@ mod lowering { .unwrap() } + /// The admitted view with its scene read over an empty speaker-state store. + fn admitted( + packet: Packet, + inv: &InventoryStore, + corpus: &str, + entities: &EntityStore, + ) -> LoweringInput { + let view = admitted_unread(packet, inv, corpus, entities); + let dir = tempfile::tempdir().unwrap(); + let state = weksa::SpeakerStateStore::open(&dir.path().join("s.cc")).unwrap(); + let corpus = CorpusStore::open(&data(corpus)).unwrap(); + view.with_scene(&inv.inventory().unwrap(), &state, &corpus).unwrap() + } + + #[test] + fn lowering_prompt_refuses_an_unread_scene() { + let cast = InventoryStore::open(&data("interlingua/inventory.cc")).unwrap(); + let entities = EntityStore::open(&data("entity/entity.cc")).unwrap(); + let inv = cast.inventory().unwrap(); + let item = CorpusItemId::parse("item:nibu:1").unwrap(); + let held = Held(vec![nibu(&[("en-US", &["Nibu"])])]); + let unread = admitted_unread(proof_packet(), &cast, "corpus/corpus.cc", &entities); + let empty = weksa::Ontology::empty(); + assert_eq!( + Prompt::lower(&unread, &item, "en-US", &inv, &held, &empty).err(), + Some(LowerError::SceneUnread) + ); + let read = admitted(proof_packet(), &cast, "corpus/corpus.cc", &entities); + assert!(Prompt::lower(&read, &item, "en-US", &inv, &held, &empty).is_ok()); + } + fn dao(index: usize) -> (LoweringInput, Inventory, EntityStore, CorpusItemId) { let inv = InventoryStore::open(&data("daodejing/inventory.cc")).unwrap(); let entities = EntityStore::open(&data("daodejing/entity.cc")).unwrap(); diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 0b22f4c..3c36f33 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -24,12 +24,13 @@ fn rewrite_run(root: &Path, change: impl FnOnce(&mut RunRecord)) { cache.put(format!("run:{}", record.label), &record).unwrap(); } -fn paths(dir: &Path) -> (PathBuf, PathBuf, PathBuf, PathBuf) { +fn paths(dir: &Path) -> (PathBuf, PathBuf, PathBuf, PathBuf, PathBuf) { ( dir.join("c.cc"), dir.join("i.cc"), dir.join("entities.cc"), common::ontology_in(dir), + common::speaker_state_in(dir), ) } @@ -42,7 +43,7 @@ fn a_run_pins_its_stores_and_refuses_drift() { let mut w = world(); let dir = w.dir.path().to_path_buf(); let root = dir.join("run"); - let (c, i, e, o) = paths(&dir); + let (c, i, e, o, s) = paths(&dir); let at = |label: &str, model: &str| { RunStores::create( &root, @@ -52,6 +53,7 @@ fn a_run_pins_its_stores_and_refuses_drift() { inventory: &i, entities: &e, ontology: &o, + speaker_state: &s, }, model, "bag", @@ -147,7 +149,7 @@ fn a_run_works_on_its_own_copy_of_the_inventory() { let w = world(); let dir = w.dir.path().to_path_buf(); let root = dir.join("run"); - let (c, i, e, o) = paths(&dir); + let (c, i, e, o, s) = paths(&dir); let source_bytes = std::fs::read(&i).unwrap(); let mut run = RunStores::create( &root, @@ -157,6 +159,7 @@ fn a_run_works_on_its_own_copy_of_the_inventory() { inventory: &i, entities: &e, ontology: &o, + speaker_state: &s, }, "fake-model", "bag", @@ -236,6 +239,7 @@ fn create_refuses_a_root_that_holds_an_inventory_and_never_truncates_it() { inventory, entities: &dao.entities, ontology: &dao.ontology, + speaker_state: &common::speaker_state_beside(&dao.ontology), }, "fake-model", "bag", @@ -285,6 +289,7 @@ fn a_create_that_fails_leaves_no_inventory_and_the_retry_succeeds() { inventory, entities, ontology: &dao.ontology, + speaker_state: &common::speaker_state_beside(&dao.ontology), }, "fake-model", "bag", @@ -398,11 +403,13 @@ fn a_store_changed_between_its_digest_and_its_open_is_drift() { } let (c, i, e) = (dir.join("c.cc"), dir.join("i.cc"), dir.join("entities.cc")); let o = dir.join("ontology.cc"); + let s = common::speaker_state_beside(&o); let paths = || weksa::run::RunPaths { corpus: &c, inventory: &i, entities: &e, ontology: &o, + speaker_state: &s, }; assert_eq!( RunStores::create_between( @@ -457,6 +464,7 @@ fn window( inventory: &i, entities: &e, ontology: &o, + speaker_state: &common::speaker_state_beside(&o), }; let created = RunStores::create_between( &root, @@ -905,6 +913,7 @@ fn dao_build( inventory: &inventory, entities: &entities, ontology: &ontology, + speaker_state: &common::speaker_state_beside(&ontology), }, "fake-model", "bag", @@ -1087,6 +1096,8 @@ fn lowering_prompts(dao: &Dao, inventory: &InventoryStore) -> Vec<(String, Promp let view = store .admit(packet, inventory, &corpus, dao.run.entities()) .unwrap(); + let state = weksa::SpeakerStateStore::open(&dir.path().join("s.cc")).unwrap(); + let view = view.with_scene(&inv, &state, &corpus).unwrap(); let prompt = Prompt::lower(&view, &key, "en-US", &inv, dao.run.entities(), &weksa::Ontology::empty()).unwrap(); (key.to_string(), prompt) }) @@ -1560,6 +1571,7 @@ fn errors_echo_no_input() { inventory: &missing, entities: &missing, ontology: &missing, + speaker_state: &common::speaker_state_beside(&missing), }; let e = RunStores::create( &root, @@ -1629,6 +1641,7 @@ fn errors_echo_no_input() { inventory: &dao.inventory, entities: &dao.entities, ontology: &dao.ontology, + speaker_state: &common::speaker_state_beside(&dao.ontology), }, "fake-model", "bag", @@ -2551,6 +2564,7 @@ fn a_changed_head_is_drift() { inventory: &dao.inventory, entities: &dao.entities, ontology: &dao.ontology, + speaker_state: &common::speaker_state_beside(&dao.ontology), }, "fake-model", "bag", @@ -2891,6 +2905,7 @@ fn a_summary_line_says_how_lowerings_were_answered() { inventory: &dao.inventory, entities: &dao.entities, ontology: &dao.ontology, + speaker_state: &common::speaker_state_beside(&dao.ontology), }; RunStores::create( &root, @@ -3615,6 +3630,7 @@ fn mixed_family_round(family: ModelFamily) -> u32 { inventory: &dir.join("i.cc"), entities: &dir.join("entities.cc"), ontology: &dir.join("ontology.cc"), + speaker_state: &common::speaker_state_beside(&dir.join("ontology.cc")), }, "not-a-draft", "bag", @@ -4402,7 +4418,7 @@ fn open_refuses_a_bad_panel() { let created = RunStores::create( &root, "p", - RunPaths { corpus, inventory, entities, ontology: &dao.ontology }, + RunPaths { corpus, inventory, entities, ontology: &dao.ontology, speaker_state: &common::speaker_state_beside(&dao.ontology) }, "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, @@ -4762,6 +4778,7 @@ fn create_refuses_an_entry_its_inventory_cannot_read() { inventory: &dir.join("i.cc"), entities: &dir.join("e.cc"), ontology, + speaker_state: &common::speaker_state_beside(&ontology), }, "fake-model", "bag", diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 2a48196..8b06da7 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -307,6 +307,7 @@ fn committed_ontology_is_model_drafted_and_reads_every_inventory() { inventory: &i, entities: &e, ontology: &onto, + speaker_state: &common::speaker_state_beside(&onto), }, "m", "bag", @@ -351,6 +352,7 @@ fn the_recall_probe_covers_every_live_original_item_and_names_no_work() { inventory: &i, entities: &e, ontology: &o, + speaker_state: &common::speaker_state_beside(&o), }, "m", "bag", @@ -429,6 +431,7 @@ fn original_round(family: ModelFamily) -> (weksa::run::RunStores, weksa::run::Ro inventory: &tmp.join("inventory.cc"), entities: &tmp.join("entity.cc"), ontology: &tmp.join("ontology.cc"), + speaker_state: &common::speaker_state_beside(&tmp.join("ontology.cc")), }, "nope", "bag", @@ -588,6 +591,7 @@ fn no_test_item_is_offered_to_a_round_or_recall_or_recorded() { inventory: &p("inventory.cc"), entities: &p("entity.cc"), ontology: &p("ontology.cc"), + speaker_state: &common::speaker_state_beside(&p("ontology.cc")), }, "nope", "bag", diff --git a/crates/weksa/tests/scene.rs b/crates/weksa/tests/scene.rs index a85b2e0..66f3cb7 100644 --- a/crates/weksa/tests/scene.rs +++ b/crates/weksa/tests/scene.rs @@ -27,7 +27,7 @@ struct W { fn world() -> W { let dir = tempfile::tempdir().unwrap(); let mut inventory = InventoryStore::open(&dir.path().join("i.cc")).unwrap(); - for (rev, entries) in [seed(), proof_revision(), speaker_scene_revision()] { + for (rev, entries) in [seed(), proof_revision(), speaker_scene_revision(3)] { inventory.admit_revision(rev, entries).unwrap(); } let entities = entities_in(dir.path()); @@ -60,7 +60,7 @@ fn feat(gloss: &str, value: FeatValue) -> FeatureValue { let feature = seed() .1 .into_iter() - .chain(speaker_scene_revision().1) + .chain(speaker_scene_revision(3).1) .find(|e| e.gloss == gloss) .unwrap_or_else(|| panic!("no feature {gloss}")) .id; diff --git a/crates/weksa/tests/speaker_state.rs b/crates/weksa/tests/speaker_state.rs index dfb4c59..d79b9dd 100644 --- a/crates/weksa/tests/speaker_state.rs +++ b/crates/weksa/tests/speaker_state.rs @@ -26,7 +26,7 @@ struct W { fn world() -> W { let dir = tempfile::tempdir().unwrap(); let mut inventory = InventoryStore::open(&dir.path().join("i.cc")).unwrap(); - for (rev, entries) in [seed(), proof_revision(), speaker_scene_revision()] { + for (rev, entries) in [seed(), proof_revision(), speaker_scene_revision(3)] { inventory.admit_revision(rev, entries).unwrap(); } let entities = entities_in(dir.path()); @@ -60,7 +60,7 @@ fn feat(gloss: &str, value: FeatValue) -> FeatureValue { let feature = seed() .1 .into_iter() - .chain(speaker_scene_revision().1) + .chain(speaker_scene_revision(3).1) .find(|e| e.gloss == gloss) .unwrap_or_else(|| panic!("no feature {gloss}")) .id; diff --git a/schemas/cultnet/weksa.loop_run.schema.json b/schemas/cultnet/weksa.loop_run.schema.json index e5b144b..346f41c 100644 --- a/schemas/cultnet/weksa.loop_run.schema.json +++ b/schemas/cultnet/weksa.loop_run.schema.json @@ -66,6 +66,14 @@ }, "prompt_digest": { "type": "string" + }, + "speaker_state": { + "description": "Canonical absolute path of the speaker-state store (scenes): read-only\nfor the run, pinned by `speaker_state_digest`.", + "type": "string" + }, + "speaker_state_digest": { + "description": "16 hex of the speaker-state file's bytes.", + "type": "string" } }, "required": [ @@ -83,7 +91,9 @@ "lowerer_reach", "panel", "ontology", - "ontology_digest" + "ontology_digest", + "speaker_state", + "speaker_state_digest" ], "$defs": { "LowererReach": { diff --git a/schemas/cultnet/weksa.ontology_entry.schema.json b/schemas/cultnet/weksa.ontology_entry.schema.json index 080275b..4483fa7 100644 --- a/schemas/cultnet/weksa.ontology_entry.schema.json +++ b/schemas/cultnet/weksa.ontology_entry.schema.json @@ -89,7 +89,7 @@ ] }, "Condition": { - "description": "One feature of one slot of the view, tested. Only `Stance` and `Discourse`\nreach a lowering view today.", + "description": "One feature of one slot of the view, tested. `Stance`, `Discourse` and\n`Scene` reach a lowering view; `Speaker` does not yet.", "type": "object", "properties": { "feature": { From 966a3678a70df3ff410fd371216c47585f5d781d Mon Sep 17 00:00:00 2001 From: Weksa Date: Thu, 12 Nov 2026 16:31:55 +0000 Subject: [PATCH 15/57] language-ontology-scene s3-6: inventory rev 4 in both beds, empty speaker-state stores committed, scene tests, original corpus checks, lowering map Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/main.rs | 2 +- crates/weksa/src/run.rs | 4 +- crates/weksa/tests/cli.rs | 65 ++++++- crates/weksa/tests/common/mod.rs | 15 ++ crates/weksa/tests/golf_loop.rs | 217 ++++++++++++++++++++++- crates/weksa/tests/original_corpus.rs | 38 +++- crates/weksa/tests/scene.rs | 57 ++++++ crates/weksa/tests/stores.rs | 41 ++++- data/corpus/speaker-state.cc | 1 + data/daodejing/inventory.cc | Bin 10559 -> 11369 bytes data/daodejing/speaker-state.cc | 1 + data/original/inventory.cc | Bin 10559 -> 11369 bytes data/original/speaker-state.cc | 1 + docs/targets/interlingua-lowering-map.md | 8 +- 14 files changed, 429 insertions(+), 21 deletions(-) create mode 100644 data/corpus/speaker-state.cc create mode 100644 data/daodejing/speaker-state.cc create mode 100644 data/original/speaker-state.cc diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index efae4ff..c4d7be6 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -844,7 +844,7 @@ fn main() -> Result<()> { } ValueKind::Flag => FeatValue::Flag(value.parse().map_err(|_| bad())?), }; - features.push(FeatureValue { feature: entry.id.clone(), value }); + features.push(FeatureValue { feature: entry.id, value }); } let mut participants = vec![from.clone()]; if let Party::Named(a) = &to diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 6d598d9..8b49f8e 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -2953,7 +2953,7 @@ mod tests { } /// Answers the Dao encode prompts from `answers` by item, the candidate - /// inventory's (rev 4) from the good drafts, and the editor with one concept. + /// inventory's (rev 5) from the good drafts, and the editor with one concept. struct Fake { degraded: HashMap, good: HashMap, @@ -2980,7 +2980,7 @@ mod tests { }; return Ok(format!(r#"{{"text":"{text}"}}"#)); } - let answers = if prompt.text().contains("\n\ninventory rev:4\n") { + let answers = if prompt.text().contains("\n\ninventory rev:5\n") { &self.good } else { &self.degraded diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index e90297b..65563ea 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -1416,7 +1416,7 @@ fn loop_open_and_unanswered_round() { assert!(second.status.success(), "{}", out(&second)); assert_eq!( out(&second), - "round 1 rev 3 en-US train n=1 scored=0 chrf=0.00 meaning=0.0000 refused=1 validation n=2 scored=0 chrf=0.00 meaning=0.0000 refused parse=2 admit=0 lower=0 parse-reasons=syntaxx3 lowering=tool-less-relay\n" + "round 1 rev 4 en-US train n=1 scored=0 chrf=0.00 meaning=0.0000 refused=1 validation n=2 scored=0 chrf=0.00 meaning=0.0000 refused parse=2 admit=0 lower=0 parse-reasons=syntaxx3 lowering=tool-less-relay\n" ); // The global store flags are not read by loop round: a drifted store is refused. @@ -1549,7 +1549,7 @@ fn loop_cycle_asks_then_decides() { assert!(done.status.success(), "{}", out(&done)); assert_eq!( out(&done), - "cycle 1 rev 3->none en-US base=0.0000 candidate=none floor=0.0000 threshold=0.0050 rejected edit-parse lowering=tool-less-relay\n" + "cycle 1 rev 4->none en-US base=0.0000 candidate=none floor=0.0000 threshold=0.0050 rejected edit-parse lowering=tool-less-relay\n" ); // The next cycle asks afresh in its own directory. @@ -1814,3 +1814,64 @@ fn ontology_admit_and_dump() { assert!(!out(&o).contains("LEAKYCHOICE"), "{}", out(&o)); assert_eq!(out(&weksa(d.path(), &["ontology", "dump"])), before); } + +#[test] +fn scene_put_and_dump() { + let d = tempfile::tempdir().unwrap(); + cast_entities(d.path(), &["nibu", "brokkr"]); + for step in ["seed", "proof", "speaker-scene"] { + let o = weksa(d.path(), &["inventory", step]); + assert!(o.status.success(), "{}", out(&o)); + } + let mut add = ADD.to_vec(); + add.extend(["--persona", "nibu", "--fixture", "f.yaml", "--author", "Ada Writer"]); + let o = weksa(d.path(), &add); + assert!(o.status.success(), "{}", out(&o)); + let put = |group: &str, to: &str, sets: &[&str]| { + let mut args = vec![ + "scene", "put", "--group", group, "--to", to, "--inventory-rev", "3", + "--model-family", "anthropic", "--generator", "a scene annotator", + ]; + for s in sets { + args.extend(["--set", s]); + } + weksa(d.path(), &args) + }; + let sets = ["relative_status=above", "intimacy=200", "in_group=false"]; + let o = put("scene-1", "unnamed", &sets); + assert!(o.status.success(), "{}", out(&o)); + let dump = out(&weksa(d.path(), &["scene", "dump"])); + for want in ["scene:scene-1@1", "rev:3", "Anthropic", "a scene annotator", "entity:nibu->unnamed", "relative_status=above", "intimacy=200", "in_group=false"] { + assert!(dump.contains(want), "{want}: {dump}"); + } + // A repeat is unchanged and writes nothing. + let o = put("scene-1", "unnamed", &sets); + assert!(out(&o).contains("unchanged"), "{}", out(&o)); + assert_eq!(out(&weksa(d.path(), &["scene", "dump"])), dump); + // A changed value is the next revision. + let o = put("scene-1", "unnamed", &["relative_status=level", "intimacy=200", "in_group=false"]); + assert!(o.status.success(), "{}", out(&o)); + assert!(out(&weksa(d.path(), &["scene", "dump"])).contains("scene:scene-1@2")); + // An unknown gloss and an unknown group are refused and not echoed. + for (group, set) in [("scene-1", "canarygloss=1"), ("canarygroup", "intimacy=1")] { + let before = out(&weksa(d.path(), &["scene", "dump"])); + let o = put(group, "unnamed", &[set]); + assert!(!o.status.success()); + assert!(!out(&o).contains("canary"), "{}", out(&o)); + assert_eq!(out(&weksa(d.path(), &["scene", "dump"])), before); + } + // A group with two live speakers is refused: multi-party scenes wait. + let mut two = add.clone(); + let at = two.iter().position(|a| *a == "entity:nibu").unwrap(); + two[at] = "entity:brokkr"; + let at = two.iter().position(|a| *a == "nibu").unwrap(); + two[at] = "brokkr"; + two.retain(|a| *a != "Mind the hatch."); + let at = two.iter().position(|a| *a == "--text").unwrap() + 1; + two.insert(at, "Brokkr speaks."); + let o = weksa(d.path(), &two); + assert!(o.status.success(), "{}", out(&o)); + let o = put("scene-1", "unnamed", &sets); + assert!(!o.status.success()); + assert!(!out(&o).contains("scene-1"), "{}", out(&o)); +} diff --git a/crates/weksa/tests/common/mod.rs b/crates/weksa/tests/common/mod.rs index 723a371..fa2deb9 100644 --- a/crates/weksa/tests/common/mod.rs +++ b/crates/weksa/tests/common/mod.rs @@ -123,6 +123,21 @@ pub fn ontology_in(dir: &std::path::Path) -> std::path::PathBuf { path } +/// A new inventory store at `path` holding the Dao inventory as it stood at +/// rev 3 (seed, proof, the Dao revision): the loop tests' fixed head, whose +/// round and candidate revisions they pin. The committed +/// `data/daodejing/inventory.cc` is rev 4 and is read by the corpus tests. +pub fn dao_rev3_at(path: &std::path::Path) { + let mut store = weksa::InventoryStore::open(path).unwrap(); + for (rev, entries) in [ + weksa::inventory::seed(), + weksa::inventory::proof_revision(), + weksa::daodejing::revision(), + ] { + store.admit_revision(rev, entries).unwrap(); + } +} + /// An empty speaker-state store beside `ontology` (same directory), made if absent. pub fn speaker_state_beside(ontology: &std::path::Path) -> std::path::PathBuf { let path = ontology.with_file_name("speaker-state.cc"); diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 3c36f33..d2bf719 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -896,9 +896,11 @@ fn dao_build( std::fs::copy(src.join(from), dir.path().join(to)).unwrap(); dir.path().join(to) }; + // The Dao inventory at rev 3: the pins below are written against it. + common::dao_rev3_at(&dir.path().join("i.cc")); let (corpus, inventory, entities) = ( copy("corpus.cc", "c.cc"), - copy("inventory.cc", "i.cc"), + dir.path().join("i.cc"), copy("entity.cc", "e.cc"), ); edit(&corpus, &entities); @@ -4778,7 +4780,7 @@ fn create_refuses_an_entry_its_inventory_cannot_read() { inventory: &dir.join("i.cc"), entities: &dir.join("e.cc"), ontology, - speaker_state: &common::speaker_state_beside(&ontology), + speaker_state: &common::speaker_state_beside(ontology), }, "fake-model", "bag", @@ -4834,3 +4836,214 @@ fn round_lowering_prompts_carry_selected_entries() { } assert_eq!(seen.len(), 6, "three items in each language"); } + +/// Admits the speaker-state revision (rev 4) onto the rev-3 Dao inventory at +/// `inventory`, and the scene for the group of `item:daodejing:3`: Laozi's +/// relation to an unnamed party, above, intimacy 200. +fn put_dao_scene(speaker_state: &Path, inventory: &Path, corpus: &Path, entities: &Path) { + use weksa::packet::{FeatValue, FeatureValue}; + let mut inv_store = InventoryStore::open(inventory).unwrap(); + let (rev, entries) = weksa::inventory::speaker_scene_revision(4); + inv_store.admit_revision(rev, entries).unwrap(); + let inv = inv_store.inventory().unwrap(); + let id = |gloss: &str| inv.entries().find(|e| e.gloss == gloss).unwrap().id; + let corpus = CorpusStore::open(corpus).unwrap(); + let item = corpus.get("item:daodejing:3").unwrap().unwrap(); + let scene = weksa::Scene { + group: item.group.clone(), + n: 1, + inventory_rev: 4, + participants: vec![item.speaker.clone()], + relations: vec![weksa::PairRelation { + from: item.speaker.clone(), + to: weksa::Party::Unnamed, + features: vec![ + FeatureValue { + feature: id("relative_status"), + value: FeatValue::Option(common::lab("above")), + }, + FeatureValue { + feature: id("intimacy"), + value: FeatValue::Grade(common::grade(200)), + }, + ], + }], + annotator: weksa::Annotator::Operator, + }; + weksa::SpeakerStateStore::open(speaker_state) + .unwrap() + .put_scene(scene, &inv_store, &corpus, &weksa::EntityStore::open(entities).unwrap()) + .unwrap(); +} + +/// A Dao run whose store holds a scene for item 3's group and whose ontology +/// holds en-US:1, selected when the speaker is above the addressee. +fn scene_dao(extra: impl FnOnce(&Path, &Path)) -> Dao { + dao_build( + |_, _| {}, + &["en-US"], + |ontology, inventory| { + use weksa::{Condition, Layer, OntologyEntry, OntologyStore, Slot, Test}; + let dir = ontology.parent().unwrap(); + put_dao_scene( + &common::speaker_state_beside(ontology), + inventory, + &dir.join("c.cc"), + &dir.join("e.cc"), + ); + let inv_store = InventoryStore::open(inventory).unwrap(); + let status = inv_store + .inventory() + .unwrap() + .entries() + .find(|e| e.gloss == "relative_status") + .unwrap() + .id; + OntologyStore::open(ontology) + .unwrap() + .admit( + vec![OntologyEntry { + language: "en-US".into(), + n: 1, + inventory_rev: 4, + layer: Layer::Discourse, + when: vec![Condition { + slot: Slot::Scene, + feature: status, + test: Test::Is(weksa::packet::FeatValue::Option(common::lab("above"))), + }], + choice: "ENABOVE".into(), + examples: vec![], + annotator: weksa::Annotator::Operator, + }], + &inv_store, + ) + .unwrap(); + extra(&dir.join("c.cc"), &dir.join("e.cc")); + }, + ) +} + +/// Answers for items 1 to 3 where item 3 names referent n2 (no entity) as its +/// addressee, or nobody. +fn addressing(addressee: Option<&str>) -> Vec<(usize, String)> { + (1..=3) + .map(|n| { + let mut v = serde_json::to_value(draft_of(n)).unwrap(); + if n == 3 { + v["discourse"]["addressee"] = serde_json::json!(addressee); + } + (n, v.to_string()) + }) + .collect() +} + +fn lowering_of(model: &FakeModel, marker: &str) -> Vec { + lowering_texts(model) + .into_iter() + .filter(|t| t.contains(marker)) + .map(String::from) + .collect() +} + +#[test] +fn round_prompts_carry_scene_selected_entries() { + // Above an unnamed addressee: item 3's prompt holds the entry, the other + // two (no scene for their groups) read NoScene and print without it. + let mut dao = scene_dao(|_, _| {}); + let mut model = fake(&addressing(Some("n2"))); + recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + assert_eq!(lowering_texts(&model).len(), 3); + let with = lowering_of(&model, "\nonto:en-US:1 [discourse] ENABOVE"); + assert_eq!(with.len(), 1, "only item 3 has a scene and an addressee"); + assert_eq!( + lowering_texts(&model).iter().filter(|t| t.contains("ontology: none")).count(), + 2 + ); + // The same item with no addressee selects no Scene entry. + let mut dao = scene_dao(|_, _| {}); + let mut model = fake(&addressing(None)); + recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + assert!(lowering_of(&model, "ENABOVE").is_empty()); + assert_eq!(lowering_texts(&model).len(), 3); +} + +#[test] +fn a_scene_read_failure_is_a_lower_refusal() { + // A live item of item 3's group, in a language outside the panel, spoken by + // someone the scene does not cover: V16 for that group. + let mut dao = scene_dao(|corpus, entities| { + let mut entity_store = weksa::EntityStore::open(entities).unwrap(); + entity_store.put(ent("stranger", "en-US", &["Stranger"])).unwrap(); + let mut store = CorpusStore::open(corpus).unwrap(); + let mut twin = store.get("item:daodejing:3").unwrap().unwrap(); + twin.id = "item:daodejing:93".into(); + twin.language = "pt-BR".into(); + twin.text = "PORTUGUESEMARK 93 o sabio".into(); + twin.speaker = common::eid("stranger"); + twin.names = vec![]; + twin.prior_turns = vec![]; + store.add(twin, &entity_store).unwrap(); + }); + let mut model = fake(&addressing(Some("n2"))); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + // Item 3 never reached the model; items 1 and 2 did, and the round recorded. + assert_eq!(lowering_texts(&model).len(), 2); + assert_eq!(r.n(), 1); +} + +#[test] +fn a_run_pins_its_speaker_state_and_refuses_drift() { + let dao = scene_dao(|_, _| {}); + let r = dao.run.record(); + let path = dao.ontology.with_file_name("speaker-state.cc"); + assert_eq!(r.speaker_state, std::fs::canonicalize(&path).unwrap().to_str().unwrap()); + assert_eq!(r.speaker_state_digest, hex(&path)); + let root = dao.root.clone(); + assert_eq!(RunStores::open(&root).unwrap().record(), dao.run.record()); + let mut bytes = std::fs::read(&path).unwrap(); + bytes.push(0xc0); + std::fs::write(&path, bytes).unwrap(); + assert_eq!( + RunStores::open(&root).err(), + Some(RunError::Drift(DriftOf::SpeakerState)) + ); +} + +#[test] +fn create_refuses_a_scene_its_inventory_cannot_read() { + // The scene is admitted against a copy that gained rev 4; the run's head + // inventory stops at rev 3. + let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/daodejing"); + let dir = tempfile::tempdir().unwrap(); + for (from, to) in [("corpus.cc", "c.cc"), ("entity.cc", "e.cc")] { + std::fs::copy(src.join(from), dir.path().join(to)).unwrap(); + } + common::dao_rev3_at(&dir.path().join("i.cc")); + common::dao_rev3_at(&dir.path().join("rich.cc")); + let state = common::speaker_state_in(dir.path()); + put_dao_scene(&state, &dir.path().join("rich.cc"), &dir.path().join("c.cc"), &dir.path().join("e.cc")); + let ontology = common::ontology_in(dir.path()); + let root = dir.path().join("run"); + let create = || { + RunStores::create( + &root, + "x", + RunPaths { + corpus: &dir.path().join("c.cc"), + inventory: &dir.path().join("i.cc"), + entities: &dir.path().join("e.cc"), + ontology: &ontology, + speaker_state: &state, + }, + "fake-model", + "bag", + weksa::run::LowererReach::ToolLessRelay, + ModelFamily::Unrecorded, + &["en-US".to_string()], + ) + }; + assert_eq!(create().err(), Some(RunError::SpeakerState)); + assert!(!root.join("run.cc").exists()); + assert!(!root.join("inventory.cc").exists()); +} diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 8b06da7..8838554 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -18,7 +18,7 @@ use weksa::{CorpusStore, EntityStore, InventoryStore}; fn stores() -> (CorpusStore, EntityStore) { let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/original"); let tmp = tempfile::tempdir().unwrap(); - for f in ["corpus.cc", "entity.cc", "inventory.cc"] { + for f in ["corpus.cc", "entity.cc", "inventory.cc", "speaker-state.cc"] { std::fs::copy(dir.join(f), tmp.path().join(f)).unwrap(); } let keep = tmp.keep(); @@ -241,14 +241,22 @@ fn the_first_six_original_items_are_unchanged() { } #[test] -fn the_original_inventory_is_the_dao_revision_three() { +fn the_original_inventory_is_the_dao_inventory() { let dao = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/daodejing/inventory.cc"); let own = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/original/inventory.cc"); assert_eq!(std::fs::read(dao).unwrap(), std::fs::read(&own).unwrap()); let tmp = tempfile::tempdir().unwrap(); let copy = tmp.path().join("inventory.cc"); std::fs::copy(&own, ©).unwrap(); - assert!(InventoryStore::open(©).is_ok()); + let inventory = InventoryStore::open(©).unwrap().inventory().unwrap(); + // Revision 4 is the speaker-state vocabulary: the same six features the + // generator admits, ids and all. + assert_eq!(inventory.latest().unwrap().n, 4); + let (rev, entries) = weksa::inventory::speaker_scene_revision(4); + assert_eq!(inventory.latest().unwrap().changes, rev.changes); + for e in &entries { + assert_eq!(inventory.entry(&e.id), Some(e), "{}", e.gloss); + } } /// The committed seed ontology: every entry model-drafted, exactly the two @@ -270,8 +278,12 @@ fn committed_ontology_is_model_drafted_and_reads_every_inventory() { match &e.annotator { Annotator::Model { family, generator } => { assert_eq!(*family, ModelFamily::Anthropic, "{}", e.key()); - assert_eq!( - generator, "claude-sonnet-5-5 (Weksa Hands, model-drafted seed ontology)", + assert!( + [ + "claude-sonnet-5-5 (Weksa Hands, model-drafted seed ontology)", + "claude-sonnet-5-5 (Weksa Hands, model-drafted scene ontology)", + ] + .contains(&generator.as_str()), "{}", e.key() ); @@ -283,6 +295,15 @@ fn committed_ontology_is_model_drafted_and_reads_every_inventory() { langs.sort(); langs.dedup(); assert_eq!(langs, PANEL); + // Every entry from number 10 on is conditioned on the scene relation, and + // only an entry with a Scene condition may name a politeness level. + for e in entries.iter().filter(|e| e.n >= 10) { + assert!( + e.when.iter().any(|c| c.slot == weksa::inventory::Slot::Scene), + "{} has no Scene condition", + e.key() + ); + } for lang in PANEL { let stance = |c: &Condition| c.slot == weksa::inventory::Slot::Stance; assert!( @@ -299,6 +320,7 @@ fn committed_ontology_is_model_drafted_and_reads_every_inventory() { tmp.path().join(format!("{corpus}-{f}")) }; let (c, i, e) = (copy("corpus.cc"), copy("inventory.cc"), copy("entity.cc")); + let s = copy("speaker-state.cc"); RunStores::create( &tmp.path().join(format!("run-{corpus}")), corpus, @@ -307,7 +329,7 @@ fn committed_ontology_is_model_drafted_and_reads_every_inventory() { inventory: &i, entities: &e, ontology: &onto, - speaker_state: &common::speaker_state_beside(&onto), + speaker_state: &s, }, "m", "bag", @@ -419,7 +441,7 @@ fn original_round(family: ModelFamily) -> (weksa::run::RunStores, weksa::run::Ro use weksa::run::{LowererReach, RoundOutcome, RunPaths, RunStores, round}; let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/original"); let tmp = tempfile::tempdir().unwrap().keep(); - for f in ["corpus.cc", "entity.cc", "inventory.cc"] { + for f in ["corpus.cc", "entity.cc", "inventory.cc", "speaker-state.cc"] { std::fs::copy(src.join(f), tmp.join(f)).unwrap(); } std::fs::write(tmp.join("ontology.cc"), [0x90]).unwrap(); @@ -578,7 +600,7 @@ fn no_test_item_is_offered_to_a_round_or_recall_or_recorded() { use weksa::run::{LowererReach, RoundOutcome, RunPaths, RunStores, round}; let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/original"); let tmp = tempfile::tempdir().unwrap(); - for f in ["corpus.cc", "entity.cc", "inventory.cc"] { + for f in ["corpus.cc", "entity.cc", "inventory.cc", "speaker-state.cc"] { std::fs::copy(src.join(f), tmp.path().join(f)).unwrap(); } std::fs::write(tmp.path().join("ontology.cc"), [0x90]).unwrap(); diff --git a/crates/weksa/tests/scene.rs b/crates/weksa/tests/scene.rs index 66f3cb7..5974883 100644 --- a/crates/weksa/tests/scene.rs +++ b/crates/weksa/tests/scene.rs @@ -408,3 +408,60 @@ fn only_live_speakers_of_the_views_group_and_the_views_own_speaker_must_be_cover ["V16"] ); } + +mod select { + use super::*; + use weksa::speaker::Annotator; + use weksa::{Condition, Layer, OntologyEntry, OntologyStore, Slot, Test, select}; + + fn scene_feature(gloss: &str) -> weksa::InvId { + speaker_scene_revision(3).1.into_iter().find(|e| e.gloss == gloss).unwrap().id + } + + fn entry(n: u32, slot: Slot, gloss: &str, test: Test) -> OntologyEntry { + OntologyEntry { + language: "ja-JP".into(), + n, + inventory_rev: 3, + layer: Layer::Grammar, + when: vec![Condition { slot, feature: scene_feature(gloss), test }], + choice: "a choice".into(), + examples: vec![], + annotator: Annotator::Operator, + } + } + + #[test] + fn select_reads_the_scene_relation() { + let mut w = world(); + put(&mut w, standard("g", 1)); + let mut store = OntologyStore::open(&w.dir.path().join("o.cc")).unwrap(); + store + .admit( + vec![ + entry(1, Slot::Scene, "relative_status", Test::Is(FeatValue::Option(lab("above")))), + entry(2, Slot::Scene, "intimacy", Test::AtLeast(grade(150))), + ], + &w.inventory, + ) + .unwrap(); + let ontology = store.ontology().unwrap(); + let keys = |read: SceneRead, w: &W| { + let view = view_of(w, 3, Some("n3")).with_scene(&inv(w), &w.state, &w.corpus).unwrap(); + assert_eq!(view.scene_read(), &read); + select(&ontology, "ja-JP", &view).iter().map(|e| e.n).collect::>() + }; + // Brokkr is above-status at intimacy 200: both Scene entries select. + assert_eq!(keys(selected(1, status()), &w), [1, 2]); + // No scene for the group: neither selects. + let empty = world(); + assert_eq!(keys(SceneRead::NoScene, &empty), Vec::::new()); + // No addressee, and no relation to the addressee: neither selects. + let view = view_of(&w, 3, None).with_scene(&inv(&w), &w.state, &w.corpus).unwrap(); + assert_eq!(view.scene_read(), &SceneRead::NoAddressee { rev: 1 }); + assert!(select(&ontology, "ja-JP", &view).is_empty()); + let mut lone = world(); + put(&mut lone, scene("g", 1, &["nibu", "brokkr"], vec![relation("brokkr", Party::Named(eid("nibu")), intimacy(99))])); + assert_eq!(keys(SceneRead::NoRelation { rev: 1 }, &lone), Vec::::new()); + } +} diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs index f027611..5b26e85 100644 --- a/crates/weksa/tests/stores.rs +++ b/crates/weksa/tests/stores.rs @@ -1740,9 +1740,9 @@ mod ontology { "five conditions", vec![OntologyEntry { when: (0..5).map(|_| restraint()).collect(), ..entry("ja-JP", 1) }], )); - for slot in [Slot::Speaker, Slot::Scene, Slot::Predication] { + for slot in [Slot::Speaker, Slot::Predication] { cases.push(( - "slot not Stance or Discourse", + "slot not Stance, Discourse or Scene", vec![OntologyEntry { when: vec![when(slot, "restraint", Test::AtLeast(Grade::new(1).unwrap()))], ..entry("ja-JP", 1) @@ -1818,6 +1818,43 @@ mod ontology { assert_eq!(s.all().unwrap().len(), 64); } + #[test] + fn ontology_admit_takes_scene_refuses_speaker() { + use weksa::inventory::{proof_revision, speaker_scene_revision}; + let d = dir(); + let mut inventory = InventoryStore::open(&d.path().join("i.cc")).unwrap(); + for (rev, entries) in [seed(), proof_revision(), speaker_scene_revision(3)] { + inventory.admit_revision(rev, entries).unwrap(); + } + let scene_feature = |gloss: &str| { + speaker_scene_revision(3).1.into_iter().find(|e| e.gloss == gloss).unwrap().id + }; + let at = |slot, feature, test| OntologyEntry { + inventory_rev: 3, + when: vec![Condition { slot, feature, test }], + ..entry("ja-JP", 1) + }; + let above = Test::Is(FeatValue::Option(lab("above"))); + let mut s = store(&d); + // A Scene condition on a Scene feature admits. + assert_eq!( + s.admit(vec![at(Slot::Scene, scene_feature("relative_status"), above.clone())], &inventory) + .unwrap(), + AddOutcome::Added + ); + // Speaker and Predication slots refuse, and a Scene condition naming a + // Stance feature refuses. + for bad in [ + at(Slot::Speaker, scene_feature("education"), Test::AtLeast(Grade::new(1).unwrap())), + at(Slot::Predication, feature("modality"), Test::Is(FeatValue::Option(lab("actual")))), + at(Slot::Scene, feature("restraint"), Test::AtLeast(Grade::new(1).unwrap())), + ] { + let bad = OntologyEntry { n: 2, ..bad }; + assert!(s.admit(vec![bad], &inventory).is_err()); + } + assert_eq!(s.all().unwrap().len(), 1); + } + #[test] fn ontology_admit_is_all_or_nothing() { let d = dir(); diff --git a/data/corpus/speaker-state.cc b/data/corpus/speaker-state.cc new file mode 100644 index 0000000..9e99dd5 --- /dev/null +++ b/data/corpus/speaker-state.cc @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/data/daodejing/inventory.cc b/data/daodejing/inventory.cc index 373eb3cf259663321fdf94128b4f11a984998f01..3904b809b75c3558a065b8043c36df0a79b8aeaf 100644 GIT binary patch delta 653 zcmaLT!Ab%_7zc1Pp%C!_f>0{*(!p7`z$F4Nb*dl>qC?9#^R;e`JF+{w;n61uS@Q*I zFzt{abP9^-5qg1+y+te4y6d6Kz<&LFIDN15hUb8VYlDW$k2WwYOb>KN1brw;W&AfblEI zzyl_6jeF3MRRR{Ie3WgcLqqY4$^3R~!s>K_QoJ`UVKy*n$z2=M5P9uRg~`sjtr)Qq zAy7c*hRxmP-44UX-x^WD|6I!N#Q-$|C{GS2CWoLS-Z07JREwIH@w-vK+5~(Fb}A;~ zg?uF@79wJ40MsQxaoqY3Q74DhV#;wuQHR(7o$xxhMOgfeBz2R~A*P&x6BI5UTHLh^ z9aEPH`SSv!V0XgB$D{1^;(S-y;AyJ=yteGbW0{*(!p7`z$F4Nb*dl>qC?9#^R;e`JF+{w;n61uS@Q*I zFzt{abP9^-5qg1+y+te4y6d6Kz<&LFIDN15hUb8VYlDW$k2WwYOb>KN1brw;W&AfblEI zzyl_6jeF3MRRR{Ie3WgcLqqY4$^3R~!s>K_QoJ`UVKy*n$z2=M5P9uRg~`sjtr)Qq zAy7c*hRxmP-44UX-x^WD|6I!N#Q-$|C{GS2CWoLS-Z07JREwIH@w-vK+5~(Fb}A;~ zg?uF@79wJ40MsQxaoqY3Q74DhV#;wuQHR(7o$xxhMOgfeBz2R~A*P&x6BI5UTHLh^ z9aEPH`SSv!V0XgB$D{1^;(S-y;AyJ=yteGbW` constant in `crates/weksa`; published JSON Schema at `schemas/cultnet/` | Bumped by a commit that changes the Rust types; the published schema is regenerated in the same commit and a test fails on drift | A cut (Hands, under a spec); never the loop | | Inventory entry | Key `:`, `n` canonical decimal >= 1 (families: concept, frame, role, relation, speech_act, feature, referent_kind, extension); type `weksa.inventory_entry`, with a bounded gloss, value kind (option labels bounded), the slots a feature may sit in (`Discourse`, `Referent`, `Predication`, `Relation`; `Stance` from cut `speaker-state`, `Speaker` and `Scene` from cut `speaker-scene`), `introduced_rev` and optional `retired_rev`. Speaker axes, scene relations and affects are features of those slots, so the loop grows them like any closed-class family and convergence covers them | Added or retired by an inventory revision; never edited in place (a changed meaning is a retire plus an add); a retired id still resolves for packets of earlier revisions | The loop's accepted rounds, whose minting call the context builder assembles from Train items only; Self on merge to main. Admission checks form, not text | | Inventory revision | Key `rev:`, type `weksa.inventory_revision`, with parent, change list and `frozen` | Appended per accepted round; exactly one revision is ever marked frozen, by the convergence test | The loop's accept rule appends; the convergence test freezes; Self records the freeze as a ruling | -| Target profile | Locale or language-project id (`en-US`, `ja-JP`, a conlang's slug). Until entries grow in rounds there is no profile revision document: a run names the profile it lowered with by the ontology store's path and digest in its `RunRecord` (cut `language-ontology`); `profile::` revision documents arrive with follow-up `language-ontology-growth` | The ontology store is revised by commit; a run pins it and refuses drift | Profile authors and accepted rounds; never the lowerer's defaults (`persona-drives-register`) | -| Language ontology entry (cut `language-ontology`) | Key `onto::`, `n` from 1 per language; type `weksa.ontology_entry` in `data/ontology/ontology.cc`: the language, the inventory revision its conditions are read at, a layer (lexicon, grammar, discourse, phonology, orthography), at most four conditions (a feature id of slot `Stance` or `Discourse` with `AtLeast`/`AtMost` a grade or `Is` an option or flag; slots `Speaker` and `Scene` and community ids join with follow-up `language-ontology-view-assembly`), a choice of 1 to 200 bytes, at most three examples of 1 to 120 bytes, and an `Annotator` | Admitted in all-or-nothing batches through `OntologyStore::admit`, immutable, never edited; retire arrives with `language-ontology-growth`. The run pins the store like the entity store (path and digest, `Drift(Ontology)`), never copies it, because no loop step writes it yet. Cut `language-ontology-seed` converts by drafting, not by parse: a fresh model drafts each language's seed from that language's YAML and the inventory's feature lines with no corpus text, then deletes `data/target-language-ontology/` and `.weksa/cultural-ontology/` in the same cut | Seed: a labelled model (`Annotator::Model`), no operator gate, never Test evidence (`ja-ontology-owner-answer`); attestation (operator, native reader) appends later. Accepted rounds later, through a mint call from Train items only. Selection is `ontology::select(ontology, language, view)`, called only inside `Prompt::lower`, so no caller picks entries; the prompt prints each selected entry's id, layer, choice and examples. Its text is minted text for the leak diagnostic, which today has no runtime caller (follow-up `leak-diagnostic-in-round`) | +| Target profile | Locale or language-project id (`en-US`, `ja-JP`, a conlang's slug). Until entries grow in rounds there is no profile revision document: a run names the profile it lowered with by the ontology store's path and digest in its `RunRecord` (cut `language-ontology`); `profile::` revision documents arrive with follow-up `language-ontology-growth` | The ontology store is revised by commit; a run pins it and refuses drift; it likewise pins the speaker-state store (`RunRecord` keys 17 and 18, `Drift(SpeakerState)`, cut `language-ontology-scene`) | Profile authors and accepted rounds; never the lowerer's defaults (`persona-drives-register`) | +| Language ontology entry (cut `language-ontology`) | Key `onto::`, `n` from 1 per language; type `weksa.ontology_entry` in `data/ontology/ontology.cc`: the language, the inventory revision its conditions are read at, a layer (lexicon, grammar, discourse, phonology, orthography), at most four conditions (a feature id of slot `Stance`, `Discourse` or `Scene` with `AtLeast`/`AtMost` a grade or `Is` an option or flag; a `Scene` condition reads the relation of the item's speaker to its addressee that `with_scene` selected; slot `Speaker` and community ids wait on follow-up `language-ontology-communities`), a choice of 1 to 200 bytes, at most three examples of 1 to 120 bytes, and an `Annotator` | Admitted in all-or-nothing batches through `OntologyStore::admit`, immutable, never edited; retire arrives with `language-ontology-growth`. The run pins the store like the entity store (path and digest, `Drift(Ontology)`), never copies it, because no loop step writes it yet. Cut `language-ontology-seed` converts by drafting, not by parse: a fresh model drafts each language's seed from that language's YAML and the inventory's feature lines with no corpus text, then deletes `data/target-language-ontology/` and `.weksa/cultural-ontology/` in the same cut | Seed: a labelled model (`Annotator::Model`), no operator gate, never Test evidence (`ja-ontology-owner-answer`); attestation (operator, native reader) appends later. Accepted rounds later, through a mint call from Train items only. Selection is `ontology::select(ontology, language, view)`, called only inside `Prompt::lower`, so no caller picks entries; the prompt prints each selected entry's id, layer, choice and examples. Its text is minted text for the leak diagnostic, which today has no runtime caller (follow-up `leak-diagnostic-in-round`) | | Community (cut `language-ontology`, follow-up `language-ontology-communities`) | Key `community:`; type `weksa.community`, converted from `data/cultural-ontology/cultures/` and `referents/` (YAML deleted in the same cut) | Revised by commit; a speaker names the communities it belongs to by id | The cultural ontology's author (owner-attested evidence only; Discord-only content stays deleted, `discord-only-content-deleted`) | | Speaker (cut `speaker-scene`) | Key `speaker:@`; type `weksa.speaker`: stable axes as feature values of slot `Speaker` (origin, socioeconomic standing, age cohort, education and whatever the loop adds), at an inventory revision, and an `Annotator` | A new revision appends, never edits; a corpus item and every prior turn name a speaker slug, and a round records the speaker revisions it read | The annotator it names: `PersonaSnapshot` (a projection of a `gamecult.persona_state.v0` document, naming its source document and `updatedAt`), `Operator`, `WorkMetadata` (a public-domain work's own cast list and narration), or `Model { family, generator }` (the 2b `ModelFamily`) | -| Scene (cut `speaker-scene`) | Key `scene:@`, the item's split group; type `weksa.scene`: participants (speaker slugs) and, per ordered pair, relation features of slot `Scene` (relative status, intimacy, in-group, audience and whatever the loop adds), at an inventory revision, and an `Annotator` | As Speaker. Keyed by the group, so a scene never straddles the split | As Speaker | +| Scene (cut `speaker-scene`) | Key `scene:@`, the item's split group; type `weksa.scene`: participants (speaker slugs) and, per ordered pair, relation features of slot `Scene` (relative status, intimacy, in-group, audience and whatever the loop adds), at an inventory revision, and an `Annotator` | As Speaker. Keyed by the group, so a scene never straddles the split. `data/original/speaker-state.cc` holds Model-annotated keeper scenes, one per Train or Validation group of both languages (`weksa scene put|dump`); a Test group has none and reads `NoScene` | As Speaker; a model-annotated scene is never Test evidence | | Corpus item | Key `item::`, `n` canonical decimal >= 1 (no sign, no leading zero, from cut 2b); type `weksa.corpus_item` with text, language, speaker, split group, prior turns, source. Source `ModelWritten` carries a `ModelFamily` enum, the generator string as the fixture wrote it, and a prompt digest (cut 2b). The item carries no speaker state of its own: its speaker slug and group name the Speaker and Scene documents | Admitted once; the whole item is immutable (re-adding an identical item is a no-op, any difference is refused; a correction is a new item that `supersedes` the old); text already held under another key is refused unless the new item supersedes that key (cut 2b); withdrawn, never deleted, so old scores stay explainable | Importers and `weksa corpus add`, under `corpus-cast-plus-public-domain`; the source enum has no variant for Discord or any training-forbidden source (`corpus-clean`) | | Split assignment | Derived: `split(scheme, item.group)` | Fixed for a scheme; a new scheme is a new constant, never a reassignment. A model-written item whose group derives test is in no split; a superseding item keeps its predecessor's group | The hash function. Nobody assigns by hand, so there is no writer to forbid | | Loop round | Key `run: