From 44b66f94dc57748ca7ce5cb8698263b42daf36f6 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 05:29:01 +0000 Subject: [PATCH 01/21] validation-growth: 42 blind-written Validation items and scenes (35 per language) (#75) --- crates/weksa/tests/original_corpus.rs | 162 +++++++++++++++++++---- data/original/corpus.cc | Bin 28720 -> 45018 bytes data/original/speaker-state.cc | Bin 19417 -> 32508 bytes docs/targets/interlingua-lowering-map.md | 67 +++++----- 4 files changed, 176 insertions(+), 53 deletions(-) diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 99e7e30..a3eae63 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -1,6 +1,6 @@ -//! The original corpus (`data/original`), in en-US and ja-JP: per language, 28 +//! The original corpus (`data/original`), in en-US and ja-JP: per language, 49 //! passages written for the non-memorised golf-loop target (Train and -//! Validation, all model-written) and 8 public-domain Test sentences, each with +//! Validation, all model-written: 28, then 21 more Validation items grown blind) and 8 public-domain Test sentences, each with //! its full provenance. Test is only read at a checkpoint: no round or recall //! prompt over this store holds it. The ja-JP passages are written in Japanese, //! not translated; the ja-JP Test sentences are Aozora Bunko excerpts. @@ -32,7 +32,7 @@ fn stores() -> (CorpusStore, EntityStore) { fn every_original_item_records_its_author_and_the_public_domain_ones_their_provenance() { let (corpus, entities) = stores(); let items = corpus.all().unwrap(); - assert_eq!(items.len(), 72); + assert_eq!(items.len(), 114); let (mut model, mut public) = ([0; 2], [0; 2]); for item in &items { let lang = PANEL.iter().position(|l| *l == item.language).unwrap(); @@ -43,7 +43,8 @@ fn every_original_item_records_its_author_and_the_public_domain_ones_their_prove model[lang] += 1; assert!( item.id - .starts_with(["item:original:", "item:original-ja:"][lang]), + .starts_with(["item:original:", "item:original-ja:"][lang]) + || item.id.starts_with(["item:vgrow:", "item:vgrow-ja:"][lang]), "{}", item.id ); @@ -116,7 +117,7 @@ fn every_original_item_records_its_author_and_the_public_domain_ones_their_prove item.id ); } - assert_eq!((model, public), ([28, 28], [8, 8])); + assert_eq!((model, public), ([49, 49], [8, 8])); } #[test] @@ -140,7 +141,7 @@ fn the_original_items_spread_over_train_and_validation_and_test_is_public_domain }; assert_eq!( (of(Split::Train), of(Split::Validation), of(Split::Test)), - (14, 14, 8), + (14, 35, 8), "{lang}" ); } @@ -165,6 +166,85 @@ fn the_original_items_spread_over_train_and_validation_and_test_is_public_domain } } +/// The 42 Validation items grown after the 56: written blind, one new group +/// each, every group bucketed Validation by its name alone, none sharing text +/// with a Test item in either direction. +#[test] +fn grown_items_hold_no_test_text_and_every_group_buckets_validation() { + let (corpus, _) = stores(); + let all = corpus.all().unwrap(); + let mut grown: Vec<_> = all + .iter() + .filter(|i| i.id.starts_with("item:vgrow")) + .collect(); + grown.sort_by(|a, b| a.id.cmp(&b.id)); + assert_eq!(grown.len(), 42); + let test: Vec<_> = all + .iter() + .filter(|i| SplitScheme::V1.live(i) == Some(Split::Test)) + .collect(); + assert_eq!(test.len(), 16); + let mut groups = std::collections::BTreeSet::new(); + for item in &grown { + match &item.source { + Source::ModelWritten { + family, generator, .. + } => { + assert_eq!(*family, ModelFamily::Anthropic, "{}", item.id); + assert!( + generator + .as_deref() + .is_some_and(|g| g.contains("claude-sonnet-5-5")), + "{}", + item.id + ); + } + other => panic!("{}: {other:?}", item.id), + } + assert_eq!(item.speaker.as_str(), "entity:keeper", "{}", item.id); + assert_eq!( + SplitScheme::V1.live(item), + Some(Split::Validation), + "{}", + item.id + ); + assert!(groups.insert(item.group.clone()), "{}", item.group); + assert!( + all.iter() + .filter(|o| o.id != item.id) + .all(|o| o.group != item.group), + "{} shares its group", + item.group + ); + for t in &test { + assert!( + !leaks(std::slice::from_ref(&item.text), t), + "{} holds Test text", + item.id + ); + // The reverse: the Test item contains the new text. + assert!( + !t.text.contains(&item.text), + "{} is inside Test text", + item.id + ); + } + } + let lines: Vec = grown + .iter() + .map(|i| format!("{}\n{}\n{}\n{:?}", i.id, i.text, i.group, i.source)) + .collect(); + let parts: Vec<&[u8]> = lines.iter().map(|l| l.as_bytes()).collect(); + assert_eq!( + weksa::split::fnv1a64_of(&parts), + DIGEST_OF_THE_42, + "a grown item changed" + ); +} + +/// Taken over the 42 grown items at admission, 2026-11-13. +const DIGEST_OF_THE_42: u64 = 15237292705431734093; + /// The 28 model-written items are byte-for-byte what they were before Test /// was added: id, text, group and source, folded in key order. #[test] @@ -460,10 +540,10 @@ fn the_recall_probe_covers_every_live_original_item_and_names_no_work() { }; assert_eq!(reading.languages.len(), 2); for r in &reading.languages { - assert_eq!((r.n, r.answered), (28, 28), "{}", r.language); + assert_eq!((r.n, r.answered), (49, 49), "{}", r.language); assert!(r.chrf < 25.0, "{} {}", r.language, r.chrf); } - assert_eq!(model.0.len(), 56); + assert_eq!(model.0.len(), 98); // No item outside Train and Validation is shown to the model. let all = CorpusStore::open(&c).unwrap().all().unwrap(); for item in &all { @@ -544,14 +624,14 @@ fn original_round(family: ModelFamily) -> (weksa::run::RunStores, weksa::run::Ro fn own_family_counts_and_labels() { // The original corpus is Anthropic-written items, all offered. let (run, r) = original_round(ModelFamily::Anthropic); - assert_eq!(r.own_family(), 56); + assert_eq!(r.own_family(), 98); assert_eq!( r.parse_reasons().iter().map(|c| c.count).sum::(), - 56, + 98, "every item was refused at Parse and says why" ); let line = run.round_line(&r); - assert!(line.contains(" own-family 56"), "{line}"); + assert!(line.contains(" own-family 98"), "{line}"); // A different answerer family, or none recorded, labels nothing. for family in [ModelFamily::OpenAi, ModelFamily::Unrecorded] { @@ -664,6 +744,20 @@ impl weksa::model::ModelPort for Record { } } +/// A Test item's text, or any run of five of its words (twelve characters of a +/// text written without spaces), in a prompt. +fn leaks(prompts: &[String], item: &weksa::corpus::CorpusItem) -> bool { + let w: Vec<&str> = item.text.split_whitespace().collect(); + let c: Vec = item.text.chars().collect(); + prompts.iter().any(|t| { + t.contains(&item.text) + || w.windows(5).any(|k| t.contains(&k.join(" "))) + || (w.len() == 1 + && c.windows(12) + .any(|k| t.contains(&k.iter().collect::()))) + }) +} + /// The public-domain Test items are held out: no prompt of round 1 over /// `data/original`, no recall prompt, and no round record names one. #[test] @@ -717,9 +811,9 @@ fn no_test_item_is_offered_to_a_round_or_recall_or_recorded() { let RoundOutcome::Recorded(r) = round(&mut run, &mut model, &scorer).unwrap() else { panic!("answered"); }; - // The check bites: the round did show live items, 56 of them, and the + // The check bites: the round did show live items, 98 of them, and the // probe sees a shown Train item of each language. - assert!(model.0.len() >= 56, "{}", model.0.len()); + assert!(model.0.len() >= 98, "{}", model.0.len()); for lang in PANEL { assert!( items @@ -740,7 +834,7 @@ fn no_test_item_is_offered_to_a_round_or_recall_or_recorded() { panic!("answered"); }; for rr in &reading.languages { - assert_eq!((rr.n, rr.answered), (28, 28), "{}", rr.language); + assert_eq!((rr.n, rr.answered), (49, 49), "{}", rr.language); } for item in &held { assert!(!leaks(&recall.0, item), "recall prompt held {}", item.id); @@ -783,7 +877,7 @@ fn ja_test_text_carries_no_aozora_markup() { .into_iter() .filter(|i| i.language == "ja-JP") .collect(); - assert_eq!(ja.len(), 36); + assert_eq!(ja.len(), 57); for item in &ja { for bad in ["《", "》", "|", "[#", "※"] { assert!(!item.text.contains(bad), "{} holds {bad}", item.id); @@ -810,7 +904,7 @@ fn every_cited_speaker_has_a_form_in_its_items_language() { } /// The panel [en-US, ja-JP] over `data/original`: both languages are read, each -/// with 14 Validation items, and no Test item is in any call. +/// with 35 Validation items, and no Test item is in any call. #[test] fn a_panel_round_reads_both_languages() { let (run, r) = original_round(ModelFamily::Unrecorded); @@ -821,7 +915,7 @@ fn a_panel_round_reads_both_languages() { let refused = &l.validation_refused; assert_eq!( refused.parse + refused.admit + refused.lower, - 14, + 35, "{}", l.language ); @@ -941,24 +1035,46 @@ fn committed_scenes_cover_original_groups() { .filter(|i| matches!(SplitScheme::V1.live(i), Some(Split::Train | Split::Validation))) .map(|i| i.group) .collect(); - assert_eq!(live.len(), 56); + assert_eq!(live.len(), 98); let held: BTreeSet = scenes.iter().map(|s| s.group.clone()).collect(); assert_eq!(held, live, "a scene for each live Train or Validation group, none other"); // Revisions append: a group's later revisions re-derived its relation. assert!(scenes.iter().all(|s| s.n <= 2)); + const S7: &str = "claude-sonnet-5-5 (Weksa Hands, model-annotated original scenes)"; + const GROWTH: &str = "claude-sonnet-5-5 (Weksa Hands, model-annotated validation-growth scenes; annotator is not the writer)"; + let grown: BTreeSet = CorpusStore::open(&data.join("corpus.cc")) + .unwrap() + .all() + .unwrap() + .into_iter() + .filter(|i| i.id.starts_with("item:vgrow")) + .map(|i| i.group) + .collect(); + assert_eq!(grown.len(), 42); + let mut by_growth = BTreeSet::new(); for s in &scenes { match &s.annotator { Annotator::Model { family, generator } => { assert_eq!(*family, ModelFamily::Anthropic, "{}", s.group); - assert_eq!( - generator, "claude-sonnet-5-5 (Weksa Hands, model-annotated original scenes)", - "{}", - s.group - ); + assert!(generator == S7 || generator == GROWTH, "{}", s.group); + if generator == GROWTH { + by_growth.insert(s.group.clone()); + // One relation, keeper to unnamed, the three Scene features. + assert_eq!(s.n, 1, "{}", s.group); + assert_eq!(s.relations.len(), 1, "{}", s.group); + let rel = &s.relations[0]; + assert_eq!(rel.from.as_str(), "entity:keeper", "{}", s.group); + assert_eq!(rel.to, weksa::speaker::Party::Unnamed, "{}", s.group); + assert_eq!(rel.features.len(), 3, "{}", s.group); + } } other => panic!("{} is {other:?}", s.group), } } + assert_eq!( + by_growth, grown, + "the growth scenes are exactly the vgrow groups" + ); } /// The sweep both seed-pair tests share: over a grid of grades per feature (each diff --git a/data/original/corpus.cc b/data/original/corpus.cc index 9c53141206b52a394c56f05303a5e2b7230c6d54..6e8b471ab8e2b0afbe32cc205d98cd4813c4cc47 100644 GIT binary patch literal 45018 zcmdsg+jARtmR`JDS*L2NlFh>=*-E9h$FsYZ)}n+f0m_rz(#_+m?6EXO6{87hy_%O5Y5j-ZV`pxSNGVlZ zAC;9{LH!_^molBN-08RmZ|%HHd*|clRNOryZWlAsa~q%Qc-wR9tP9?e6g4Slovy*H zaal&nL~zY>8-MoxzdhFZtB(HXf1sSaMW1$KLQcozLVjSExY6M=-SK&Rg9BdgfY-Ov z>)+@JZ1fMl_sie-c&hWw+P3Po+3Ni5^4Otru~ZqqTst^kxpcc+ysqDzTz)i#7mc%H z<>KM;*g@myk#g}Se~pXJl#AEO#qsjk;qvaWtGmR3SKqoRW%J4W=*C=ZK$TS~mAupZ zoC6U#8cm4Vfx)B28*ddN>11@PoDq}RGurd|1>EmSxp=uewx>LHLRhm&{<*A(!sbFY zB@0nei3y>gZ|y^Uda^QAG9KKjTwD~^Y!~xtAtMPp_*bpfHYW13e5tQ~QM0&x$wfy{aFY z5-R5}>U$4V4=q-%A6_RImo6LgR|MnYqH+6k6m9#wUtPdC)E114bA?OjLk8()`Z-_{7Y#AY+knn4~VZ5G~vc%!FkFs)B z64uB{ayXe4)8-8&RT6RvmSgR&o>gX>AMCIH!_iLO`Z@{Tg|`nodF#_o=rqGjbJ0R`y*QfEy^1U;ayW{!^tjsQCwYo~|q|@Q*GnFCALGf4^Lusa!AO-{t!g{4Fk+W$s>V z?1F{nE=Y~aDRHS~6Qtx(aUhZ0l~j@gfsT#vgWBAuwf&z|i>IBoK+xx1`<1bHur_(Q zdgzw0=9QlpGD+bz`W3Fo?+JP=^5ae)_&mX|-z5LP??Hb5(BFmpjZfK!`JbIBj{(i2 z<>EI_ocW%v%=ZsHR_3SsG2ic+`JZ$$->-cHmQ;4%s2%`g=RkE>;L5{UW@D*V{6s$j zGB1F}Y@8>{WBYJ{{>>i!(V1HD6un41$6mk%yb*R^D38I2-KbnYRvtTEIRwJPwqN8M zUN0A6{9&4#^e#2&Ef0&yR(9vbl&DIPLRK2^cC!12jnD1G?m*BN2(GEqCK++T` z8y~(w>6DzwiK^PL<2sD{t;4l*`<^tF;gGr+ai8z8Qu*KXqjJDCl|SmFazNW&9$PB! zxdguIdrpyc9y?UMKUX{TRk=6;z?rO|CTuq`wQ?8c9_&2FjGTetHVz&plg)f>*x%bt zf{F?pUos;;Y%#%!WL}OJRB<3H4@3$v*})D!zr3{DIPeWNO7+;~>d|p0zWTi0wGWN+ z2ld4-jrnV}J)htr>$eFz#grtwv&p`o#ai4CHaQUTnVg;PVUvCSZk(;&od#yL#jE<* zvE_&7p+g#MtsS^yoS)G@Ij2u9H7LtYTqU&-d(x!!b+^Ah|6?WX?tY{Vx*}}|iupz- zX@lA?YM;-5pXK5gEGB03qxYDoP{Yv2*Fef})y$y-H+l--qFY+-mbQOL+nLfdC7&OoH%W9ZY^MB~Qk zvz4R!^!sO=2p;fJ-y7)TGbJHEVhL_Z21&+3<oD|g5A^DxSIFRalq zD@<%r5Ut!5cO{dzTezD>%#$7Pw@c$k#_Wq3A-Xa*#C$Cfw--6M8;=B!Z=k4QF6?^;fHsqSOzvwj3AuS5)W*qxM z|LOt&Cz$6EV3drd%Va~xjBTsjrJ$V-tZ4kKA@K%o>u01;PgsB@I3U{ThC;V8o#jbwT4ZEi9cqetk+BPUpGm?Y4 zmdN=VI>5Ag@+7yHWnE4Hv+(&y&GBa13f!bn=4o(&F;xPR7-4=E# zqMDG&+Il<4`Gdx9bmXKjv{w7W>Y=OHVOZVedxw|5K8y?NzeIR@w2+n6LP~O_fy1GC zW%gTZ8rQ)o!;<9pKqhr17&=8s_mWH zzuWeqlocYQoGpxt>2xwXEXZ+T1kr^u5L4uwDu~v1IXN#RRRO8kG({eiUyRFg8u`Yk zoI&O?UP!0uz0!xt^g8^D8zwdunJq+W44FuYz?hLO=m?stQfD?~ORF&o;D8>auP&wrHH&b05E zY%~c_gglbB{F}3CTEt0+@X~tW4e=x44PXsOIo0j)U{}4R$h+`|3qP*^Mt-YMQ1J;z zlE{mL`^mh@e*}&_l$3y*A)PN}qmqyj^H^?&?6gM<+q2V@hUR>8$Rq43Aasr;8Ev4} z_K`~UPx>Dp&`15%vwzkkv4?WYb}IHvEZa!@OdE-P+MmC|#2k^pKdeSV{7@29P}1bB zkjx6BazPQYQgS#Ek(Goj$JWt>!%18iDImL>jbaDLoS%uOWf8c8A(E)%6Ba{nH5p20 z43b#5SvSffVmbyEcc#t;-sPWwmH_R|tgsXJ+*-(w3U9`RElF$=VQW&AK_RSkR9bvl z$||G6R`J7R443ln(0ne*6l5+1nMxz=RBE1zga=9YJ}IgUxu%Ns=P`ZhGd*lOm)Mpa zro)fMvBaPBq)yMwY%6u1dDuoBzqVnEoXNm2i2`&fcoC!TP>aJKou*s^X}+QFw{ZcXmihQc@S+7Sp+eKtz#+ZRV}qT9BbYfL0y< z>AURVm9gyBA80N+K2yJC_x4=&o>|P+W&e$T*tYBe?XNarU9-=tLRQQ}?61QCURD(( z*hv^wG9i)@fu~S}5lM=v?CTWsR4py#1tcd0RT8o6Vg#OmeI;8AoUWM+ib-l-kxj2E z(@ibnv;@%+F)C*l>a^kS95mhYWHe0X(8t*#<8w-Bq_0f_LY4&?cQHNxk^&Ck6y6gCvfA#EZ zJ#A0V99=7Up3&OKGpK#A8Bz>30kSfV^%t$9dR0v4>)Mo+$BPZ!hKn=n1muTt`cL9Z z7?BHUI_D?d4XWWRU^pv6E#@G{E%vk=_+=p2OLap--*jq~S;&*up=Oo*=*?H9#e z1*jk44P31ax+0A~FytZL@PGRmlyHa=#ZSth47#QR{e31s@qzl-lV`W|q(sm3myHsC z(?*Gq_BO11mOaR>0=8B{gibFgS?pqx?XVCLta%cIA$vaz`O05_7D6G*@1T&2UrP>W z;SRD6HY7*!LnCRcP!t0BKj3SRsBYl12ya_IzWNru%cUYgWe&wO^dDaT&)Q5!vpmNi z2n(CHY_r*p%{ZG7Gtla<$|F)fZ?4MFlcb6J(5CGpEwp{sUp@O~Pipi`FWIPpLMdBl zAJ*RA4jZB7B!q*i8QF-r9P9(QfzWhRP}A~AdUTyYg*{>%Q5yrNHW`qZOlrcPG7SGdqKB(lPihHZ0#^VVrff4RN!wA$Iy-3 zGRNl&6LavNzLw*OYFYP`F#kTwxAm8@?Y84KFLS*R%Y8*);Jsfy^Zb;Ne(BB#6j0B0Bb zY8Mwx)GMdV`kn>jQ=Gybr?a?Tpx&vY05$+r981V4X?39Lu9}rl-Rgv@S5x$*F+Qs) z)z0lT(Zk_&bvM3V(C?wJ^^9?mvgS-oWOqSuFc(ZeND1NH=lB(H7Da)afTuT>s$VXe zHJEs#aSDe;IJDv7*OpUFJFjg^kc=jh5s@ojR{^nm=Fb`h#k2&!>$C(uBKFsnuV5Qs z13+WIbzR?oouHeXFb|H27VzxFbW1!!9N;s~8j@YyAOK~;YEwuHhn;}XhH7H}|RJV4Rk zAs}1(6m-WS5zgZA3eGN@XG1+;MG8}5S-%Qk?U}f-!Mec-)`0e2xp=6$7r+6Xv)1|9 z*uh%ylCc-$Mco+)JBJeDx^J6j#HsWQN5V{S*bfjK#cL?rS*YBAE$`v&zG~$!Ri+_f zj0!YT&Z_|Jo_P)%+FP8^4r&D)U>nmYk|Gy}G7+=Bq`JGvC!JKYYAnLDk+~&naR>GQ z^lh4FILX?D&#@xLrei;_Fgn4)%I_dFcV(r60uV z&DD#7!TLk`9V|i*+hP}ZbmW{WtPZ;O48v_;zu*LRNPFKvW&79!dOuiHuxo>%!Fm4J zJg)#Yz&<+g|K=`UtX`Sqk`r8Zy+-l_1_hQI8vwN`D3OI_!H=f+^}}8Kqm9;Rj#(vA zx@S~tL;4>(AsyEKv@-KhpPUA$wf!I{&;@0=nvx5L5w7T0A*D;zy(qd|&>wu&60vl1 zinm%#L`p*a;Oe05kgK?aq{hE=0&_^)uI~luN?dJicDwM`2uhYH2*U||%VP73K zz~?TCCOH3z6P#XcGv|4DXA|F8($W`Sl#6gnbIe0Hq?^^FNd8=|T!26}b3V{zSOi!5 zezgbzpSz7uX#T(nO`o>2uB%G=0|dK}O~tY03r^EVea8_-v4&Hs;-jF?>;#GM^5Sz_hunMQXqH2CPt8?egPL;|cWp7C;P*Q6O-i-QD z&%!W3{liG0k7cfcaR*})IOL@(@9&|sde`)>%_<-PvukCd+Zl1W69#k=@%s)DmtWgy zC0fYyo8c=2oU;sl)~Qy(%=Mv?)qck^<#)3umLrv5UjuB0BZ=7rL1#z9Q}?}K1YB%SG`TDfD?y= z=JL`(*1-#ARf)0qxp~&Weh)_v<6UH~&D;*J4s+{HC=i++2QmJUfTbbxP>(w%sCm8w?SX4xozaFk<`l`ut{8b@|eh5@;qOed!ZE^~YS9z+bHG3jZY$ysbl#HPkaim_EY!80t8zPJ@ z=>gdLU5m(eNa=V>4Oxz6nSF=SF>X$~*#=Y3R5zD!di)!G-uJ#WraIoL{ozXl2^FxR z3=<`qXt*?+Nl^HYx;0fm)n&@6+qR?A+{e~CS1dq1lh>^P{jUyyyjtKDDsN5*nNbi0 zg{)?MJ6dR@>@Z+!;Dw4(s&M6YQ%AgRG}weQ>lKSt&(w4)R{x~~E1&klb{uHLMgc+= zEk$Dm4*i zP%jFgWI@X0(0@sYMMRZKq0vgODDsFSQccs8!2VsaK=lj>TY>tU4xj?s4+$s*eXt4! zSE`s~O10Z)UJ~=%eyobaQXgzm$#kz=@Y6NhJtMkSZ2oTtHbL$Ax6)|xLs`9=V#-*2 z;wCQ3o>R-QBR)+fkQQSxNm(K6Id;PCUw->{Pqw?Oo^e?#WS#qNP+R|JXnd5Em}qV? zq7U!6B+9j^wUT^1w;`x$DOYb&v45l{a%xUN)7%O{?q~;4JtM7Fp!5!Ig|t7~Dx;bm z0}&vf*c^dK4V%udkf?H`R?~osS+xDESbTcMPObRVI`9c=YqrW{v0t{ z2&Sj0jwm&G#J+HaVAO7*e(#4T+X9b%>Zu;G{PLM++hZ1=_Da*=jYt^yh^}wmequI7 ztx=}QF)dU>eVUgjvJ9n85ltaq$YlF$O?Ig(1kw)E)w^(~m3AEh6h%90Um=A+{q5ZM zY31H_@$5vCxVI$#q0iY2TYZueO$WRu>EyppLmS#4|Q# z^=>BKwT?V$Do-FqEZI=x`*KtjUZ1DNDKYGJbR#w1=&TpDTgyTJ@l|&Jv`^t>kDV0Z&Yduqx%*6m0Ru?69Of@64|z zNO_QP#e(#A4{v;fW?F%VGYBzEX(9oiQzT?wOc8I&usHG!2&s`?81BO+FmGhV0_8fq z`QIJ;Fr)>xna$}G|G^-k3msF&L^Dy$`z^}bkN{L+vsm8u0Sk@3nH39`+gOKA-P!_f zUl`*btTYrGCCjv%)`~SMzYJRO)D;YzgU2*U$k()_N$Dr7Q~O|vJnBmEm`B^?eLw$L zm-pZPpLTWc*Frn%8jHgo!5ZWilhHq3z*BC_B)qvzX~>2dF*%3XO?yUfUop&U3x@i6 z%LZD1+m2R1+d^o;?_oZpf(dp=&rnEBb0&yKq`+uk1c@>h7U=Rv^ux|T!B^eYYcmOt zd#qI8`Z>!waKC)!x7v{lYOnq@i<}If#zxAKdZ3k*d+d%f-3bpI<)l7mhLJ`V=VaO%q7$@ExcV5#g(&Z}*r4NU3l-GBbj(?{o zNL391JYDaZhy_#gt(sDAN?WKMLB&l8kBy`18$6MTit(sI4^?Hxg%ezCg60bR6onzE zdMZ}NQ8R)%ooRli2ntW|mf6bQQuP*9xbs*I++oHzg(@CAjsf?w+f(;=FL}&0r)Z1$^FkbTID4l={Ku7#=^3;V}ho z6XBk<6t>lQ)e_4s@5P)WZnom;Yc5KqHY+r+Q2`bAxtUXWQ2lGQ4RTE@%79S~kAJD= z${a+%rLXuF|C{Yn&+}!YoB$(vGRdD~9y^jJz3MAnu0rq$tis^XkS7$1$Gt<5_@EdH z_F09V=}p@zq&il?t9{I+l&GOXRTc0*sh-~u$cOO?3$j@sckvl=~xcG zmZV8wRP$-$e`S`+K2bZ)Zey}`x?bmrhDfW^fgflSzk3vf$AA^GRMu=JY>*NSrU1wo z%yeK0TL8U0`h9Abter-gzF8xR%g{Ul@Xf9hYzLn6;cTF6EfMe{-xKtEhS4_k|GWeJ zpq8%eJzJRtnq-f-+jktVP>H@d$k6J3;Kvn{3t=C>=U92qB3UJKdIe2@I$9aWP=F)W zyJ(CEb`%^2`<}7=-M56zPC*{wh__d(%Iet_J)?8mx^%jYK`q{x?qp2tr`d+ZIsE{t z(ou0vZo9T0y(uz-n!sAH^I+#q*pg1FdtwIM62Y)X@ay;ypq{JDan}JqP}XWH zfKawYGkDe+`jeovVBoOV5SPF|_2@2WmT19|;Ng7V-H!}D0X-~w#K@o+lmcEc7?A>j zK6a?@m~g3*nT%NJk_p@V@j784HTCdD(;r+9=AuHmg!f9SAr#_Yd@q{D*r zRRuP!hUZW|4kz^V^De!szHHk^Y~Ma8`;s~1QmUOr9t8t0OrU|XY0s!JP@lYM9OPEk ztB7(;FLj#{ABSzzO9%9muv=yhW|}RWWfZFvZeri@&Y}!kw_aD<%mF;f8!Lm9%=f{z z^&b9YTb!SDERI)8R&FCKLFf(?=bI9XW}yOdYUM7oZ05H45(|kEw;eC&`)-ghqj|Kr z61K_;HTj{`lhwU*^?)Bg?WWmm%Q9MsBqNU}SNZgfnEP=5gLWh4*V2q?2_4})d3GAk z-nfU17+lIj-WG7-uuJ%Nl6DU*1cji+XJ?GrPk5uiS?%GZ0DGa6pYj^9s6LOC!i#lT z;H(2|gKN(_VNF}B1qPghk&<({)D zUHbi}zcB7I0v&!o2*1DkYV8vIK&g5H4&-=qzz0GbcK6leOOyv7%SIlbCP7c2xw=%l zh6y)N0_KqbdB;gc-{_g|IwTq*B4Ti%0R1Ch00T*R)Ks9c=qi(%3h>e1@2 zH4yjvu`k@G08;JC5Ym28_wRQ(dIYS181y!$GGG}N5N&d1pKx#Z6w~QroiL+Vr=eO{ zze89;;}o7-fglriz$;ikBtWk(-^aJJ##y8|aV5MGZi~B}b+cJ8VxaICGBr=%db!U6 zDDA5hMugcSvh*3a>jh)+GFzZ)`dOOE#fxTob%1StF;iQ_R4B{5kq4mur+LWqun{aimJSPr*J0th`5K!#!_%a3>MK2t)0n5fSnl%yKWksC zAx(y_2gmt&BXj0NR7+14FPg|#9?_F7up&61;Yo_6dKzbzhb&@ItOS8E0a@1wMzK5X zkv~RTJ3E}VWM|L%9G3#%DV!|d;8_VA`1hVC(FkK`>gt9m+xxVa-{q2E)ajrp5(QvX z<`hA_PNx2JYAQ$&xRi5*Mgri%=TWB~Lsio{JnffSA7WGq)u#-KZ~5ILM(-0ySO(&L zU2{Vm~ z;id$jDrZsCRA+qs*?5SQGg)c0k0C(^!*H(=BQEXr)26n&3dusH{! z9J>2OA(bseQ|Ldoo;Zj}Bw0Zrn@v^ZDYX0J3Uunxb8u1IJ0ha@505lOr`9kwMLeG# zbu2h+x=8E(tDoQQQWpey$PRfxdma64ptB+%XMjh*QdJ)wGK~H|@`zNGPh*(S!Ua(A zOl!b5kYt*LndBA4}_d;dCaigb~i)d`fphMc)QC;5Z*ts;~ms~LM5|NF-@h! zN}bs`3~@r~I!h72JwlDp=3}0-=4c)i_G2Uu`b+2uocJ0hi5~p?5f;Od<4w@(ziRb^ z?k?*>h6;aVLR#}?G3AS7FsWcvf*J?;b;YFSA{6E$D@fE z*c{sTM9hH0lcHgXs2*6rkxUkE?nCqX@A&S3`=5X^jCmvBs6XKK`u$OFEb8y24Bh7o zUbF)q)^=Dr9M$%4iwtO{kOqc$j68Ww1sx0K%iOh4pBYDtHuWM8S}^+^Xt#jDj<7d7 z8~FCCA9MF%zu%7ekoM|Mpl54&qdp|lN<>EqS~yGU+{%j91GSYxBtpZBVCYqu42 zU?2!aMVgA7-7C#Epz(;ATpzs6-}2q@e#&wEuW?5I2R3fwfc{6#1WFE5GnmwPI5lZu_eip`dqIx9)^vWC?b<+QIc_I0ob$~<}Fqw)U9^IeN_1;HoM{1 z-mzpzUJ^-OklD2c%0klB6Mu+iQpoz{C=#M@NW}zLD3K`E!J4c}rW}?UX%xJ@OPg&I zb@P{fchsMP^=L)iePFrKhI&BTL4GlTRO)C`Hm7k;KxTvNHkN^A)Z!~{&c#D2Gc*Xq zRN!bCL13 zhH)fW#5V|Ql%$z@YOxo#EG+CeQYYL;O#P8at_%;u((`UG#}M};IQ_O&54pR5lEV1S zHpoNTdo77?D=I`ndpMCt^peF2Fnehb3BW~yoI3u^XeX7-Bd3M$(-aqtnwf73NMTtb z^pYd+O|qN!T=mGii!2FwyYvp5dD(n|pt3S3#Cceh{74dGcr3=#I99kuTp^9ZeB;c4 z)`t8hY*AJiR(R5w3%t#nuX?=QMU{lN-M9>CJL-~!A-9Hmq%hkkPpm~6kj%U)jjV%g mp&X#O4d5z?@hsKxnhL4Tkr2ZE;Y1&bn~VG1A^-b7eE9zbGQ0W! delta 14 Wcmcb0pJ~Gb#tG*cJvLrQ$Oiy9xCcc5 diff --git a/data/original/speaker-state.cc b/data/original/speaker-state.cc index c4f563d1a4d37b3448236825ecbb96c147a60095..2503674ddbe09cd0598caeb70e5130c618d4ccd8 100644 GIT binary patch literal 32508 zcmd^|TaP106~{BP`5@LjAR?_-GqaZ@&dc%w2#Fw}aM2|785jsdN74 zRCW2*vx71z25b=W3wa?je&OWc=68QQ=5fw0GG@c7s)q__^2;m-RphjtKYl{fAJH4z3*#JzF7S!J&{Gj zC}l~PrGrppSC{^RJBRy+SGorW-Ghsd?O*O4Jls2Y=-S=CPoF`mo%?S-#nU1xCcT*R zW1ii-wfyft_(Gb>&k)@D&X0~bE6R+^Uv}gBpF1w(-+nvcCp*zolmw?8Wa`Tie%A}zv997lt9js^)U1K!O=n)0H1xqG?u@h>Ufolmng z$j^61N+CaDDhgKcpmV~KC}8S8-W_IQT!fv9M*c`={vMHaqP!#D-YG)f8D~+!vj?7h zp~37VN+@K9j~!g;UA)-azjW>HZ>LWKWKv{FjP=C{2W)C$yWa*sofG?&hS=40i7o#8 zIVg(7A&j%r@Tpi_dZ@R5QNv=_E_!GdUv9F{MUOp;J2)=#aAj0T5^qg^vcTG9d1x6E z^eTGK7l-E&hdxW=l_2Eu_|5dG8idkpKM9;E`Yex96BIC0R3c|A40Egf>S(awNitcR zg%UYqvAGo@#;Xg6OchHjL~6F5L?bFP&Vse0LWwL{^m!u2AVb!*bmwvK(=*(MgB4P- zShJS)mB_M|(kx!z919^vR9w`y^ucR){{}o(Ka(QMxFGB4?#G%ebRNr>D}5PN-2qE@ z8nD$~UPAj|`m?(7E_ANA?P#*>Cb{7bXdp|B88z4e%~Y1mNDph8gM7u>OE}h|Ty{oA z&gOU1pPq)=OlC=qT%o#1bL0GR6&r|hNP2%NTU8=U7GwnNzSgd6g?ofR&{GH4LPH0n zG?u(t7&}K1PgXl1)InFGdyqvY@`9@RFf3rdbfiRuIUCVk)<1K z!5YE!CS+M(l%%zRkpb(?TxJ;?G!JjRG{wUpIy5{iKTr3<2IXsW)Oocwb%l~FN2 zW^#j@#}O4FFg5>s`g}c_fm+KrAy>_g;WXLTRO^>beuFTI<(_O*&EeGwuA za%T`*Hcow|2{B-i1VQnGIqDD7XCc0s&?h-J3-i?`6M0TXm<83{e@>r+Q07WkRNA(! zfm=J7n2ySfbWJdtV5>WqSyEfeG`Wc-iibhX)9~wI! zf=wGHqI8xqvX(b+6C;|@Tr5U1MVAFRyoLD&jpk~tD>7S9;o?bYG?ZI9+^DjS(*3`1 z(|B~#JtySRmCPBnmidBTG(Gw#`np`IBg6jYKhqbWOFtF6BG|T#a$Ve&v>}BB08Obz zQs$GIoBXeFnj$8*6d(??Q6*LPCXESM2hXB0vE9UMfj7`}zFCwG1t-r_LuY_G-?`E< zMo3rPHeaBbVw&v@pebhNthVVgvzr{F=|gfB9T{aAs2+9uBi;GpoZZ^C2?35FAB$-8{IN1*hkwSs&tkC z(t^UJY0C;0i|!>87l_)jnb496X#o9gBhpL^){T~l<;tO%cq>PmjYRdoTN-~e7+4l3$i~|c5CngC6jRPy)_7l;xOl2B~m>DraEz?}N z{Zuqrb|O!)ex%AX6K+2fO{7fZV?c&mb|_Nj%6pSa$%bd_i9=tHsmzqyPeoIaCuKkC zo6bPd(VklCj7-Z04w@12q>Lk(myK#SIuN~(I-}>!>1nxS@^M7^F%3A#LZkJGJVv3U zuDF;EK79=!@Ljyu=_}ppBqgt)g?I5vi7Q(E2BP@Jia1KhV+Az`bknHPY(ELyAH-=n zh!Zl$)N4szj|c~ObD1+9n(VyEK^{K1l&~P+t1n(cw=c|8mdsi(w<2g5j3eMNt{Dx> zSp-dEW4BNxEH%0XQU(X?xaTs<=pbh^rj^NlOAcq41rLMp(D1N~51NPO20LAbvFCwn zPjbc;zIE+E_rNQSWn{EqHieI=$#cX8-wlAiF`^P}KMR~gM^Pr9Rima-unfb8u4XD% zG)!}bFE`tWpE4P&eZTBTi7bhcSw*)ipvx-0h~X$pIS&lX9{OIJnaYwGoqsfyx=qZE zdB49tvzf}085w77yT=%>%q%)Fu*WE~S0YPdbd-6USm1guZ{Ih%ycgc-tTdJ+THuVe zBn)gICSmCTX%gOI0_Wj;Fp*!SwQjYasVw7yULh^Z!+7T|SG7w%5n@6OKF+_;S#qOu zv0*3StL8Vjg`!N(m{vez2z#LwR6Fa zP1fA|;CsuI#ubUSJ$%*3?Z@irGm2OpI%qbSS1MP`{7u@i6O>>2X>d&~a)OoUo@61& zi?Zu9jBXZ}EQGV0vfK6`4PHwHXUDXAhREPlWTvu=AG+QfKKkLS-s5~-glxPr`=I%v z(pd6pLFH;hPk7eX=#kM=sVr&G(PMZk^v0qm3dp{su3b2Kq-N8+{`YhGLXlXrTU8p% zZk3K4&0~9R>WfGM3*3w{AvJ8q1hyK}Knih|<+xN`+R(SZcPP1kSDMQwg%pP`P!c QvZO)Ib4(w~^psow3wAautpET3 delta 197 zcmezKm+|Iw#tBOq-8bK2Oy!*HD=M(~Eg$3L{WSuc^#$$NK#a*xC4Wukvl8AMBP-54 zS>I21Gq*A$<78eR;my&iyiAkrxdkR~(D*aiF-mx|whk-fWLaO~&94oVnZZf~EhU*j zN;a3-nz2mYtSG$M+{GRwC9?U9halr-Vee}^K&_&i?IM{OH`hlCF#+uo*nB` mod 100, under 40 Train, -under 80 Validation, else Test; a model-written item in a Test group is in no -split), and every item so far is its own group (`original-`). Growth is -therefore: about 21 new model-written items per language, each its own new -group, the group name chosen before the text is written (the bucket is then a -property of the name, not of the text), keeping names whose bucket is -Validation, with source `ModelWritten` (family, generator, prompt digest), -a speaker and a Scene as the existing items have, and the inventory and -ontology beds unchanged for the measure. No Test text is read by the writer: -the writer's prompt holds a noun list, the register note and the language, -never a Test sentence or `weksa corpus dump` output. -Costs and hazards, all real: (a) the corpus digest changes, so a run record -pins a new corpus and every calibration number above is redone on it (eight -prompted draws of 35 items per language, about 2.5 times the 14-item cost); -(b) the inventory was drafted with Validation informing it (ja inventory rev -5), so fresh Validation items also measure inventory coverage, and some -will refuse at encode (counted 0) until concepts are added, which is a result, -not a defect; (c) the writer and the lowerers are one model family, so a -lowerer may be favoured by its own prose (self-preference, Panickssery et al. -2024); a cross-family writer for the new Validation items removes that -(question `validation-growth-writer`). -Not built here because the writer fork is open, the scenes and entities for -~42 items are a real authoring cut, and the digest change invalidates the -runs in flight. +Power needs about 35 or more Validation items per language. `data/original` +now holds 35 per language (it held 14; Train 14 and Test 8 are unchanged): +42 model-written items, `item:vgrow:1..21` (en-US) and `item:vgrow-ja:1..21` +(ja-JP), live Train+Validation 56 to 98. The slug is new on purpose: the +`item:original:` prefix filter of `DIGEST_OF_THE_28` still selects the first 28 +and passes; `DIGEST_OF_THE_42` freezes the new prose. + +The split is derived from an item's group alone (`SplitScheme::V1`: FNV-1a of +`weksa-split-v1:` mod 100, under 40 Train, under 80 Validation, else +Test). The names were chosen before any text existed: 90 candidate nouns per +language, dropped by script when the group already existed or the noun +occurred in a Test text, then the first 24 whose bucket fell in 40..=79 (the +bucket came from `SplitScheme::V1` itself, cross-checked on the 72 existing +items). Each language was written by three blind writer agents (8 nouns each, +a brief of language, register note, noun and speech act, no Test text and no +existing item), 24 texts per language written, the first 21 in name order +admitted through `weksa corpus add`; each group is its own group. The scenes +are `weksa scene put` documents (keeper to `unnamed`, inventory rev 4), +annotated by a separate agent; a second annotator measured agreement only and +is not committed. + +The writer is Claude (family Anthropic recorded): no cross-family generator was +reachable from the body that ran this cut. The self-preference effect (a Claude +lowerer favouring Claude prose, Panickssery et al. 2024) is therefore not +measured. The writers' blindness is by brief content, not sandbox. + +Expect encode refusals: the existing 56 were written onto the Dao r3 concepts +and the inventory was drafted from them, while the new items were written free +of the inventory (question `validation-growth-vocabulary` stays open). Some new +packets will be refused at encode (inventory coverage) and score 0 as the means +do; they stay in the corpus. That is a result about inventory generalization, +not a defect. + +Every calibration number above (92/0/8 prompted rates, the 29% power, the 0.079 +noise term) is for the 14-item corpus; the corpus digest changed, and the +numbers are redone on the grown corpus in a separate recalibration step. ### Smallest measurable distil slice (`distil-incontext`, not built) From 76fa6856b2d939a99c0890a6137e58fd4cfb0916 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 07:33:17 +0000 Subject: [PATCH 02/21] map: recalibration on the 35-item Validation corpus (#76) Co-Authored-By: Claude Sonnet 5.5 --- docs/targets/interlingua-lowering-map.md | 74 ++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 4 deletions(-) diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 0182cc0..3fc873c 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -406,7 +406,7 @@ of 1120 trials, NoGain 71%, Regressed 0%: a net gain of 0.079 against a mean noise term of 0.079. At 14 items per language the rule is therefore a low-power filter for a gain of this size; by 1/sqrt(n), about 35 or more items per language would bring the noise term near 0.05 (an extrapolation, not measured). -### Validation growth (`validation-growth`, built; recalibration not run) +### Validation growth (`validation-growth`, built and recalibrated) Power needs about 35 or more Validation items per language. `data/original` now holds 35 per language (it held 14; Train 14 and Test 8 are unchanged): @@ -441,9 +441,75 @@ packets will be refused at encode (inventory coverage) and score 0 as the means do; they stay in the corpus. That is a result about inventory generalization, not a defect. -Every calibration number above (92/0/8 prompted rates, the 29% power, the 0.079 -noise term) is for the 14-item corpus; the corpus digest changed, and the -numbers are redone on the grown corpus in a separate recalibration step. +Every calibration number in the Distil measure section above (92/0/8 prompted +rates, the 29% power, the 0.079 noise term) is for the 14-item corpus and stays +the stated baseline of that corpus. + +Recalibration on the grown corpus (`runs/growth-base`, main 44b66f9, panel +en-US and ja-JP, round 1 recorded by briefed subagents; 6 prompted draws +including the recorded one, 2 mild degradations, an all-"Yes." control and 4 +Train-guide draws by Claude sonnet-5-5; the same decision rule at FLOOR_Z 3 and +3-arm floors; 120 prompted trials from the 30 ordered pairs of 6 draws with the +other 4 as floor candidates, so trials share draws and are not independent). +Soul pass `cut-validation-growth.s3` re-derived every number below; its +findings shaped the wording. + +- Live Validation items: 28 en-US and 32 ja-JP of 35 each; 10 are refused at + parse (en-US original:18, 23, 25, 27 and vgrow:4, 5, 6; ja-JP original-ja:17, + 22 and vgrow-ja:2); `relower` skips an item with no packet, so every reading + here is over live items only. Old 14: 10 en-US and 12 ja-JP live; new 21: 18 + and 20. +- Prompted repeat: NoGain 96.7%, Accepted 3.3%, Regressed 0% (14 items: 92 / 0 / + 8). All four false accepts are one pair, base p6 and candidate the recorded + lowering R: R comes from the round, the other five draws from `relower`, and + in en-US R scores +0.024 per item above them [+0.002, +0.052], so R is not + exchangeable with a repeat (a Plausible Soul finding, cause not isolated). No + false-accept bound can be drawn from 6 draws. +- Degraded arms read Regressed: all-"Yes." 100% (as before); mild-1 31.7%; + mild-2 0% (14 items: 100% and 75% on a subtler arm). The mild arms are weak + and not independent: they were rewritten from R with one detail blurred, the + original wording is not on disk, mild-2 equals R word for word in 5 of 60 + items and mild-1 in 2, and mild-1 equals mild-2 in 27 of 60 although they + were asked to blur different details. Their mean composite is 0.003 to 0.015 + below the prompted mean (the old subtler arm: 0.037 and 0.052). The + sensitivity to a small regression is not re-measured. +- Noise: the panel noise term of the sum rule is 0.045 on average over the base + and floor choices (0.031 to 0.064), against 0.079 at 14 items. The 1/sqrt(n) + forecast (about 0.05) assumed per-item variance stayed the same; it did not: + the per-item SD of the difference between two prompted draws is about 0.085 + on the old items and 0.034 on the new ones, so the pooled term landed near + the forecast because less variable items were added. Not a confirmation of + the forecast. +- Power: the Train-guide arm (14 Train sentences as a register guide) is + Accepted in 21.2% of 240 trials, NoGain 78.8%, against 29% at 14 items, and + its mean composite gain over the prompted draws is 0.0185, 0.41 of the mean + noise term (0.37 on the first floor choice per base); with `relower`-only + bases (R removed) it is Accepted in 35% (28 of 80). Split by item class + (paired bootstrap over items, 95%): old items +0.029 [+0.005, +0.052] (en-US + +0.049 [+0.023, +0.084] on 10 items, ja-JP +0.013 [-0.021, +0.037] on 12); + new items -0.002 [-0.011, +0.007] pooled, but two opposite effects: en-US + -0.017 [-0.027, -0.007] on 18 items and ja-JP +0.011 [+0.001, +0.022] on 20. + So the old-versus-new contrast exists in en-US; in ja-JP old (+0.013) and new + (+0.011) do not differ. Measured: the 14-item gain does not carry to the new + en-US items. Inferred, not isolated: that it was in-sample register fit. The + new items differ from the old in several ways at once (written free of the + inventory, longer, a different scene convention, lower scores, a smaller + vocabulary overlap with Train in en-US: 0.29 against 0.52) and were briefed to + the same register as Train, by Claude as the old were. The 29% power figure + stays the stated baseline of the 14-item corpus; no better arm is known on + the new items, so the rule's power against a real gain is not re-measured. +- The two classes are different strata (Soul s2 `register-stratum-shift`): the + new items score lower (composite 0.41 against 0.64 pooled) so a pooled + 60-item mean mixes them; report old and new separately. +- The self-preference effect is not measured (ruling + `validation-growth-writer-claude-checked`). The new scenes use a different + `in_group` convention from the old ones (follow-up + `vgrow-scene-in-group-convention`), so the new-item scores are for scenes that + will be re-annotated; the corpus digest changes again then. + +Evidence lives in the Self workspace (`cal2/`, `runs/growth-base`), not in git, +and uses a DUMP patch kept out of the repo; no committed harness reruns these +numbers (Soul finding `evidence-outside-git`, follow-up owed). ### Smallest measurable distil slice (`distil-incontext`, not built) From ba42639c3bba281226a8694b58d0d77c1ee2561c Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 08:24:45 +0000 Subject: [PATCH 03/21] scene-in-group: rubric r1 and 98 labelled scenes (#77) Co-Authored-By: Claude Sonnet 5.5 --- README.md | 4 +- crates/weksa/tests/original_corpus.rs | 83 +++++++++++++++- data/original/speaker-state.cc | Bin 32508 -> 60158 bytes docs/scene-annotation.md | 116 +++++++++++++++++++++++ scripts/scene-annotation/agreement.py | 99 +++++++++++++++++++ scripts/scene-annotation/build_briefs.py | 37 ++++++++ scripts/scene-annotation/put_labels.py | 15 +++ 7 files changed, 349 insertions(+), 5 deletions(-) create mode 100644 docs/scene-annotation.md create mode 100644 scripts/scene-annotation/agreement.py create mode 100644 scripts/scene-annotation/build_briefs.py create mode 100644 scripts/scene-annotation/put_labels.py diff --git a/README.md b/README.md index ff6eadd..2b5d0d3 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ culture, and constraint, and the planned lowerer that turns an agent's meaning packet into the reader's display language. **What is valuable here.** The interlingua standard and the lowering design: a -meaning packet owns what is meant, the scene state (live: `weksa scene put|dump`; speaker -documents are planned) owns the speaker's relation to the addressee, which selects register, +meaning packet owns what is meant, the scene state (live: `weksa scene put|dump`, labelled by the rule in +`docs/scene-annotation.md`; speaker documents are planned) owns the speaker's relation to the addressee, which selects register, address and politeness, and a lowerer owns surface text and leaves a trace. **The problem it solves.** An agent's outward speech today is one English diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index a3eae63..44af0cc 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -1039,7 +1039,7 @@ fn committed_scenes_cover_original_groups() { let held: BTreeSet = scenes.iter().map(|s| s.group.clone()).collect(); assert_eq!(held, live, "a scene for each live Train or Validation group, none other"); // Revisions append: a group's later revisions re-derived its relation. - assert!(scenes.iter().all(|s| s.n <= 2)); + assert!(scenes.iter().all(|s| s.n <= 3)); const S7: &str = "claude-sonnet-5-5 (Weksa Hands, model-annotated original scenes)"; const GROWTH: &str = "claude-sonnet-5-5 (Weksa Hands, model-annotated validation-growth scenes; annotator is not the writer)"; let grown: BTreeSet = CorpusStore::open(&data.join("corpus.cc")) @@ -1051,16 +1051,31 @@ fn committed_scenes_cover_original_groups() { .map(|i| i.group) .collect(); assert_eq!(grown.len(), 42); + const RUBRIC: &str = + "claude-sonnet-5-5 (Weksa Hands, scene rubric r1, docs/scene-annotation.md)"; + assert!( + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../docs/scene-annotation.md") + .is_file(), + "the rubric r1 names a doc, and the doc is in the repo" + ); let mut by_growth = BTreeSet::new(); for s in &scenes { match &s.annotator { Annotator::Model { family, generator } => { assert_eq!(*family, ModelFamily::Anthropic, "{}", s.group); - assert!(generator == S7 || generator == GROWTH, "{}", s.group); + assert!( + generator == S7 || generator == GROWTH || generator == RUBRIC, + "{}", + s.group + ); if generator == GROWTH { by_growth.insert(s.group.clone()); - // One relation, keeper to unnamed, the three Scene features. + // Written once, never revised by its own generator. assert_eq!(s.n, 1, "{}", s.group); + } + if generator == GROWTH || generator == RUBRIC { + // One relation, keeper to unnamed, the three Scene features. assert_eq!(s.relations.len(), 1, "{}", s.group); let rel = &s.relations[0]; assert_eq!(rel.from.as_str(), "entity:keeper", "{}", s.group); @@ -1075,8 +1090,70 @@ fn committed_scenes_cover_original_groups() { by_growth, grown, "the growth scenes are exactly the vgrow groups" ); + // The label a run reads is each group's latest revision: all 98 are rubric r1's. + let mut latest = std::collections::BTreeMap::new(); + for s in &scenes { + if latest.get(&s.group).is_none_or(|l: &&weksa::speaker::Scene| l.n < s.n) { + latest.insert(s.group.clone(), s); + } + } + let mut lines = Vec::new(); + for (group, s) in &latest { + let Annotator::Model { generator, .. } = &s.annotator else { + panic!("{group} is not model-annotated"); + }; + assert_eq!(generator, RUBRIC, "{group}: the latest revision follows rubric r1"); + lines.push(format!("{group}\n{}\n{:?}", s.n, s.relations[0].features)); + } + let parts: Vec<&[u8]> = lines.iter().map(|l| l.as_bytes()).collect(); + assert_eq!( + weksa::split::fnv1a64_of(&parts), + DIGEST_OF_THE_SCENES, + "a latest scene changed without a rubric revision" + ); + // The revisions before rubric r1 are provenance and never change: 113 of them. + let mut earlier: Vec<&weksa::speaker::Scene> = scenes + .iter() + .filter(|s| !matches!(&s.annotator, Annotator::Model { generator, .. } if generator == RUBRIC)) + .collect(); + earlier.sort_by(|a, b| (&a.group, a.n).cmp(&(&b.group, b.n))); + assert_eq!(earlier.len(), 113); + let lines: Vec = earlier + .iter() + .map(|s| format!("{}\n{}\n{:?}\n{:?}", s.group, s.n, s.annotator, s.relations)) + .collect(); + let parts: Vec<&[u8]> = lines.iter().map(|l| l.as_bytes()).collect(); + assert_eq!( + weksa::split::fnv1a64_of(&parts), + DIGEST_OF_THE_EARLIER_SCENES, + "an earlier scene revision changed" + ); + // Rubric r1 is text the labels cite: the section, as committed, is pinned. + let doc = std::fs::read_to_string( + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../docs/scene-annotation.md"), + ) + .unwrap(); + let from = doc.find("## Rubric r1").expect("the doc holds Rubric r1"); + let rest = &doc[from + "## Rubric r1".len()..]; + let section = &doc[from..from + "## Rubric r1".len() + rest.find("\n## ").expect("a section follows r1")]; + assert_eq!( + weksa::split::fnv1a64_of(&[section.trim().as_bytes()]), + DIGEST_OF_THE_RUBRIC, + "rubric r1 was edited; a new rubric is a new section and a new generator" + ); } +/// Taken over the 113 scene revisions that precede rubric r1, 2026-11-13. +const DIGEST_OF_THE_EARLIER_SCENES: u64 = 2598828882325200913; + +/// Taken over the text of the `## Rubric r1` section of docs/scene-annotation.md +/// (trimmed), 2026-11-13. +const DIGEST_OF_THE_RUBRIC: u64 = 11823581212652524671; + +/// Taken over each live group's latest revision (group, n, features) at rubric r1, +/// 2026-11-13. +const DIGEST_OF_THE_SCENES: u64 = 2505219080875817063; + /// The sweep both seed-pair tests share: over a grid of grades per feature (each /// also ABSENT) and the production `Condition::holds`, no listed pair of one /// language's entries co-holds; and each witness entry fires where it should and diff --git a/data/original/speaker-state.cc b/data/original/speaker-state.cc index 2503674ddbe09cd0598caeb70e5130c618d4ccd8..07bd001fcc60dc9db82fb63563f121474be7943f 100644 GIT binary patch literal 60158 zcmd^|OK&7s6~|+T57L@NM5OUB_Pi3a2^&C>011VMj6kaHu5nlTQC3y8tp>3ei5061 ziJC7!KtLuym?S&`NWtug5)xh$!kcwgh(LJWTUFh2`qVky$3Atd%f&9SeLU{Je&>B} z)#J}}#)G8eb$b4VI0+;F!r}JKFMj>V@5kPSxZ{s}_}@>1ubUr)-@5bgXU@guw+y`x zdPD!*?m6(I=g#Nf`luhpK{#6cuK7jg2ZsKzZN7VM_Y`nNKAA94T*PeJj{Zn?s zDA*51-eAicb$Vg+(86DEd*|ZL)vfLAt?f&XT)eWo{owBQ1AE7RpMC>bZM@^=Eq{~* z$z-?h`v-n>^YO+1|H?;3Bl8)$kALB7*L^P;N51)GH@^I#gT(ypPY(P;e{lCZI~Q*} z`M${O`Y&xf2EMv|<4f-yc>6Ejc=7{b*Zllf-!n>jQFst^?p*H-ym8mxio?;!PqwaX zU1`1ZV^;6h`@K;&KHnNz3-b|2Vd5ozw{_?Zg05%(@vZ$RJW6`4RE_xEt^8-gs1?L5 z^W&|g=eLfcAn~L7In8#UxC@$H+r46ByS8`yMnSVpSDO7*rrE=*G)om}MdNl9bXw8& z`BpdV#P3SKI#bfNQoxVfwOGei-VB_513_B zBWHXF>otAsptTSH0y9ghMWYGv2ZPCp39&}b2&pW)6bMPRSVf3wZx#r#Mol855Ld&a z)rGn#J{B^Ju{ryEkV**fSjaF&i6+re@R`W>x+jhw)=0@m+aH8Spi7zeaT}ikUtKa0 zaFAhAEf_$!KCW$zl!WkFytrO1R`qc>mRc>8eH^caFt3zZtrxu|42PVz%m-*@81xqC zZrhPDQhE#9!))GHB!rrJR5d@N6=QsA-t!eWGKwjEidRfHCn}Sp@1m}+;|=^#*IPY{ zH7LD3{notmo7uUdiJFzHlO|R-0Z_jqbaTZNr5iPGE0ZZa-S3z}Rz?(@?q^m?p7`0D zFqc;qofypoG{Nb9rdg9%`VNjyL2R}mqRGQ+({J7lPnnsLCN^?Qy-Z&ja(W0r`E5~; zCXs^kdnX+4v$JikB1(;vl&~>p^M!g>M44g&lHimgHFCw#=9@Cb%FJH4H}8i0W*V*& z;grJKC@HEztRa;U-~p!;RtZ;hNTY6ijfWY{XYSCyy zqA*Ul&Kw31xGA}Ajg*A&d5GxDsh&VztrQ4JwO|P0=2)o~>n6mN7AKrzrCKQGSS%Oo zh^M%o?Ril<9JAvG=r6Fo&&`w`!{^+}c8i=I0-quDPu@?+ZyydqxRZ`@p`X!YR-~sz* zt3;FNDELe`^4U{DY?QG^Rq|1*D3dBCz-S@LSfeI!g!gZcdRyMm>xPnq1^1RSJ0(pg zs+8AB6RVp5D7d$r>DDBs@D9#?#t~Zr&0A43+Qo>Ghs=IN-sKHPIM|{|4Voym4uLjYBx~cdWH3Y*7 z>@%^aoK^5@2)B@BZcToGw)i$G9C_vvq~8xX&w<6?f2PmQSNYIZ=|60g-4M>VsyZFA zR6oYo;OOE{jDAXw0Kk zj{i0NHbf@3bj3{Fj&3i}PJq@QNbQ=$7G7$Y66yLlVrL;R7Qv-qZlh{wGplV%un;L#1Ei#1bH!shF`AKl15{zC$qP!_YQDS6@}^%^emkvv7B zBv#-l3Y0vZ$kV8Lr>Y#rsdgHkqCm;hiBxl)!Col#s&=-~_O`A{<{Q77-iFQU%(P0b zD24RdY?&?sYW_&;(j;<-PCe-R`<`sfkER^Cr7LdEz?5UAod7O0<;X3Y#1_$W1d$0e zTu6dt6YAAcJ7-)8`}(T#9IKmvD%7i`x+$6BE2lLqCW(b@kg&}fBnn*=|CAZPc8B2r@QZ+%DT^-?PeXlKeN}f4D^mFh?2t!k9r{sz4>o(38j}TGX z=<6j`5r!2wh@}*9MT>OZ?@*CdqgXQ%L9sYxL^+G)86o!46eF(v#}E2!G=^&oOpCQs z`j5IJb3;q22;)EVDJwyXwNv_!T9GnoA&$iK!=Wh$#|e8F59V)lq-bvIikiB08%y;Q z@P&>P&Gl;%V^sc{j75iu79NgeGrG0=e_@N&O~4h-a#lAbQ*33uj_nXJb(G#)K(ktF z=8Pz!ey-W9HdRdk6q@~8GbK;8%7Z^F&T5Cjr%e(Zda}{Tf2YsEY3*HOS8P#Uw@$W} zD<~lB&zyoy;*5%0W1cWroTD=}Lt|DYTU6hougy6<1iavgWb|kfDa3pz?6U{3At>ld zh_O2RoG@w#FhSEYqejyynXsMPGD;@eYgFodu8$@ZB}R=?P%_b8qf)C;W|vRw6Qf|X zANuSQKw*!M_KCTrGN!OIv$}QoOSBVkh4zWLcFG9EYDd+CRNgL=E#hpN?94Perx!Wv z!lbI`;}5Nc02BC-0W;=^8r547bBx#vu8V3JG?7m2l#xdrnf08oiU?3albuu%Wjs+U zQf53M?mxv|-y4aF8gwTtGgR`z&b{vaTyXMnlcug2!(n2qPDJ7|ik{Di-53gSn z!4eSNlgl_!ZUJlWMSj08EBOAoc#AMj4V7NRsv~{KLRIH7C|}EzV|)#+3o%!zq0-k_ z<)n2@nbDazyBYVq{@7d(mmDDb_u|>jnW;0jYBqmgrrrv=b}sH6qm!>Q^_s*NadY0F zH4G|jAAqM9fuP@%52i8i;*qkSQe2%^jSfLJs5G)3Gb(9qG>Ii+%xjHM#0_$Mt zsSALeCe;SSL?iBnj|MXT`sLz1@Z3`ATk0IAr)P;Oj9<+Xzah z_spm4kE2P*A1;GqM6@BjYwV1jnmvAKxq<@7(1!G`f=%L#*a*g9TXcL8ZAh(|Gq!{$ zJN0pi(?h@u+K^g3noi1C7K_jspxBsINeVS3s;a-Gc(SUUf+u6vc%FuAUkjU#SHY_Q z%=|tc54C4AxE;?=-!2Z=s1=W!z+*la2MX*ma3D5jN>DgCUaIaI2I)9P4?$mn1F_LV z*-)tUD6=#q))|R+x+@2%U6rBOjqk9E2%;=n@T7_;y{A@<1TkPuyneWKSu)*0%Z$uW z$qPHWsO~}&uL%PTjJBjEN@93TDl2?>O#~4a4dOFSlrdMWNtrR1xOieeq#0Ss#yG!T ze3tFpR_ROXNTaV?mMX)z6CCBBD>-vzlpdv4rc8Q>S|;(vqKEXO+RmDteP%d41h}Bu z&g#)5Qm9JCMj1)6Mu&G2Yj^fpVHFX;f@?dgNRwzGhUnu_H|%>NUeIbGw>8m<9np941pkc}AdC z85+mnfj3J0NR*`EmNz$3dJSLIR(3&%(?j4kgdR%2;q|z$=r>eO(nk?a4}s&rY?Je$ z^c-G~HM|N4S9F6g97@_ASle0~rO&9NiJlQs3C3aOr)W%Mg|)3!Lg_7PCCWTKgCFx^ zo)H=MBzV9_-lRrnpAT9K0U+>^H>nm)A_P|?!yq#6Sm6+X%@2GbMrP%Tk-E*|7|jGI z!747(%oR)Pn7|c0Y&__&(FUUQ*7WK5pk-){lq9hknmQ*ktYS)z zc*Tef3Z+nJ!B4ED^930Bjok&+~}hv~a5 zS_=UpFmt6^G>H)0gGOPPaJBS&(_{dF-MF<;@==3HC#eL32b}t1C6sjVN(gh2-cv{= z1cNI$m#|DIr&j8i;W8`7ok`LYt=wRRmsu%&h1aF#6<)3x<5O_L1uMMFO6gU+W2^}4ayw^Cy;qxvaQ9;X)KF4dMATW1=KU9l7=O z5T%0v4|qP!=%6G+twWg^3|EM8=pArqz)15qa6Zl2xMHJjFS;Q0T7}Tg;-=5))g*@S z{4YMRZ}8wKngL{~v#Zh%P+8j4ag3sfJCW1W#?^Bpk7A*BlZw!AgzJJ|nai z0z}{hE7hXmgoFbwjLegfS-rp=o>3(nxFV!(kKrdcGK!s?qi~?)h@Evw_vdBSG&lk$ zzIVXSLLo?S!joAkN#Z+8q6?_g4lmbCfD+7FnU#_ywPt0qgzLSe=N)k&2_`K#OSU#j zj?_JazIP&(5byzK$yNy^9cm@Yq@xgNdXevSIX>7(lUgYW;k`yU!>Vdd!)PWTX(7_2 znkiXgHIsG=WwKQ8ub?X`j~611u~|1q4kC?Jg5e{Lx%oT1uoAB5plTXwg6BwyRr@fx#w delta 882 zcmXX^X-Je&6y<%+rjyM#y-{;gMjexeGR=}mD8~vSwZ(-ZjLLSSL}@RiC`$=thHhpm zrDQ5iO4_xE$QeZ0NTY-{@kbUKsWW5md!PRv_r7!QIp-elfREcl>~$o+r9Pg3 zt4#!*LWRCQf&dn^0peBT5Ys3^ie?Jz$s*?X`4cFaGtG-lbLC>UXcTaoh7qv!ikK9* zh(OPvh-LF52@Iu*wZlfj!P?LW0uygU$S}#k{BSm{CQ^wNX11^7s)!v?W&ynxGpB+n zOJ)JER0g!fvVyc^GH!OsrucaTPp{Mn&;}G83DE?sYfNZd9Yw%>mV(WyhRAX=G*;=L zHL(k}R8nk8vPi@z`mXmEpeBprx(y3u@ioaRsNDFMaiQ=lRf*1&d79t=ri{sy1i&V%ZLp0R20?oStVFqbGM1xHp^I zL-FxGCU;vQMdiU5&I^tm_P~Wz6w8n32sn>U@r9;54UXorh0>9}XSwB?iR?v|kYk zj1?=;StQB5=Eq%uwWTv`FvUDufMjndv|smdXU1=MB-$tEv=%!+CL`BD`j zd{yi1AcCXS0^!ob?lS2k4FPbcPK~Aa^aR`M*%he60H*s=#sVPW!357hMiaXoGZI`a zYIHnQ%E@^DNECSL^iOyeAm*u?nIvM`Gp54T7KpXaL*xT|bFLt`tb>y8D$$x4EenJ2M diff --git a/docs/scene-annotation.md b/docs/scene-annotation.md new file mode 100644 index 0000000..5fca125 --- /dev/null +++ b/docs/scene-annotation.md @@ -0,0 +1,116 @@ +# Scene annotation + +How the keeper scenes of `data/original/speaker-state.cc` are labelled. A scene +label has a committed rule that produced it; this file is that rule. + +## Why + +Two findings forced a written rule. + +- `annotation-rank-from-register`: an annotator who reads the line's register + (politeness forms, honorifics, endings) to decide the speaker's rank hands the + lowering its own answer back, because register is what the lowering must + produce from the scene. +- The addressee gate (`with_scene` in `crates/weksa/src/validate.rs`): a scene is + read only when the packet names a discourse addressee. A line that addresses no + one reads `NoAddressee` and no Scene entry selects, so labels on narration are + inert and the only labels that bite are those on lines said to someone. + +The earlier labels followed two briefs that were never committed and that +disagreed with each other (the relation to the reader, and a situation read with +the register ignored). The rubric below replaces both for every live group. + +## Provenance + +Cut `weksa-lowering:cut_spec:cut-scene-in-group.r1`, ruling +`weksa-lowering:ruling:scene-rubric-whole-bed`: all 98 live Train and Validation +groups of `data/original` receive a new scene revision from annotator A under +rubric r1, generator `claude-sonnet-5-5 (Weksa Hands, scene rubric r1, +docs/scene-annotation.md)`. Earlier revisions stay as provenance. The rubric uses +no corpus line as an example on purpose: a labelled example would make the blind +check against the old labels circular. + +A later rubric revision gets a new section and a new generator string; r1 is +never edited. + +## Rubric r1 + +The speaker of every line is the keeper, a quiet caretaker of a house and garden. First decide whether the line addresses a listener: it asks them something, asks them to do something, tells them about themselves, wishes something for them, or speaks to them as you. A line that addresses no one (narration, description, a remark about the world or about other people, a thought or wish with no one addressed) is told to the reader, a stranger: answer addressed false, relative_status level, intimacy 500, in_group false. People named inside a line are not its listener. + +For a line that addresses a listener, answer addressed true and judge the keeper's relation to that listener from the situation the line states or implies, never from its register: politeness forms, honorifics, sentence endings, contractions, please and casual particles are what the lowering must produce from the scene, so they cannot decide it. relative_status: above when the situation gives the keeper standing over the listener (instructing, advising or permitting as elder, host, employer or teacher); below when it gives the listener standing over the keeper (the keeper serves, thanks, or asks a favour of a guest, employer, elder or official); level when it shows neither. + +intimacy, 0 to 1000: 500 when the line shows nothing; up to 800 when it shows shared daily life (the listener's body, sleep or habits, the household's routine); above 800 only for a stated close bond; down to 200 for a stated first meeting or a service relation. in_group: true only when the line shows the listener belongs to the keeper's household, family or working circle, stated or plainly implied; a request, a question or a kind wish alone is not enough; otherwise false. Answer one JSON line per group, in the order given and nothing else: {"group": G, "addressed": true|false, "relative_status": S, "intimacy": N, "in_group": true|false}. + +## Procedure + +- Annotator A (committed) and annotator B (measurement only) are separate fresh + model agents, one pair per language (49 live groups each). Each sees only the + text of Rubric r1 above and that language's sentences as `group"text"`, + shuffled by a fixed seed. Neither sees an existing scene, the other annotator's + answer, the ontology, which items are old or new, or any Test text. +- An output that does not parse or misses a group is re-drawn once and logged. +- A's labels are written with `weksa scene put` (one revision per group, + `--to unnamed --inventory-rev 4`). `addressed` is not stored: no Scene feature + carries it. +- Disagreements are reported, never adjudicated. + +## Agreement at r1 (this cut, 98 live groups, 49 per language) + +Four fresh agents drew the labels (A-en, A-ja, B-en, B-ja); the briefs were the +rubric section plus one language's shuffled sentences (SHA-256 of the brief +bodies: en-US `db087476...dbc04e`, ja-JP `c5b81b27...97488c`; no Test text in +either). No draw was repeated. Annotator model: `claude-sonnet-5-5`. + +A against B (B measures only): pooled `addressed` equal 96/98 (kappa 0.96), +`in_group` equal 93/98 (kappa 0.79), `relative_status` equal 89/98, intimacy +within 150 on 98/98. en-US: addressed 47/49, in_group 47/49 (0.86), +relative_status 45/49. ja-JP: addressed 49/49, in_group 46/49 (0.64), +relative_status 44/49. + +Blind check, A against the 56 older labels: `relative_status` equal 49/56, +`in_group` equal 53/56 (kappa 0.64), intimacy within 150 on 55/56. Three +in_group differences: `original-ja-shimoyado-2` and `original-ja-sakana` (both +narrations, true to false) and `original-ja-asa` (an instruction to a household +member, false to true). + +In_group true across the 98: 42 before, 15 after; every unaddressed line is now +false (it was true on 17 of the 21 unaddressed vgrow groups). Disagreements are +reported, not adjudicated. + +## Reproducing the cut + +`scripts/scene-annotation/` holds the standard-library scripts of this cut, run +from the repo root with `WEKSA` (the binary) and `SCENE_SCRATCH` (a directory +that is never committed): `build_briefs.py` writes the two annotator briefs from +the Rubric r1 section and prints each brief's Test-leak count (must be 0) and +sha256 (it reproduces the briefs of this cut byte for byte); `put_labels.py` +puts A's `A-en.jsonl` and `A-ja.jsonl`; `agreement.py` prints the tables above and +must run before the puts, against the store at 76fa685, because the older labels +are then its latest revisions (it also reads the round-1 encoder answers of +`runs/growth-base` through `ROUND1`). `DIGEST_OF_THE_RUBRIC` and +`DIGEST_OF_THE_EARLIER_SCENES` in `crates/weksa/tests/original_corpus.rs` pin the +rubric text and the 113 earlier revisions. + +## Known limits of r1 labels + +Found by Soul's pass on this cut (`weksa-lowering:verdict:cut-scene-in-group.s1`). +They are not fixed in r1; a reviewer's pass over the labels is follow-up +`weksa-lowering:follow_up:scene-labels-reader-review`. + +- The new labels change which ontology entries are selected for 37 groups. Eight + addressed lines move onto the casual in-group entries (en-US:12: lantern, + broom, mirror, seedling; ja-JP:12: furo, asa, irori, yurikago), while their + references are polite requests. The reader default (level, 500, false) is + exactly the condition of ja-JP:11. +- `relative_status` tracks request form for some lines: imperatives read above, + favour questions read level or below, so the same household request gets + different standing in two forms (hane above/650/true, kabin below/500/false; + B gave both level/false). Form is what signals the speech act, which is what + part 2 asks for, and register is what part 2 forbids. +- `in_group` is true on some requests alone (lantern, mulch, stool, hane) and on + a line whose only family member is named inside it (original-ja-ane), though r1 + says a request or a person named inside the line is not enough. A and B + agreeing does not clear this. +- A run opened on the old speaker-state refuses to continue after merge: + `runs/growth-base` pins the store of `/var/lib/mind-weksa-self/Weksa-growth` + (main), and that checkout must not be pulled while the run is wanted. diff --git a/scripts/scene-annotation/agreement.py b/scripts/scene-annotation/agreement.py new file mode 100644 index 0000000..0a200e3 --- /dev/null +++ b/scripts/scene-annotation/agreement.py @@ -0,0 +1,99 @@ +"""Agreement of annotator A, B, the older labels and the round-1 encoder. Run it BEFORE the +rubric r1 revisions are put: the older labels are the store's latest revision then. +Standard library only. Usage: as build_briefs.py, plus ROUND1=.""" +import re,sys,subprocess,json,glob,os +import os +W=os.environ.get("WEKSA","target/debug/weksa") # the weksa binary +R=os.getcwd() # run from the repo root +S=os.environ.get("SCENE_SCRATCH","scene-scratch")+"/" # A-*.jsonl, B-*.jsonl, briefs; never committed +A=["--corpus","data/original/corpus.cc","--entities","data/original/entity.cc","--inventory","data/original/inventory.cc","--speaker-state","data/original/speaker-state.cc"] +def run(*a): return subprocess.run([W]+A+list(a),cwd=R,capture_output=True,text=True).stdout.split("\n") +out=run("corpus","dump") +items=[] +for i,l in enumerate(out): + m=re.match(r'^item:(\S+) \[(\S+) \S+ group=(\S+) (\w+)\]$',l) + if m and m.group(4)!="Test": + t=out[i+1].strip(); t=json.loads(t) if '\\' in t else t[1:-1] + items.append(dict(id=m.group(1),lang=m.group(2),group=m.group(3),text=t,new=m.group(1).startswith("vgrow"))) +G={x['group']:x for x in items} +assert len(G)==98 +# scene dump: latest and @1 per group +sd=run("scene","dump"); lab={} +for i,l in enumerate(sd): + m=re.match(r'^scene:(\S+)@(\d+) rev:\d+ Model \S+ (.*) \[entity:keeper\]$',l) + if m: + f=re.search(r'relative_status=(\w+) intimacy=(\d+) in_group=(\w+)',sd[i+1]) + lab.setdefault(m.group(1),{})[int(m.group(2))]=dict(rs=f.group(1),im=int(f.group(2)),ig=f.group(3)=="true",gen=m.group(3)) +old={g:v[max(v)] for g,v in lab.items()} +def load(f): return {json.loads(l)['group']:json.loads(l) for l in open(S+f+".jsonl")} +Aa={**load("A-en"),**load("A-ja")}; Bb={**load("B-en"),**load("B-ja")} +assert set(Aa)==set(G)==set(Bb) +def kappa(x,y): + n=len(x); + if n==0: return float('nan') + po=sum(a==b for a,b in zip(x,y))/n + pa=sum(x)/n; pb=sum(y)/n + pe=pa*pb+(1-pa)*(1-pb) + return float('nan') if pe==1 else (po-pe)/(1-pe) +def tab(x,y): + return "[TT=%d TF=%d FT=%d FF=%d]"%(sum(a and b for a,b in zip(x,y)),sum(a and not b for a,b in zip(x,y)),sum((not a) and b for a,b in zip(x,y)),sum((not a) and (not b) for a,b in zip(x,y))) +def cmp(name,P,Q,gs,bkey=None): + # P,Q: dict group-> record with keys addressed(optional) relative_status intimacy in_group + def rs(r): return r.get('relative_status',r.get('rs')) + def im(r): return r.get('intimacy',r.get('im')) + def ig(r): return r.get('in_group',r.get('ig')) + n=len(gs) + if n==0: print(f"{name}: n=0"); return + e=lambda f: sum(f(P[g])==f(Q[g]) for g in gs) + s=f"{name}: n={n} rs_eq={e(rs)}/{n} ig_eq={e(ig)}/{n} im<=150={sum(abs(im(P[g])-im(Q[g]))<=150 for g in gs)}/{n}" + s+=f" ig_kappa={kappa([ig(P[g]) for g in gs],[ig(Q[g]) for g in gs]):.2f} ig_tab(A.row,B.col)={tab([ig(P[g]) for g in gs],[ig(Q[g]) for g in gs])}" + if all('addressed' in P[g] and 'addressed' in Q[g] for g in gs): + s+=f" addr_eq={e(lambda r:r['addressed'])}/{n} addr_kappa={kappa([P[g]['addressed'] for g in gs],[Q[g]['addressed'] for g in gs]):.2f} addr_tab={tab([P[g]['addressed'] for g in gs],[Q[g]['addressed'] for g in gs])}" + print(s) +L={"en-US":[g for g in G if G[g]['lang']=="en-US"],"ja-JP":[g for g in G if G[g]['lang']=="ja-JP"],"pooled":list(G)} +print("== (1) A vs B") +for k,gs in L.items(): cmp(k,Aa,Bb,gs) +oldg=[g for g in G if not G[g]['new']]; newg=[g for g in G if G[g]['new']] +print("== (2) blind check: A vs old latest, the 56") +for k,gs in L.items(): + gg=[g for g in gs if g in oldg] + cmp(k+" all",Aa,old,gg) + cmp(k+" A.addr=true",Aa,old,[g for g in gg if Aa[g]['addressed']]) + cmp(k+" A.addr=false",Aa,old,[g for g in gg if not Aa[g]['addressed']]) +print("differing in_group (old->A):") +for g in oldg: + if old[g]['ig']!=Aa[g]['in_group']: print(" ",g,old[g]['ig'],"->",Aa[g]['in_group'],"addr",Aa[g]['addressed']) +print("differing any label, old vs A:") +for g in oldg: + o=old[g];a=Aa[g] + if (o['rs'],o['im'],o['ig'])!=(a['relative_status'],a['intimacy'],a['in_group']): print(" %s old %s/%d/%s A %s/%d/%s addr=%s"%(g,o['rs'],o['im'],o['ig'],a['relative_status'],a['intimacy'],a['in_group'],a['addressed'])) +print("== (3) A vs vgrow @1, the 42") +for k,gs in L.items(): cmp(k,Aa,{g:lab[g][1] for g in newg},[g for g in gs if g in newg]) +print("== (4) A addressed vs round-1 encoder addressee") +enc={} +RD=os.environ.get("ROUND1","runs/growth-base/round-1")+"/" +for p in glob.glob(RD+"*.prompt"): + pt=open(p).read() + if not pt.startswith("Encode the line"): continue + ap=p[:-7]+".answer" + if not os.path.exists(ap): continue + # the line text; take the item whose text appears in the prompt + hit=[g for g in G if G[g]['text'] in pt] + if len(hit)!=1: continue + try: a=json.load(open(ap)) + except Exception: continue + enc[hit[0]]=a.get('discourse',{}).get('addressee') is not None +print("encoder answers matched:",len(enc),"of 98") +for k,gs in list(L.items()): + gg=[g for g in gs if g in enc] + print(k,"n=",len(gg),"A.row,enc.col",tab([Aa[g]['addressed'] for g in gg],[enc[g] for g in gg]),"kappa=%.2f"%kappa([Aa[g]['addressed'] for g in gg],[enc[g] for g in gg])) +for nm,grp in (("old",oldg),("new",newg)): + gg=[g for g in grp if g in enc]; print(nm,"enc-addressee true:",sum(enc[g] for g in gg),"of",len(gg),"; A addressed true among old/new:",sum(Aa[g]['addressed'] for g in grp),"of",len(grp)) +print("== (5) in_group true counts, old/new x addressed, before -> after") +for nm,grp in (("old56",oldg),("new42",newg)): + for ad in (True,False): + gg=[g for g in grp if Aa[g]['addressed']==ad] + before=sum(old[g]['ig'] for g in gg); after=sum(Aa[g]['in_group'] for g in gg) + print(f"{nm} A.addressed={ad}: n={len(gg)} in_group true before={before} after={after}") +print("totals before",sum(old[g]['ig'] for g in G),"after",sum(Aa[g]['in_group'] for g in G)) +print("A addressed true:",sum(Aa[g]['addressed'] for g in G),"B:",sum(Bb[g]['addressed'] for g in G)) diff --git a/scripts/scene-annotation/build_briefs.py b/scripts/scene-annotation/build_briefs.py new file mode 100644 index 0000000..472f475 --- /dev/null +++ b/scripts/scene-annotation/build_briefs.py @@ -0,0 +1,37 @@ +"""Build the two annotator briefs (rubric r1 section + one language's shuffled live +sentences), print the Test-leak count of each (must be 0) and its sha256. +Standard library only. Usage: from the repo root, WEKSA=... SCENE_SCRATCH=dir python3 -I scripts/scene-annotation/build_briefs.py""" +import re,sys,subprocess,random,json,hashlib,os +import os +W=os.environ.get("WEKSA","target/debug/weksa") # the weksa binary +R=os.getcwd() # run from the repo root +S=os.environ.get("SCENE_SCRATCH","scene-scratch")+"/" # A-*.jsonl, B-*.jsonl, briefs; never committed +A=["--corpus","data/original/corpus.cc","--entities","data/original/entity.cc","--inventory","data/original/inventory.cc","--speaker-state","data/original/speaker-state.cc"] +out=subprocess.run([W]+A+["corpus","dump"],cwd=R,capture_output=True,text=True).stdout.split("\n") +live=[];test=[] +for i,l in enumerate(out): + m=re.match(r'^item:(\S+) \[(\S+) \S+ group=(\S+) (\w+)\]$',l) + if not m: continue + t=out[i+1].strip() + assert t.startswith('"') and t.endswith('"') + t=json.loads(t) if '\\' in t else t[1:-1] + if m.group(4)=="Test": test.append(t) + else: live.append((m.group(2),m.group(3),t)) +assert len(live)==98,len(live) +rub=open(R+"/docs/scene-annotation.md").read() +sec=rub[rub.index("## Rubric r1"):] +# rubric section ends at next "## " heading +rest=sec[len("## Rubric r1"):] +nx=rest.find("\n## ") +sec=sec[:len("## Rubric r1")+(nx if nx>=0 else len(rest))].strip()+"\n" +def leaks(brief,text): + w=text.split(); c=list(text) + return text in brief or any(" ".join(w[k:k+5]) in brief for k in range(len(w)-4)) or (len(w)==1 and any("".join(c[k:k+12]) in brief for k in range(len(c)-11))) +for lang in ("en-US","ja-JP"): + rows=[(g,t) for (l,g,t) in live if l==lang] + assert len(rows)==49 + random.Random(f"scene-r1-{lang}").shuffle(rows) + body=sec+"\nThe groups, one per line as group\"text\":\n\n"+"\n".join(f'{g}\t"{t}"' for g,t in rows)+"\n" + n=sum(leaks(body,t) for t in test) + print(lang,"lines",len(rows),"test_leaks",n,hashlib.sha256(body.encode()).hexdigest()) + open(S+f"brief-{lang}.txt","w").write(body) diff --git a/scripts/scene-annotation/put_labels.py b/scripts/scene-annotation/put_labels.py new file mode 100644 index 0000000..9de31a1 --- /dev/null +++ b/scripts/scene-annotation/put_labels.py @@ -0,0 +1,15 @@ +"""Put annotator A's labels through weksa scene put, one revision per group. +Standard library only. Usage: as build_briefs.py.""" +import json,subprocess +import os +W=os.environ.get("WEKSA","target/debug/weksa") # the weksa binary +R=os.getcwd() # run from the repo root +S=os.environ.get("SCENE_SCRATCH","scene-scratch")+"/" # A-*.jsonl, B-*.jsonl, briefs; never committed +A=["--corpus","data/original/corpus.cc","--entities","data/original/entity.cc","--inventory","data/original/inventory.cc","--speaker-state","data/original/speaker-state.cc"] +rows=[json.loads(l) for f in ("A-en","A-ja") for l in open(S+f"{f}.jsonl")] +for r in sorted(rows,key=lambda r:r['group']): + c=[W]+A+["scene","put","--group",r['group'],"--to","unnamed","--inventory-rev","4", + "--set",f"relative_status={r['relative_status']}","--set",f"intimacy={r['intimacy']}","--set",f"in_group={'true' if r['in_group'] else 'false'}", + "--model-family","anthropic","--generator","claude-sonnet-5-5 (Weksa Hands, scene rubric r1, docs/scene-annotation.md)"] + p=subprocess.run(c,cwd=R,capture_output=True,text=True) + print(r['group'],p.returncode,p.stdout.strip()[:80],p.stderr.strip()[:200]) From d8afc4ab85c9462fc36ea802e55eeba18923afa3 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 09:18:54 +0000 Subject: [PATCH 04/21] calibrate-harness: weksa loop calibrate over answered draws (#78) Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/main.rs | 130 +++++- crates/weksa/src/run.rs | 524 ++++++++++++++++++----- crates/weksa/tests/cli.rs | 93 ++++ crates/weksa/tests/golf_loop.rs | 254 +++++++++++ docs/calibration.md | 99 +++++ docs/targets/interlingua-lowering-map.md | 21 +- 6 files changed, 1001 insertions(+), 120 deletions(-) create mode 100644 docs/calibration.md diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index f70887a..3cd2235 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -15,7 +15,7 @@ use weksa::packet::{FeatValue, FeatureValue, Grade}; use weksa::packet::proof_packet; use weksa::model::{ModelPort, RecordedAnswers}; use weksa::run::{ - AblationOutcome, CycleOutcome, LowererReach, OntologyMintOutcome, RecallOutcome, RelowerOutcome, RoundOutcome, RunPaths, RunStores, cycle, round, + AblationOutcome, CalibrateOutcome, CycleOutcome, FLOOR_ARMS_MIN, LowererReach, OntologyMintOutcome, RecallOutcome, RelowerOutcome, RoundOutcome, RunPaths, RunStores, cycle, round, }; use weksa::score::ScoreInput; use weksa::{ @@ -174,6 +174,31 @@ enum LoopCommand { #[arg(long, default_value = "http://127.0.0.1:11434")] ollama_url: String, }, + /// Replay the accept rule over answered relower draws of round ROUND, to + /// measure how often it accepts, rejects or regresses what it should. Reads + /// the recorded lowering and each draw (--prompted: repeats of it, at least + /// four; --degraded: arms that should regress; --candidate: arms to rate), + /// answered through `weksa loop relower ROOT --round N --arm LABEL`, and + /// prints counts over all Validation items and over each --class + /// NAME=PREFIX[,PREFIX...] (items whose key starts with a prefix). Writes + /// no prompts; exits 1 naming the draws still unanswered. Procedure: + /// docs/calibration.md. A diagnostic, never a gate; records nothing; Test + /// and Train are never touched. Labels match [a-z0-9-]{1,32} and differ. + Calibrate { + root: PathBuf, + #[arg(long)] + round: u32, + #[arg(long)] + prompted: Vec, + #[arg(long)] + degraded: Vec, + #[arg(long)] + candidate: Vec, + #[arg(long)] + class: Vec, + #[arg(long, default_value = "http://127.0.0.1:11434")] + ollama_url: String, + }, /// Run one edit-accept cycle of the run at ROOT. The stores are read only /// from the run's record. The editor's prompt, then the candidate round's /// encode prompts, then its lowering prompts are written in turn; the @@ -660,6 +685,12 @@ impl AddArgs { } } +/// A draw or arm label: [a-z0-9-]{1,32}. +fn is_label(l: &str) -> bool { + (1..=32).contains(&l.len()) + && l.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') +} + /// The split of an item as `Train`, `Validation`, `Test` or `excluded`. fn split_text(item: &CorpusItem) -> String { SplitScheme::V1 @@ -1146,12 +1177,8 @@ fn main() -> Result<()> { floor, ollama_url, }) => { - let label = |l: &str| { - (1..=32).contains(&l.len()) - && l.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') - }; - if !label(&arm) - || floor.iter().any(|f| !label(f) || *f == arm) + if !is_label(&arm) + || floor.iter().any(|f| !is_label(f) || *f == arm) || floor.iter().enumerate().any(|(i, f)| floor[..i].contains(f)) { anyhow::bail!("arm and floor labels must match [a-z0-9-]{{1,32}} and differ"); @@ -1197,6 +1224,95 @@ fn main() -> Result<()> { } } } + Command::Loop(LoopCommand::Calibrate { + root, + round, + prompted, + degraded, + candidate, + class, + ollama_url, + }) => { + let labels: Vec<&String> = prompted.iter().chain(°raded).chain(&candidate).collect(); + if labels.iter().any(|l| !is_label(l)) + || labels.iter().enumerate().any(|(i, l)| labels[..i].contains(l)) + { + anyhow::bail!("draw labels must match [a-z0-9-]{{1,32}} and differ"); + } + if prompted.len() < FLOOR_ARMS_MIN + 1 { + anyhow::bail!("at least {} --prompted labels are needed", FLOOR_ARMS_MIN + 1); + } + let mut classes: Vec<(String, Vec)> = Vec::new(); + for c in &class { + let Some((name, prefixes)) = c.split_once('=') else { + anyhow::bail!("--class is NAME=PREFIX[,PREFIX...]"); + }; + if !is_label(name) + || name == "all" + || classes.iter().any(|(n, _)| n == name) + || prefixes.split(',').any(str::is_empty) + { + anyhow::bail!("--class is NAME=PREFIX[,PREFIX...] with a new label NAME other than all"); + } + classes.push((name.to_string(), prefixes.split(',').map(String::from).collect())); + } + let run = RunStores::open(&root)?; + let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder)); + match run.calibrate(round, &prompted, °raded, &candidate, &classes, &scorer)? { + CalibrateOutcome::Unanswered(missing) => { + for (label, n) in &missing { + println!("unanswered {label} {n}"); + } + eprintln!( + "answer it through weksa loop relower ROOT --round N --arm LABEL first" + ); + std::process::exit(1); + } + CalibrateOutcome::Read { unusable, classes } => { + println!( + "{}", + run.labelled(&format!( + "calibrate draws prompted={} degraded={} candidate={} floor-arms={FLOOR_ARMS_MIN} unusable={unusable}", + prompted.len(), + degraded.len(), + candidate.len(), + )) + ); + for c in classes { + let items: String = c + .languages + .iter() + .map(|l| format!(" {} n={} skipped={}", l.language, l.n, l.skipped)) + .collect(); + let t = &c.trials; + println!("{}", run.labelled(&format!("calibrate class={} items{items}", c.name))); + println!( + "{}", + run.labelled(&format!( + "calibrate class={} prompted trials={} nogain={} accepted={} regressed={} noise={:.4}", + c.name, t.prompted.trials, t.prompted.nogain, t.prompted.accepted, t.prompted.regressed, t.noise + )) + ); + for (label, d) in degraded.iter().zip(&t.degraded) { + println!( + "{}", + run.labelled(&format!( + "calibrate class={} degraded={label} trials={} regressed={}", + c.name, d.trials, d.regressed + )) + ); + } + println!( + "{}", + run.labelled(&format!( + "calibrate class={} candidate trials={} accepted={} nogain={} regressed={}", + c.name, t.candidate.trials, t.candidate.accepted, t.candidate.nogain, t.candidate.regressed + )) + ); + } + } + } + } Command::Loop(LoopCommand::Recall { root }) => { let run = RunStores::open(&root)?; let mut files = run.answer_files(run.recall_dir())?; diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 463db61..b15fd07 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -44,7 +44,7 @@ use crate::encode::{ EditDraft, LoweringDraft, MAX_EDIT_CHANGES, OntologyDraft, PacketDraft, encoder_id, }; use crate::form::{is_attestation, is_language_tag, is_slug}; -use crate::model::{AnswerFiles, ModelPort, PortError}; +use crate::model::{AnswerFiles, ModelPort, PortError, RecordedAnswers}; use crate::packet::{Packet, Stance}; use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate, validation_items}; use crate::split::{Split, SplitScheme, fnv1a64, fnv1a64_of, hex16}; @@ -1675,27 +1675,90 @@ impl RunStores { floors: &mut [&mut dyn ModelPort], scorer: &Scorer, ) -> Result { - let store = |_| RunError::Store; check_embedder(self, scorer)?; - let held = self.held_splits(n, &[Split::Validation])?; - let inventory = self.inventory.inventory().map_err(store)?; - let ontology = self.ontology.ontology().map_err(store)?; - - let dir = self.relower_dir(n, arm_label); - std::fs::create_dir_all(&dir).map_err(|e| RunError::Io(e.kind()))?; - for suffix in ["cc", "cc.lock"] { - let _ = std::fs::remove_file(dir.join(format!("base.{suffix}"))); - } - let mut scratch = PacketStore::open(&dir.join("base.cc")).map_err(store)?; - if (1..FLOOR_ARMS_MIN).contains(&floors.len()) { return Err(RunError::FloorArms); } - // Per language, per item: the item text, then the recorded and arm - // lowerings and one per floor arm (`None` when unusable). - type Row = (String, Option, Option, Vec>); - let mut waiting: Vec = Vec::new(); - let mut floor_waiting = false; + let dir = self.relower_dir(n, arm_label); + // Ports in the order the prompts are asked: recorded, floors, arm. + let floor_count = floors.len(); + let mut ports: Vec<&mut dyn ModelPort> = vec![recorded]; + for f in floors.iter_mut() { + let f: &mut dyn ModelPort = &mut **f; + ports.push(f); + } + ports.push(arm); + let drawn = self.read_draws(n, &dir.join("base.cc"), &mut ports, scorer)?; + let arm_at = floor_count + 1; + if drawn.waiting[1..arm_at].iter().any(|w| !w.is_empty()) { + return Err(RunError::FloorUnanswered); + } + if !drawn.waiting[arm_at].is_empty() { + return Ok(RelowerOutcome::Unanswered { prompts: drawn.waiting[arm_at].clone() }); + } + let mut languages = Vec::new(); + for l in &drawn.languages { + let of_port = |p: usize| -> Vec { l.items.iter().map(|(_, s)| s[p].clone()).collect() }; + let rec = of_port(0); + let floor_scores: Vec> = (1..arm_at).map(of_port).collect(); + languages.push(RelowerReading { + language: l.language.clone(), + n: l.items.len() as u32, + skipped: l.skipped, + unusable: l.unusable[arm_at], + floor_unusable: l.unusable[1..arm_at].iter().sum(), + recorded: reading_of(&rec)?, + arm: reading_of(&of_port(arm_at))?, + floor: if floor_count == 0 { None } else { Some(paired_floor(&rec, &floor_scores)) }, + }); + } + let verdict = if floor_count > 0 { + let panel: Vec<(Reading, Reading, Reading)> = languages + .iter() + .filter_map(|l| Some((l.recorded, l.arm, l.floor?))) + .collect(); + Some(decide_panel(&panel)) + } else { + None + }; + Ok(RelowerOutcome::Read { languages, verdict }) + } + + /// Asks every port, in the order given, each live Validation item's + /// byte-identical lowering prompt of round `n` and scores the answers against + /// the items in one [`Scorer::score`] call: the one owner of per-item + /// scoring for [`relower`](Self::relower) and + /// [`calibrate`](Self::calibrate). An answer that is not a usable lowering + /// scores 0. Port 0 is the recorded lowering, which must be answered + /// (`Unrecorded` at once); any other port that is not is listed in + /// `waiting`, and nothing is scored. `scratch` is the packet store the + /// prompts are built in, removed and reopened here; its directory is + /// created here, after the round is found. + fn read_draws( + &self, + n: u32, + scratch: &Path, + ports: &mut [&mut dyn ModelPort], + scorer: &Scorer, + ) -> Result { + let store = |_| RunError::Store; + let held = self.held_splits(n, &[Split::Validation])?; + let inventory = self.inventory.inventory().map_err(store)?; + let ontology = self.ontology.ontology().map_err(store)?; + if let Some(dir) = scratch.parent() { + std::fs::create_dir_all(dir).map_err(|e| RunError::Io(e.kind()))?; + } + for suffix in ["", ".lock"] { + let mut name = scratch.as_os_str().to_owned(); + name.push(suffix); + let _ = std::fs::remove_file(PathBuf::from(name)); + } + let mut scratch = PacketStore::open(scratch).map_err(store)?; + + // Per language, per item: the item id and text, then each port's lowering + // (`None` when unusable or unanswered). + let mut waiting: Vec> = vec![Vec::new(); ports.len()]; + type Row = (String, String, Vec>); let mut rows: Vec> = Vec::new(); for (_, _, items) in &held { let mut language_rows = Vec::new(); @@ -1707,49 +1770,33 @@ impl RunStores { else { continue; }; - let text = |a: Asked| match a { - Asked::Text(t) => Some(t), - _ => None, - }; - let recorded_text = match asked(recorded, &prompt)? { - Asked::Waiting => return Err(RunError::Unrecorded), - other => text(other), - }; - let mut floor_texts = Vec::new(); - for f in floors.iter_mut() { - floor_texts.push(match asked(&mut **f, &prompt)? { + let mut texts = Vec::new(); + for (p, port) in ports.iter_mut().enumerate() { + texts.push(match asked(&mut **port, &prompt)? { + Asked::Text(t) => Some(t), + Asked::Unusable => None, + Asked::Waiting if p == 0 => return Err(RunError::Unrecorded), Asked::Waiting => { - floor_waiting = true; + let d = prompt.digest(); + if !waiting[p].contains(&d) { + waiting[p].push(d); + } None } - other => text(other), }); } - let arm_text = match asked(arm, &prompt)? { - Asked::Waiting => { - let d = prompt.digest(); - if !waiting.contains(&d) { - waiting.push(d); - } - None - } - other => text(other), - }; - language_rows.push((item.text.clone(), recorded_text, arm_text, floor_texts)); + language_rows.push((item.id.as_str().to_string(), item.text.clone(), texts)); } rows.push(language_rows); } - if floor_waiting { - return Err(RunError::FloorUnanswered); - } - if !waiting.is_empty() { - return Ok(RelowerOutcome::Unanswered { prompts: waiting }); + if waiting.iter().any(|w| !w.is_empty()) { + return Ok(Drawn { languages: Vec::new(), waiting }); } // One score call: each usable lowering against its item, in row order. let mut inputs: Vec = Vec::new(); - for (reference, recorded, arm, floor_texts) in rows.iter().flatten() { - for output in [recorded, arm].into_iter().chain(floor_texts).flatten() { + for (_, reference, texts) in rows.iter().flatten() { + for output in texts.iter().flatten() { inputs.push(ScoreInput { reference, output }); } } @@ -1759,67 +1806,272 @@ impl RunStores { scorer.score(&inputs).map_err(|_| RunError::Embed)? } .into_iter(); - let mut next = |t: &Option| -> Result { - match t { - Some(_) => scores.next().ok_or(RunError::Store), - None => Ok(ItemScore { chrf: 0.0, meaning: Some(0.0) }), - } - }; - let reading_of = |scored: &[ItemScore]| { - Reading::of(&SplitMean { - count: scored.len(), - chrf: mean(scored.iter().map(|s| s.chrf)), - meaning: mean(scored.iter().filter_map(|s| s.meaning)), - }) - .ok_or(RunError::NoValidation) - }; - let mut languages = Vec::new(); let live = self.corpus.all().map_err(store)?; - for ((language, _, _), language_rows) in held.iter().zip(&rows) { - let (mut rec, mut arm_scores) = (Vec::new(), Vec::new()); - let mut floor_scores: Vec> = vec![Vec::new(); floors.len()]; - let (mut unusable, mut floor_unusable) = (0u32, 0u32); - for (_, r, a, f) in language_rows { - rec.push(next(r)?); - arm_scores.push(next(a)?); - if a.is_none() { - unusable += 1; - } - for (scores, t) in floor_scores.iter_mut().zip(f) { - scores.push(next(t)?); - if t.is_none() { - floor_unusable += 1; - } + let mut languages = Vec::new(); + for ((language, _, _), language_rows) in held.iter().zip(rows) { + let mut unusable = vec![0u32; ports.len()]; + let mut items = Vec::new(); + for (id, _, texts) in language_rows { + let mut item_scores = Vec::new(); + for (p, t) in texts.iter().enumerate() { + item_scores.push(match t { + Some(_) => scores.next().ok_or(RunError::Store)?, + None => { + unusable[p] += 1; + ItemScore { chrf: 0.0, meaning: Some(0.0) } + } + }); } + items.push((id, item_scores)); } - let recorded_reading = reading_of(&rec)?; - languages.push(RelowerReading { + let live_validation = live + .iter() + .filter(|i| &i.language == language && SplitScheme::V1.live(i) == Some(Split::Validation)) + .count() as u32; + languages.push(DrawnLanguage { language: language.clone(), - n: language_rows.len() as u32, - skipped: live - .iter() - .filter(|i| { - &i.language == language && SplitScheme::V1.live(i) == Some(Split::Validation) - }) - .count() as u32 - - language_rows.len() as u32, + skipped: live_validation - items.len() as u32, + items, unusable, - floor_unusable, - recorded: recorded_reading, - arm: reading_of(&arm_scores)?, - floor: if floors.is_empty() { None } else { Some(paired_floor(&rec, &floor_scores)) }, }); } - let verdict = if !floors.is_empty() { - let panel: Vec<(Reading, Reading, Reading)> = languages - .iter() - .filter_map(|l| Some((l.recorded, l.arm, l.floor?))) - .collect(); - Some(decide_panel(&panel)) - } else { - None + Ok(Drawn { languages, waiting }) + } +} + +/// One panel language of [`RunStores::read_draws`]. +struct DrawnLanguage { + language: String, + /// Live Validation items of the language with no lowering prompt. + skipped: u32, + /// Per read item in key order: its id and one score per port. + items: Vec<(String, Vec)>, + /// Per port, answers that were not a usable lowering. + unusable: Vec, +} + +struct Drawn { + /// Per panel language in panel order; empty while any port is waiting. + languages: Vec, + /// Per port, the digests of the prompts it holds no answer to. + waiting: Vec>, +} + +/// A reading over per-item scores; `NoValidation` without items. +fn reading_of(scored: &[ItemScore]) -> Result { + Reading::of(&SplitMean { + count: scored.len(), + chrf: mean(scored.iter().map(|s| s.chrf)), + meaning: mean(scored.iter().filter_map(|s| s.meaning)), + }) + .ok_or(RunError::NoValidation) +} + +/// How trials of one kind read: counts, not rates. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Counts { + pub trials: u32, + pub accepted: u32, + pub nogain: u32, + pub regressed: u32, +} + +impl Counts { + fn tally(&mut self, v: Verdict) { + self.trials += 1; + match v { + Verdict::Accepted => self.accepted += 1, + Verdict::Rejected(Why::Regressed(_)) => self.regressed += 1, + Verdict::Rejected(_) => self.nogain += 1, + } + } +} + +/// What [`calibration_trials`] counted over one set of items. +#[derive(Clone, Debug, PartialEq)] +pub struct Trials { + /// Ordered (base, candidate) pairs of prompted draws, each over every floor. + pub prompted: Counts, + /// Per degraded arm in the order given: the arm as the candidate. + pub degraded: Vec, + /// Every candidate arm as the candidate, pooled. + pub candidate: Counts, + /// The mean of [`panel_noise`] over every (base, floor) choice. + pub noise: f64, +} + +/// Every `k`-subset of `pool`, in order. +fn subsets(pool: &[usize], k: usize) -> Vec> { + if k == 0 { + return vec![Vec::new()]; + } + let mut out = Vec::new(); + for (i, &first) in pool.iter().enumerate() { + for mut rest in subsets(&pool[i + 1..], k - 1) { + rest.insert(0, first); + out.push(rest); + } + } + out +} + +/// The accept rule replayed over draws that are all lowerings of one round, as +/// the live loop would meet them: [`decide_panel`] over (base, candidate, +/// [`paired_floor`] of the base against `FLOOR_ARMS_MIN` other prompted draws), +/// one row per language. `draws` is `[port][language][item]`; ports +/// `0..prompted` are prompted draws (the recorded lowering first), then +/// `degraded` and `candidate` ports follow. A language with no items reads +/// NaN, which fails closed. +pub fn calibration_trials( + draws: &[Vec>], + prompted: usize, + degraded: usize, + candidate: usize, +) -> Trials { + let languages = draws.first().map_or(0, Vec::len); + let rows = |b: usize, c: usize, floors: &[usize]| -> Vec<(Reading, Reading, Reading)> { + let read = |scores: &[ItemScore]| { + reading_of(scores).unwrap_or(Reading { chrf: f64::NAN, meaning: f64::NAN, composite: f64::NAN }) }; - Ok(RelowerOutcome::Read { languages, verdict }) + (0..languages) + .map(|l| { + let arms: Vec> = floors.iter().map(|&f| draws[f][l].clone()).collect(); + (read(&draws[b][l]), read(&draws[c][l]), paired_floor(&draws[b][l], &arms)) + }) + .collect() + }; + let pool: Vec = (0..prompted).collect(); + let without = |out: &[usize]| -> Vec { pool.iter().copied().filter(|p| !out.contains(p)).collect() }; + let (mut prompted_counts, mut candidate_counts) = (Counts::default(), Counts::default()); + let mut degraded_counts = vec![Counts::default(); degraded]; + let (mut noise, mut noises) = (0.0, 0u32); + for &b in &pool { + for floors in subsets(&without(&[b]), FLOOR_ARMS_MIN) { + for (d, counts) in degraded_counts.iter_mut().enumerate() { + counts.tally(decide_panel(&rows(b, prompted + d, &floors))); + } + for c in prompted + degraded..prompted + degraded + candidate { + candidate_counts.tally(decide_panel(&rows(b, c, &floors))); + } + noise += panel_noise(&rows(b, b, &floors)); + noises += 1; + } + for &c in pool.iter().filter(|&&c| c != b) { + for floors in subsets(&without(&[b, c]), FLOOR_ARMS_MIN) { + prompted_counts.tally(decide_panel(&rows(b, c, &floors))); + } + } + } + Trials { + prompted: prompted_counts, + degraded: degraded_counts, + candidate: candidate_counts, + noise: noise / f64::from(noises), + } +} + +/// One panel language of a class: items read and live Validation items of the +/// class not read. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ClassLanguage { + pub language: String, + pub n: u32, + pub skipped: u32, +} + +/// The trials over one class of items. +#[derive(Clone, Debug, PartialEq)] +pub struct ClassTrials { + pub name: String, + pub languages: Vec, + pub trials: Trials, +} + +#[derive(Debug)] +pub enum CalibrateOutcome { + /// Draw labels with the number of prompts each has no answer to. + Unanswered(Vec<(String, u32)>), + /// `unusable` over every prompted, degraded and candidate draw; `all` first, + /// then the classes in the order given. + Read { unusable: u32, classes: Vec }, +} + +impl RunStores { + /// Replays the accept rule over answered relower draws: reads every draw of + /// round `n` (the recorded lowering and the `relower--
]]` around each place it names a cast member. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MarkedLine { + pub line: u32, + pub marked: String, +} + +impl MarkedLine { + pub fn parse(answer: &str) -> Result { + from_json(answer) + } +} + +/// Why an import was refused before any document was written. The text names +/// the kind and the line number only, never a value. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ImportRefusal { + NotPublicDomain, + CastRepeated, + NoSuchLine { line: u32 }, + LineMarkedTwice { line: u32 }, + Group { line: u32 }, + SpeakerNotInCast { line: u32 }, + Markup { line: u32 }, + NotInCast { line: u32 }, +} + +impl fmt::Display for ImportRefusal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NotPublicDomain => write!(f, "the work's source is not public domain"), + Self::CastRepeated => write!(f, "a cast entity is listed twice"), + Self::NoSuchLine { line } => write!(f, "the answer marks a line that is not there (line {line})"), + Self::LineMarkedTwice { line } => write!(f, "line {line} is marked twice"), + Self::Group { line } => write!(f, "the group of line {line} is not a slug"), + Self::SpeakerNotInCast { line } => write!(f, "the speaker of line {line} is not in the cast"), + Self::Markup { line } => write!(f, "the marked text of line {line} is not its text with markup"), + Self::NotInCast { line } => write!(f, "line {line} marks an entity outside the cast"), + } + } +} + +impl std::error::Error for ImportRefusal {} + +impl WorkDraft { + pub fn parse(file: &str) -> Result { + from_json(file) + } + + /// The cast entities and the corpus items for the lines `marked` covers, in + /// line order, each item's spans read from its markup and labelled + /// `annotator`. Decides everything before any write; whether the documents + /// are admissible is `EntityStore::put` and `CorpusStore::add`'s to refuse. + pub fn into_import( + self, + marked: &[MarkedLine], + annotator: &Annotator, + ) -> Result<(Vec, Vec), ImportRefusal> { + let Source::PublicDomain { work, .. } = &self.source else { + return Err(ImportRefusal::NotPublicDomain); + }; + let mut entities = Vec::new(); + for c in &self.cast { + if entities.iter().any(|e: &Entity| e.id == c.entity) { + return Err(ImportRefusal::CastRepeated); + } + entities.push(Entity { + id: c.entity.clone(), + forms: vec![LanguageForms { + language: self.language.clone(), + forms: c.forms.iter().map(|f| f.nfc().collect()).collect(), + pronunciation: None, + }], + origin: EntityOrigin::PublicDomain { work: work.clone() }, + }); + } + let in_cast = |id: &EntityId| entities.iter().any(|e| &e.id == id); + let mut answer: Vec<&MarkedLine> = marked.iter().collect(); + answer.sort_by_key(|m| m.line); + let mut items = Vec::new(); + for (i, m) in answer.iter().enumerate() { + let line = m.line; + if line == 0 || line as usize > self.lines.len() { + return Err(ImportRefusal::NoSuchLine { line }); + } + if i > 0 && answer[i - 1].line == line { + return Err(ImportRefusal::LineMarkedTwice { line }); + } + let draft = &self.lines[line as usize - 1]; + let group = format!("{}-{}", self.slug, draft.group); + if !is_slug(&group) { + return Err(ImportRefusal::Group { line }); + } + if !in_cast(&draft.speaker) { + return Err(ImportRefusal::SpeakerNotInCast { line }); + } + let text: String = draft.text.nfc().collect(); + if text.contains("[[") || text.contains("]]") { + return Err(ImportRefusal::Markup { line }); + } + let marked_nfc: String = m.marked.nfc().collect(); + let (plain, names) = match unmark(&marked_nfc) { + Some((plain, names)) if plain == text => (plain, names), + _ => return Err(ImportRefusal::Markup { line }), + }; + if names.iter().any(|n| !in_cast(&n.entity)) { + return Err(ImportRefusal::NotInCast { line }); + } + items.push(CorpusItem { + id: item_key(&self.slug, line), + text: plain, + language: self.language.clone(), + speaker: draft.speaker.clone(), + group, + prior_turns: Vec::new(), + source: self.source.clone(), + admitted_on: self.admitted_on.clone(), + supersedes: None, + withdrawn: false, + names, + names_by: Some(annotator.clone()), + }); + } + Ok((entities, items)) + } +} + +/// The text of `marked` with `[[entity|form]]` replaced by `form`, and the +/// byte span of each form in that text. `None` for any malformed markup. The +/// only search is for the markup delimiters. +fn unmark(marked: &str) -> Option<(String, Vec)> { + let mut out = String::with_capacity(marked.len()); + let mut names = Vec::new(); + let mut rest = marked; + while let Some(at) = rest.find("[[") { + out.push_str(&rest[..at]); + let inner = &rest[at + 2..]; + let (id, after) = inner.split_once('|')?; + let (form, tail) = after.split_once("]]")?; + if form.is_empty() || form.contains("[[") || id.contains("]]") { + return None; + } + let entity = EntityId::parse(id).ok()?; + let start = u32::try_from(out.len()).ok()?; + out.push_str(form); + names.push(NameSpan { + entity, + start, + end: u32::try_from(out.len()).ok()?, + }); + rest = tail; + } + out.push_str(rest); + Some((out, names)) +} + /// `encoder:`, a digest of the model id and the prompt digest. pub fn encoder_id(model: &str, prompt_digest: &str) -> EncoderId { let digest = fnv1a64(format!("weksa-encoder-v1:{model}:{prompt_digest}").as_bytes()); @@ -746,4 +942,145 @@ mod tests { assert!(entries.iter().all(|e| e.annotator == by)); assert_eq!(entries[1].examples, ["x"]); } + + const WORK: &str = r#"{"slug":"lantern-wager","language":"en-US","admitted_on":"2026-11-13", + "source":{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}}, + "cast":[{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:tamsin","forms":["Tamsin","Tam"]},{"entity":"entity:wexmoor","forms":["Wexmoor"]}], + "lines":[{"group":"s1","speaker":"entity:orsa","text":"Tamsin, bring the lantern."}, + {"group":"s1","speaker":"entity:tamsin","text":"Orsa, I will not, said Tam."}, + {"group":"s2","speaker":"entity:wexmoor","text":"Quiet, both of you."}]}"#; + + fn marked(line: u32, text: &str) -> MarkedLine { + MarkedLine { line, marked: text.into() } + } + + fn model_annotator() -> Annotator { + Annotator::Model { + family: crate::corpus::ModelFamily::Anthropic, + generator: "g".into(), + } + } + + fn import( + work: &str, + answer: &[MarkedLine], + ) -> Result<(Vec, Vec), ImportRefusal> { + WorkDraft::parse(work).unwrap().into_import(answer, &model_annotator()) + } + + #[test] + fn a_work_imports_its_cast_and_marked_lines() { + let answer = [ + marked(1, "[[entity:tamsin|Tamsin]], bring the lantern."), + marked(2, "[[entity:orsa|Orsa]], I will not, said [[entity:tamsin|Tam]]."), + ]; + let (entities, items) = import(WORK, &answer).unwrap(); + assert_eq!(entities.len(), 3); + for e in &entities { + assert!(matches!(&e.origin, EntityOrigin::PublicDomain { work } if work == "The Lantern Wager")); + assert_eq!(e.forms[0].language, "en-US"); + } + assert_eq!(items.len(), 2); + assert_eq!(items[0].id, "item:lantern-wager:1"); + assert_eq!(items[1].id, "item:lantern-wager:2"); + assert_eq!(items[0].group, "lantern-wager-s1"); + let span = |e: &str, text: &str, form: &str| { + let start = text.find(form).unwrap() as u32; + NameSpan { entity: EntityId::parse(e).unwrap(), start, end: start + form.len() as u32 } + }; + let t2 = &items[1].text; + assert_eq!(items[0].names, [span("entity:tamsin", &items[0].text, "Tamsin")]); + assert_eq!( + items[1].names, + [span("entity:orsa", t2, "Orsa"), span("entity:tamsin", t2, "Tam")] + ); + // "will" is unmarked and unspanned; the line's speaker is the file's. + assert_eq!(items[1].speaker, EntityId::parse("entity:tamsin").unwrap()); + for i in &items { + assert_eq!(i.names_by, Some(model_annotator())); + assert!(i.prior_turns.is_empty() && i.supersedes.is_none() && !i.withdrawn); + } + } + + #[test] + fn an_import_refuses_before_any_document() { + let ok = marked(1, "Tamsin, bring the lantern."); + let refused = |work: &str, answer: &[MarkedLine]| import(work, answer).unwrap_err(); + let cast_authored = WORK.replace( + r#"{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}}"#, + r#""OperatorWritten""#, + ); + assert_eq!(refused(&cast_authored, std::slice::from_ref(&ok)), ImportRefusal::NotPublicDomain); + let repeated = WORK.replace(r#"{"entity":"entity:orsa","forms":["Orsa"]}"#, r#"{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:orsa","forms":["O"]}"#); + assert_eq!(refused(&repeated, std::slice::from_ref(&ok)), ImportRefusal::CastRepeated); + let stranger = WORK.replace(r#""speaker":"entity:orsa""#, r#""speaker":"entity:stranger""#); + assert_eq!(refused(&stranger, std::slice::from_ref(&ok)), ImportRefusal::SpeakerNotInCast { line: 1 }); + assert_eq!(refused(WORK, &[marked(9, "x")]), ImportRefusal::NoSuchLine { line: 9 }); + assert_eq!(refused(WORK, &[marked(0, "x")]), ImportRefusal::NoSuchLine { line: 0 }); + assert_eq!( + refused(WORK, &[ok.clone(), ok.clone()]), + ImportRefusal::LineMarkedTwice { line: 1 } + ); + for bad in [ + "[[entity:tamsin|Tamsin, bring the lantern.", + "[[entity:tamsin Tamsin]], bring the lantern.", + "[[tamsin|Tamsin]], bring the lantern.", + "[[entity:tamsin|]]Tamsin, bring the lantern.", + "Tamsin, bring the lantern", + ] { + assert_eq!(refused(WORK, &[marked(1, bad)]), ImportRefusal::Markup { line: 1 }, "{bad}"); + } + assert_eq!( + refused(WORK, &[marked(1, "[[entity:stranger|Tamsin]], bring the lantern.")]), + ImportRefusal::NotInCast { line: 1 } + ); + let group = WORK.replace(r#""group":"s1""#, r#""group":"S 1""#); + assert_eq!(refused(&group, &[ok]), ImportRefusal::Group { line: 1 }); + let bracketed = WORK.replace("Tamsin, bring the lantern.", "Tamsin [[bring]] the lantern."); + assert_eq!( + refused(&bracketed, &[marked(1, "[[entity:tamsin|Tamsin]] [[entity:orsa|bring]] the lantern.")]), + ImportRefusal::Markup { line: 1 } + ); + // The refusal text names the kind and the line, no value. + assert_eq!( + ImportRefusal::Markup { line: 1 }.to_string(), + "the marked text of line 1 is not its text with markup" + ); + } + + #[test] + fn markup_offsets_are_bytes_of_the_nfc_text() { + let work = WORK + .replace("Tamsin, bring the lantern.", "Caf\u{e9} Zo\u{eb}") + .replace(r#""forms":["Orsa"]"#, r#""forms":["Zoe\u0308"]"#); + let decomposed = "Cafe\u{301} [[entity:orsa|Zoe\u{308}]]"; + let (entities, items) = import(&work, &[marked(1, decomposed)]).unwrap(); + assert_eq!(entities[0].forms[0].forms, ["Zo\u{eb}"]); + let item = &items[0]; + assert_eq!(item.text, "Caf\u{e9} Zo\u{eb}"); + let n = &item.names[0]; + assert_eq!(&item.text[n.start as usize..n.end as usize], "Zo\u{eb}"); + assert_eq!(n.start, 6); + } + + #[test] + fn a_stray_close_bracket_and_a_scattered_duplicate_answer_are_refused() { + // A text holding only `]]` is not a marked line. + let close = WORK.replace("Quiet, both of you.", "Quiet ]] both of you."); + assert_eq!( + import(&close, &[marked(3, "Quiet ]] both of you.")]).unwrap_err(), + ImportRefusal::Markup { line: 3 } + ); + // The answer's order does not matter, and a duplicate need not be adjacent. + let (_, items) = import(WORK, &[marked(3, "Quiet, both of you."), marked(1, "Tamsin, bring the lantern.")]).unwrap(); + assert_eq!(items.iter().map(|i| i.id.as_str()).collect::>(), ["item:lantern-wager:1", "item:lantern-wager:3"]); + assert_eq!( + import( + WORK, + &[marked(1, "Tamsin, bring the lantern."), marked(3, "Quiet, both of you."), marked(1, "Tamsin, bring the lantern.")] + ) + .unwrap_err(), + ImportRefusal::LineMarkedTwice { line: 1 } + ); + } } diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index e6a6cbb..21693d5 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -1,14 +1,14 @@ //! The `weksa` command line: the same validator and store admission the library //! exposes, plus text projections (`dump`) that are for review and are not stores. -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use anyhow::{Context, Result, bail}; use clap::error::{ContextKind, ContextValue, ErrorKind}; use clap::{Args, Parser, Subcommand, ValueEnum}; use unicode_normalization::UnicodeNormalization; use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn}; -use weksa::encode::{EditDraft, OntologyDraft}; +use weksa::encode::{EditDraft, MarkedLine, OntologyDraft, WorkDraft}; use weksa::entity::{Entity, EntityOrigin, LanguageForms, Pronunciation}; use weksa::inventory::{proof_revision, seed, speaker_scene_revision}; use weksa::packet::{FeatValue, FeatureValue, Grade}; @@ -20,7 +20,7 @@ use weksa::run::{ use weksa::score::ScoreInput; use weksa::{ Annotator, CorpusStore, EntityId, EntityStore, Family, InventoryStore, OllamaEmbedder, OntologyStore, OptionLabel, - PacketStore, PairRelation, Party, Scene, Slot, SpeakerStateStore, ValueKind, Scorer, Split, SplitScheme, StoreError, lower, pronounce, validate, + PacketStore, PairRelation, Party, Scene, Slot, SpeakerStateStore, ValueKind, Scorer, Split, SplitScheme, StoreError, StoreRefusal, lower, pronounce, validate, }; #[derive(Parser)] @@ -242,6 +242,22 @@ enum CorpusCommand { Withdraw { key: String, }, + /// Import a transcribed public-domain work: mint its cast as entities and + /// add each line a name-span answer covers (`docs/name-span-annotation.md`). + Import { + /// The work file: provenance, cast, lines. + work: PathBuf, + /// The annotator's answer: one `{"line": N, "marked": "..."}` per line. + names: PathBuf, + #[arg(long, requires = "generator", conflicts_with = "operator")] + model_family: Option, + /// The attestation string naming the annotating model. + #[arg(long, requires = "model_family")] + generator: Option, + /// Annotated by the operator, not a model. + #[arg(long)] + operator: bool, + }, } #[derive(Subcommand)] @@ -681,6 +697,7 @@ impl AddArgs { supersedes: self.supersedes, withdrawn: false, names, + names_by: None, }) } } @@ -698,6 +715,22 @@ fn split_text(item: &CorpusItem) -> String { .map_or_else(|| "excluded".to_string(), |s| format!("{s:?}")) } +/// The annotator the flags name: a model family with its generator, or the operator. +fn annotator( + model_family: Option, + generator: Option, + operator: bool, +) -> Result { + match (model_family, generator, operator) { + (Some(family), Some(generator), false) => Ok(Annotator::Model { + family: family.into(), + generator, + }), + (None, None, true) => Ok(Annotator::Operator), + _ => bail!("name the annotator: --model-family with --generator, or --operator"), + } +} + fn refused(e: StoreError) -> anyhow::Error { anyhow::anyhow!("{e}") } @@ -737,13 +770,78 @@ fn main() -> Result<()> { .map_err(refused)?; println!("{outcome:?} {key}"); } + Command::Corpus(CorpusCommand::Import { + work, + names, + model_family, + generator, + operator, + }) => { + let annotator = annotator(model_family, generator, operator)?; + let draft = WorkDraft::parse(&std::fs::read_to_string(&work)?)?; + let mut marked = Vec::new(); + for (i, line) in std::fs::read_to_string(&names)?.lines().enumerate() { + if line.trim().is_empty() { + continue; + } + marked.push( + MarkedLine::parse(line) + .map_err(|e| anyhow::anyhow!("names line {}: {e}", i + 1))?, + ); + } + let (new_entities, items) = draft.into_import(&marked, &annotator)?; + // Every document passes the real doors on scratch copies of the + // stores before the first write, so a refusal leaves them untouched. + let scratch = std::env::temp_dir().join(format!("weksa-import-{}", std::process::id())); + std::fs::create_dir_all(&scratch)?; + let dry = (|| -> Result<()> { + let copy = |from: &Path, name: &str| -> Result { + let to = scratch.join(name); + if from.exists() { + std::fs::copy(from, &to)?; + } + Ok(to) + }; + let mut entities = EntityStore::open(©(&cli.entities, "e.cc")?)?; + let mut store = CorpusStore::open(©(&cli.corpus, "c.cc")?)?; + for e in new_entities.clone() { + entities + .put(e) + .map_err(|e| anyhow::anyhow!("a cast entity: {e}"))?; + } + for item in items.clone() { + let key = item.id.clone(); + store.add(item, &entities).map_err(|e| match e { + StoreError::Refused(StoreRefusal::S6TextHeld { held_by, .. }) => { + anyhow::anyhow!("{key} repeats the text of {held_by}") + } + e => anyhow::anyhow!("{key}: {e}"), + })?; + } + Ok(()) + })(); + let _ = std::fs::remove_dir_all(&scratch); + dry?; + let mut entities = EntityStore::open(&cli.entities)?; + for e in new_entities { + let id = e.id.clone(); + let outcome = entities.put(e).map_err(refused)?; + println!("{outcome:?} {id}"); + } + let mut store = CorpusStore::open(&cli.corpus)?; + for item in items { + let (key, split) = (item.id.clone(), split_text(&item)); + let outcome = store.add(item, &entities).map_err(refused)?; + println!("{outcome:?} {key} {split}"); + } + } Command::Corpus(CorpusCommand::Dump) => { for i in CorpusStore::open(&cli.corpus)?.all()? { let split = split_text(&i); let withdrawn = if i.withdrawn { " withdrawn" } else { "" }; println!( - "{} [{} {} group={} {split}{withdrawn}]\n {:?}\n names: {:?}\n source: {:?}", - i.id, i.language, i.speaker, i.group, i.text, i.names, i.source + "{} [{} {} group={} {split}{withdrawn}]\n {:?}\n names: {:?} by {:?}\n source: {:?}", + i.id, i.language, i.speaker, i.group, i.text, i.names, i.names_by, i.source ); } } @@ -880,14 +978,7 @@ fn main() -> Result<()> { generator, operator, }) => { - let annotator = match (model_family, generator, operator) { - (Some(family), Some(generator), false) => Annotator::Model { - family: family.into(), - generator, - }, - (None, None, true) => Annotator::Operator, - _ => bail!("name the annotator: --model-family with --generator, or --operator"), - }; + let annotator = annotator(model_family, generator, operator)?; let inventory_store = InventoryStore::open(&cli.inventory)?; let inventory = inventory_store.inventory()?; let corpus = CorpusStore::open(&cli.corpus)?; diff --git a/crates/weksa/src/split.rs b/crates/weksa/src/split.rs index 898439d..d22d3db 100644 --- a/crates/weksa/src/split.rs +++ b/crates/weksa/src/split.rs @@ -208,6 +208,7 @@ pub(crate) mod fixture { language: "en-US".into(), speaker: EntityId::parse("entity:nibu").unwrap(), names: vec![], + names_by: None, group: group.into(), prior_turns: vec![], source, diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index 56fcc5b..9c95e70 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -344,6 +344,9 @@ impl CorpusStore { if let Some(why) = empty_attestation(&item.source) { return Err(malformed(&item.id, why)); } + if let Some(by) = &item.names_by { + check_annotator(&item.id, by)?; + } check_entities(&item, entities)?; for held in self.all()? { if normalized(&held.text) == text && item.supersedes.as_deref() != Some(&held.id) { diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 7f624b8..bfcd2d8 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -2164,3 +2164,130 @@ fn inventory_admit_cap_is_256_and_a_refusal_writes_nothing() { assert!(admitted.status.success(), "{}", out(&admitted)); assert!(out(&admitted).contains("admitted rev:2 with 256 entries")); } + +#[test] +fn corpus_import_labels_spans_and_is_idempotent() { + let d = tempfile::tempdir().unwrap(); + let work = d.path().join("work.json"); + std::fs::write( + &work, + r#"{"slug":"lantern-wager","language":"en-US","admitted_on":"2026-11-13", + "source":{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}}, + "cast":[{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:tamsin","forms":["Tamsin","Tam"]}], + "lines":[{"group":"s1","speaker":"entity:orsa","text":"Tamsin, bring the lantern."}, + {"group":"s1","speaker":"entity:tamsin","text":"Orsa, I will not."}]}"#, + ) + .unwrap(); + let names = d.path().join("names.jsonl"); + std::fs::write( + &names, + concat!( + r#"{"line":1,"marked":"[[entity:tamsin|Tamsin]], bring the lantern."}"#, "\n\n", + r#"{"line":2,"marked":"[[entity:orsa|Orsa]], I will not."}"#, "\n", + ), + ) + .unwrap(); + let (w, n) = (work.to_str().unwrap(), names.to_str().unwrap()); + let bare = weksa(d.path(), &["corpus", "import", w, n]); + assert!(!bare.status.success()); + let flags = ["--model-family", "anthropic", "--generator", "a name annotator"]; + let run = || weksa(d.path(), &[&["corpus", "import", w, n][..], &flags].concat()); + let first = run(); + assert!(first.status.success(), "{}", out(&first)); + let text = out(&first); + assert!(text.contains("Added item:lantern-wager:1 ") && text.contains("Added item:lantern-wager:2 "), "{text}"); + assert!(text.contains("Added entity:orsa"), "{text}"); + let second = out(&run()); + assert!(second.contains("Unchanged item:lantern-wager:1 ") && !second.contains("Added"), "{second}"); + let dump = out(&weksa(d.path(), &["corpus", "dump"])); + assert!(dump.contains("by Some(Model"), "{dump}"); +} + +const LANTERN_CAST: &str = r#""cast":[{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:tamsin","forms":["Tamsin","Tam"]}]"#; +const LANTERN_SOURCE: &str = r#""source":{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}}"#; + +/// Writes a work of `lines` (text per line, all said by Orsa) and an answer of +/// `marked` per line, returns both paths. +fn lantern(dir: &std::path::Path, lines: &[&str], marked: &[&str]) -> (String, String) { + let ls: Vec = lines + .iter() + .map(|t| format!(r#"{{"group":"s1","speaker":"entity:orsa","text":"{t}"}}"#)) + .collect(); + let work = dir.join("w.json"); + std::fs::write( + &work, + format!( + r#"{{"slug":"lantern-wager","language":"en-US","admitted_on":"2026-11-13",{LANTERN_SOURCE},{LANTERN_CAST},"lines":[{}]}}"#, + ls.join(",") + ), + ) + .unwrap(); + let names = dir.join("n.jsonl"); + let rows: Vec = marked + .iter() + .enumerate() + .map(|(i, m)| format!(r#"{{"line":{},"marked":"{m}"}}"#, i + 1)) + .collect(); + std::fs::write(&names, rows.join("\n")).unwrap(); + (work.to_str().unwrap().into(), names.to_str().unwrap().into()) +} + +const MODEL: [&str; 4] = ["--model-family", "anthropic", "--generator", "a name annotator"]; + +#[test] +fn a_refused_import_writes_nothing_and_a_redraw_imports() { + let d = tempfile::tempdir().unwrap(); + let lines = ["Tamsin, bring the lantern.", "Tamsin's lantern, said Orsa."]; + let (w, n) = lantern(d.path(), &lines, &["[[entity:tamsin|Tamsin]], bring the lantern.", "[[entity:tamsin|Tamsin's]] lantern, said Orsa."]); + let import = |w: &str, n: &str| weksa(d.path(), &[&["corpus", "import", w, n][..], &MODEL].concat()); + let bad = import(&w, &n); + assert!(!bad.status.success()); + assert_eq!(out(&weksa(d.path(), &["corpus", "dump"])).trim(), ""); + assert!(!d.path().join("e.cc").exists() || EntityStore::open(&d.path().join("e.cc")).unwrap().all().unwrap().is_empty()); + // The redrawn answer, different on line 1 too, imports. + let (w, n) = lantern(d.path(), &lines, &["Tamsin, bring the lantern.", "[[entity:tamsin|Tamsin]]'s lantern, said Orsa."]); + let good = import(&w, &n); + assert!(good.status.success(), "{}", out(&good)); + assert!(out(&good).contains("Added item:lantern-wager:2")); +} + +#[test] +fn a_work_repeating_a_line_is_refused_naming_both_lines() { + let d = tempfile::tempdir().unwrap(); + let (w, n) = lantern(d.path(), &["No, my lord.", "No, my lord!", "Quiet, both of you."], &["No, my lord.", "No, my lord!", "Quiet, both of you."]); + let o = weksa(d.path(), &[&["corpus", "import", &w, &n][..], &MODEL].concat()); + assert!(!o.status.success()); + let err = String::from_utf8_lossy(&o.stderr).to_string(); + assert!(err.contains("item:lantern-wager:2") && err.contains("item:lantern-wager:1"), "{err}"); + assert_eq!(out(&weksa(d.path(), &["corpus", "dump"])).trim(), ""); +} + +#[test] +fn the_annotator_flags_decide_the_label() { + let d = tempfile::tempdir().unwrap(); + let (w, n) = lantern(d.path(), &["Quiet, both of you."], &["Quiet, both of you."]); + let none = weksa(d.path(), &["corpus", "import", &w, &n]); + assert!(!none.status.success()); + let blank = weksa(d.path(), &["corpus", "import", &w, &n, "--model-family", "anthropic", "--generator", " "]); + assert!(!blank.status.success()); + assert_eq!(out(&weksa(d.path(), &["corpus", "dump"])).trim(), ""); + let op = weksa(d.path(), &["corpus", "import", &w, &n, "--operator"]); + assert!(op.status.success(), "{}", out(&op)); + let dump = out(&weksa(d.path(), &["corpus", "dump"])); + assert!(dump.contains("by Some(Operator)"), "{dump}"); +} + +#[test] +fn an_import_refused_on_a_cast_entity_leaves_both_stores_untouched() { + let d = tempfile::tempdir().unwrap(); + // entity:tamsin is already held by a cast origin, so the work's PublicDomain + // claim on it (cast position 2, after orsa) is refused by EntityStore::put. + cast_entities(d.path(), &["tamsin"]); + let (w, n) = lantern(d.path(), &["Quiet, both of you."], &["Quiet, both of you."]); + let before = std::fs::read(d.path().join("e.cc")).unwrap(); + let o = weksa(d.path(), &[&["corpus", "import", &w, &n][..], &MODEL].concat()); + assert!(!o.status.success()); + assert!(String::from_utf8_lossy(&o.stderr).contains("a cast entity"), "{}", String::from_utf8_lossy(&o.stderr)); + assert_eq!(std::fs::read(d.path().join("e.cc")).unwrap(), before); + assert!(!d.path().join("c.cc").exists() || CorpusStore::open(&d.path().join("c.cc")).unwrap().all().unwrap().is_empty()); +} diff --git a/crates/weksa/tests/common/mod.rs b/crates/weksa/tests/common/mod.rs index fa2deb9..8ee0596 100644 --- a/crates/weksa/tests/common/mod.rs +++ b/crates/weksa/tests/common/mod.rs @@ -205,6 +205,7 @@ pub fn item_in(key: &str, text: &str, group: &str, source: Source) -> CorpusItem language: "en-US".into(), speaker: eid("nibu"), names: vec![], + names_by: None, group: group.into(), prior_turns: vec![], source, diff --git a/crates/weksa/tests/entities.rs b/crates/weksa/tests/entities.rs index 5f1f7cf..9d316bf 100644 --- a/crates/weksa/tests/entities.rs +++ b/crates/weksa/tests/entities.rs @@ -502,3 +502,96 @@ fn a_malformed_pronunciation_is_refused() { } assert_eq!(store.get(&eid("kay")).unwrap(), None); } + +const LANTERN: &str = r#"{"slug":"lantern-wager","language":"en-US","admitted_on":"2026-11-13", + "source":{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}}, + "cast":[{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:tamsin","forms":["Tamsin","Tam"]}], + "lines":[{"group":"s1","speaker":"entity:orsa","text":"Tamsin, bring the lantern."}, + {"group":"s1","speaker":"entity:tamsin","text":"Tamsin's lantern, said Orsa."}, + {"group":"s2","speaker":"entity:orsa","text":"Quiet, both of you."}]}"#; + +fn marked(line: u32, text: &str) -> weksa::encode::MarkedLine { + weksa::encode::MarkedLine { line, marked: text.into() } +} + +#[test] +fn an_imported_work_passes_the_store_doors() { + use weksa::Annotator; + use weksa::corpus::ModelFamily; + let d = tempfile::tempdir().unwrap(); + let mut entities = EntityStore::open(&d.path().join("e.cc")).unwrap(); + let mut corpus = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + let model = Annotator::Model { family: ModelFamily::Anthropic, generator: "g".into() }; + let work = || weksa::encode::WorkDraft::parse(LANTERN).unwrap(); + let (cast, items) = work() + .into_import(&[marked(1, "[[entity:tamsin|Tamsin]], bring the lantern.")], &model) + .unwrap(); + for e in cast { + assert_eq!(entities.put(e).unwrap(), AddOutcome::Added); + } + for i in items.clone() { + assert_eq!(corpus.add(i, &entities).unwrap(), AddOutcome::Added); + } + drop(corpus); + let mut corpus = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + let stored = corpus.get("item:lantern-wager:1").unwrap().unwrap(); + assert_eq!(stored.names_by, Some(model.clone())); + assert_eq!(stored.names.len(), 1); + assert_eq!(corpus.add(items[0].clone(), &entities).unwrap(), AddOutcome::Unchanged); + + // A form the entity does not list is the store's refusal, not the importer's. + let (_, bad) = work() + .into_import(&[marked(2, "[[entity:tamsin|Tamsin's]] lantern, said Orsa.")], &model) + .unwrap(); + assert!(matches!( + refusal(corpus.add(bad[0].clone(), &entities)), + StoreRefusal::NameSpanNotAForm { .. } + )); + + // An operator run over another line labels its own spans. + let (_, op) = work() + .into_import(&[marked(3, "Quiet, both of you.")], &Annotator::Operator) + .unwrap(); + assert_eq!(corpus.add(op[0].clone(), &entities).unwrap(), AddOutcome::Added); + assert_eq!( + corpus.get("item:lantern-wager:3").unwrap().unwrap().names_by, + Some(Annotator::Operator) + ); + // The first run's line keeps its label. + assert_eq!(corpus.get("item:lantern-wager:1").unwrap().unwrap().names_by, Some(model)); +} + +#[test] +fn committed_items_read_with_no_names_label() { + let data = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data"); + for sub in ["daodejing", "original"] { + let d = tempfile::tempdir().unwrap(); + let copy = d.path().join("c.cc"); + std::fs::copy(data.join(sub).join("corpus.cc"), ©).unwrap(); + let items = CorpusStore::open(©).unwrap().all().unwrap(); + assert!(!items.is_empty(), "{sub}"); + assert!(items.iter().all(|i| i.names_by.is_none()), "{sub}"); + } +} + +#[test] +fn a_blank_names_by_generator_is_refused_by_the_corpus_door() { + use weksa::Annotator; + use weksa::corpus::ModelFamily; + let d = tempfile::tempdir().unwrap(); + let mut entities = EntityStore::open(&d.path().join("e.cc")).unwrap(); + let mut corpus = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + let blank = Annotator::Model { family: ModelFamily::Anthropic, generator: " ".into() }; + let (cast, items) = weksa::encode::WorkDraft::parse(LANTERN) + .unwrap() + .into_import(&[marked(3, "Quiet, both of you.")], &blank) + .unwrap(); + for e in cast { + entities.put(e).unwrap(); + } + assert!(matches!( + refusal(corpus.add(items[0].clone(), &entities)), + StoreRefusal::Malformed { .. } + )); + assert!(corpus.all().unwrap().is_empty()); +} diff --git a/crates/weksa/tests/split_schema.rs b/crates/weksa/tests/split_schema.rs index bb6bc78..19749de 100644 --- a/crates/weksa/tests/split_schema.rs +++ b/crates/weksa/tests/split_schema.rs @@ -9,6 +9,7 @@ fn item(group: &str, source: Source) -> CorpusItem { language: "en-US".into(), speaker: weksa::EntityId::parse("entity:s").unwrap(), names: vec![], + names_by: None, group: group.into(), prior_turns: vec![], source, diff --git a/docs/name-span-annotation.md b/docs/name-span-annotation.md new file mode 100644 index 0000000..546d57b --- /dev/null +++ b/docs/name-span-annotation.md @@ -0,0 +1,52 @@ +# Name-span annotation + +How the name spans of an imported public-domain work are placed. A span has a +committed rule that produced it; this file is that rule. + +## Why + +- `pd-annotation-model-then-attest`: a labelled model places the spans of a + public-domain work; the operator may attest them afterwards. +- `leak-owned-by-flow-and-scorer-3`: spans are fixed with the text at intake and + written by no loop writer. The offsets are derived from markup the annotator + writes, never typed by a model (inline marking, as in GPT-NER, Wang et al. + 2023); `CorpusStore::add` judges range, grapheme boundary, form equality and + overlap. + +The work file (`weksa corpus import `) is a transcription of a work in +hand: provenance, cast forms, lines, scene groups and the speaker each speech +prefix names. It holds no distributor header, licence or trademark text, and the +binary fetches nothing. + +## Rubric r1 + +You are given a work's cast: each member's entity id and the forms the work spells that member with. For each numbered line, copy the line exactly, character for character, and wrap each place where the line names a cast member by one of that member's listed forms as [[|]]. Mark a form only where it refers to that cast member; a listed form used as an ordinary word, or naming someone else, stays unmarked. Do not mark pronouns, titles or descriptions unless the exact text is a listed form. When a form is listed for two members, mark the one the line refers to; when you cannot tell, leave it unmarked. Answer one JSON line per input line, in the order given, and nothing else: {"line": N, "marked": "..."}. + +## Procedure + +- One fresh model agent per work. Its brief is rubric r1, the cast as + `: , ` lines and the work's lines as ``; + nothing else (no store, no loop, no other answer). +- Import with `--model-family` and `--generator ' (name-span + rubric r1, docs/name-span-annotation.md)'`. An operator-written or + operator-reviewed answer imports with `--operator`. +- Import is all or nothing: every entity and item passes the real store doors on + scratch copies before the first write, so a refusal (a parse failure, a span + the store judges, a work that repeats a line under case or width folding) + leaves the stores untouched. Re-draw the whole answer once, log it, and import + again. A work file repeating a line's text is refused naming both lines; leave + that line out of the answer file (the rest imports with its numbering intact), + since a repeated line cannot be two corpus items. +- A line the answer does not cover is not imported by that run; a later run over + other lines of the work does not collide (keys are `item::`). +- A line whose own text holds `[[` or `]]` cannot be marked and is refused; it + can still enter by hand through `weksa corpus add --name`. + +## Label scope + +`names_by` labels the spans only, not the line or its speaker. One label per +item: a later attestation would be a superseding item with the same text and +group, but no command writes one yet; question `pd-name-span-attestation` owns +that door. +`None` means unrecorded (the cast seeds, `weksa corpus add`, the Dao De Jing +bed), not attested. diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 82d53f2..1600f6b 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -38,7 +38,7 @@ means no store holds it and nothing writes it. | Community (cut `language-ontology`, follow-up `language-ontology-communities`) | Key `community:`; type `weksa.community`, built fresh when the cut is mapped: `data/cultural-ontology/` (draft YAML profiles, read by no code) is deleted from main (cut 5e `delete-cultural-ontology`) and stays in git history before that cut's commit | Revised by commit; a speaker names the communities it belongs to by id | The cultural ontology's author (owner-attested evidence only; Discord-only content stays deleted, `discord-only-content-deleted`) | | Speaker (cut `speaker-scene`) | Key `speaker:@`; type `weksa.speaker`: stable axes as feature values of slot `Speaker` (origin, socioeconomic standing, age cohort, education and whatever the loop adds), at an inventory revision, and an `Annotator` | A new revision appends, never edits; a corpus item and every prior turn name a speaker slug, and a round records the speaker revisions it read | The annotator it names: `PersonaSnapshot` (a projection of a `gamecult.persona_state.v0` document, naming its source document and `updatedAt`), `Operator`, `WorkMetadata` (a public-domain work's own cast list and narration), or `Model { family, generator }` (the 2b `ModelFamily`) | | Scene (cut `speaker-scene`) | Key `scene:@`, the item's split group; type `weksa.scene`: participants (speaker slugs) and, per ordered pair, relation features of slot `Scene` (relative status, intimacy, in-group, audience and whatever the loop adds), at an inventory revision (provenance; a run reads the features at its own head), and an `Annotator` | As Speaker. Keyed by the group, so a scene never straddles the split. `data/original/speaker-state.cc` holds Model-annotated keeper scenes, one per Train or Validation group of both languages (`weksa scene put|dump`); a Test group has none and reads `NoScene` | As Speaker; a model-annotated scene is never Test evidence | -| Corpus item | Key `item::`, `n` canonical decimal >= 1 (no sign, no leading zero, from cut 2b); type `weksa.corpus_item` with text, language, speaker, split group, prior turns, source. Source `ModelWritten` carries a `ModelFamily` enum, the generator string as the fixture wrote it, and a prompt digest (cut 2b). The item carries no speaker state of its own: its speaker slug and group name the Speaker and Scene documents | Admitted once; the whole item is immutable (re-adding an identical item is a no-op, any difference is refused; a correction is a new item that `supersedes` the old); text already held under another key is refused unless the new item supersedes that key (cut 2b); withdrawn, never deleted, so old scores stay explainable | Importers and `weksa corpus add`, under `corpus-cast-plus-public-domain`; the source enum has no variant for Discord or any training-forbidden source (`corpus-clean`) | +| Corpus item | Key `item::`, `n` canonical decimal >= 1 (no sign, no leading zero, from cut 2b); type `weksa.corpus_item` with text, language, speaker, split group, prior turns, source and who placed its name spans (`names_by`, an `Annotator`, absent when unrecorded; cut `pd-name-spans`). Source `ModelWritten` carries a `ModelFamily` enum, the generator string as the fixture wrote it, and a prompt digest (cut 2b). The item carries no speaker state of its own: its speaker slug and group name the Speaker and Scene documents | Admitted once; the whole item is immutable (re-adding an identical item is a no-op, any difference is refused; a correction is a new item that `supersedes` the old); text already held under another key is refused unless the new item supersedes that key (cut 2b); withdrawn, never deleted, so old scores stay explainable | Importers and `weksa corpus add`, under `corpus-cast-plus-public-domain`; the source enum has no variant for Discord or any training-forbidden source (`corpus-clean`) | | Split assignment | Derived: `split(scheme, item.group)` | Fixed for a scheme; a new scheme is a new constant, never a reassignment. A model-written item whose group derives test is in no split; a superseding item keeps its predecessor's group | The hash function. Nobody assigns by hand, so there is no writer to forbid | | Loop round | Key `run:{e(f['title'])}" for f in data["fixtures"]) -b = data["build"] -page = f""" -Weksa: show her working -

Show her working

-

A meaning packet goes in. The English lowerer picks inventory entries, writes a sentence, and a scorer measures how close that sentence is to a human-written target. Every number and sentence below was produced by running the real Rust lowerer and scorer when this page was built. The refusals are results too: when the lowerer cannot say a meaning without changing it, it says so instead of dropping the part it cannot say.

-

Built from {e(b['commit'][:10])}, embedder {e(b['embedder'])}. English only. Fixtures are the committed nibu-proof-001 packet and variants of it, each variant described in its own card. Scores are against the committed corpus: chrF (0-100) and embedding cosine. Raw data: data.json.

- -{''.join(card(f) for f in data['fixtures'])} -""" -(out_dir / "index.html").write_text(page, encoding="utf-8") -print(f"wrote {out_dir}/index.html and data.json ({len(data['fixtures'])} fixtures)") From e7775541fe43200b0690429b7b668ad85da1c493 Mon Sep 17 00:00:00 2001 From: Weksa Date: Sat, 14 Nov 2026 04:21:49 +0000 Subject: [PATCH 20/21] V18: validate refuses a request_intervention predication the addressee consent does not name (#94) --- crates/weksa/src/validate.rs | 32 ++++++++++++++++++++++- crates/weksa/tests/context.rs | 35 +++++++++++++++++++++++++ crates/weksa/tests/display.rs | 1 + crates/weksa/tests/validate.rs | 47 ++++++++++++++++++++++++++++++++++ docs/interlingua-standard.md | 19 ++++++++------ 5 files changed, 125 insertions(+), 9 deletions(-) diff --git a/crates/weksa/src/validate.rs b/crates/weksa/src/validate.rs index 49c670f..71c92d5 100644 --- a/crates/weksa/src/validate.rs +++ b/crates/weksa/src/validate.rs @@ -4,7 +4,7 @@ //! lowerer sees, comes from `PacketStore::admit` alone, which builds it from //! the admitted packet and its corpus item through the crate-private //! `LoweringInput::of`. The stance is checked like every feature -//! list, in slot `Stance`. The rules are V1..V8, V10..V12 and V14..V17: the +//! list, in slot `Stance`. The rules are V1..V8, V10..V12 and V14..V18: the //! shape of an id or a grade is its type's parse, not a rule, so V9 is retired //! and not renumbered; V13 is retired too. [`LoweringInput::with_scene`] reads the //! scene of the view's own item group from the speaker-state store, and @@ -120,6 +120,12 @@ pub enum Refusal { V17AddresseeNotAReferent { id: NodeId, }, + /// V18: a predication whose frame is the inventory's `request_intervention`, + /// with no `addressee_consent` naming it. It says what the addressee did, so + /// only the packet's own record may license it. + V18RequestWithoutConsent { + predication: NodeId, + }, } impl Refusal { @@ -144,6 +150,7 @@ impl Refusal { Refusal::V15ConsentSource { .. } => "V15", Refusal::V16SpeakerNotInScene => "V16", Refusal::V17AddresseeNotAReferent { .. } => "V17", + Refusal::V18RequestWithoutConsent { .. } => "V18", } } } @@ -173,6 +180,9 @@ impl fmt::Display for Refusal { "the scene does not cover the speaker of every live item of its group" } Refusal::V17AddresseeNotAReferent { .. } => "the addressee names no referent", + Refusal::V18RequestWithoutConsent { .. } => { + "a request frame needs the addressee's consent naming it" + } }; write!(f, "{}: {what}", self.rule()) } @@ -221,6 +231,7 @@ pub fn validate(packet: &Packet, inventory: &Inventory) -> Result<(), Vec, out: &mut Vec) { + for p in &packet.predications { + if inventory + .resolve(&p.frame, packet.inventory_rev) + .is_some_and(|e| e.gloss == REQUEST_FRAME_GLOSS) + && packet.discourse.addressee_consent.as_ref() != Some(&p.id) + { + out.push(Refusal::V18RequestWithoutConsent { + predication: p.id.clone(), + }); + } + } +} + fn node_lists(packet: &Packet) -> [(&Vec, &'static str); 3] { [ (&packet.context.participants, "participant"), diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs index d2550de..a8993fc 100644 --- a/crates/weksa/tests/context.rs +++ b/crates/weksa/tests/context.rs @@ -593,6 +593,41 @@ mod lowering { assert!(Prompt::lower(&read, &item, &away(), "en-US", &inv, &held, &empty).is_ok()); } + /// The packet the encoder emits for a request: frame:3, no consent, source + /// ModelDecomposition. Admission refuses it, so no lowering view exists. + #[test] + fn an_encoder_request_never_reaches_a_lowerer() { + let cast = InventoryStore::open(&data("interlingua/inventory.cc")).unwrap(); + let entities = EntityStore::open(&data("entity/entity.cc")).unwrap(); + let corpus = CorpusStore::open(&data("corpus/corpus.cc")).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let mut store = PacketStore::open(&dir.path().join("p.cc")).unwrap(); + let mut packet = proof_packet(); + packet.provenance.source = weksa::packet::ProvenanceSource::ModelDecomposition; + packet.predications[0].frame = weksa::InvId::parse("frame:3").unwrap(); + match store.admit(packet.clone(), &cast, &corpus, &entities) { + Err(weksa::StoreError::Invalid(r)) => assert_eq!( + r, + vec![weksa::Refusal::V18RequestWithoutConsent { + predication: weksa::packet::NodeId::parse("n4").unwrap() + }] + ), + other => panic!("expected V18, got {:?}", other.map(|_| ())), + } + // Not blanket: telemetry that records the consent admits and lowers. + packet.provenance.source = weksa::packet::ProvenanceSource::GameTelemetry; + packet.discourse.addressee_consent = Some(weksa::packet::NodeId::parse("n4").unwrap()); + drop(store); + let view = admitted(packet, &cast, "corpus/corpus.cc", &entities); + let item = CorpusItemId::parse("item:nibu:1").unwrap(); + let held = Held(vec![nibu(&[("en-US", &["Nibu"])])]); + let empty = weksa::Ontology::empty(); + assert!( + Prompt::lower(&view, &item, &away(), "en-US", &cast.inventory().unwrap(), &held, &empty) + .is_ok() + ); + } + fn dao(index: usize) -> (LoweringInput, Inventory, EntityStore, CorpusItemId) { let inv = InventoryStore::open(&data("daodejing/inventory.cc")).unwrap(); let entities = EntityStore::open(&data("daodejing/entity.cc")).unwrap(); diff --git a/crates/weksa/tests/display.rs b/crates/weksa/tests/display.rs index e82b5fb..de4369c 100644 --- a/crates/weksa/tests/display.rs +++ b/crates/weksa/tests/display.rs @@ -104,6 +104,7 @@ fn refusal_display_names_the_rule_and_no_identifier() { }, Refusal::V16SpeakerNotInScene, Refusal::V17AddresseeNotAReferent { id: node() }, + Refusal::V18RequestWithoutConsent { predication: node() }, ]; assert_eq!( Refusal::V16SpeakerNotInScene.to_string(), diff --git a/crates/weksa/tests/validate.rs b/crates/weksa/tests/validate.rs index b46cd85..057e92f 100644 --- a/crates/weksa/tests/validate.rs +++ b/crates/weksa/tests/validate.rs @@ -740,6 +740,53 @@ fn a_source_that_only_read_wording_cannot_set_consent() { } } +// V18: a request frame is licensed by the packet's own consent record. + +/// The committed inventory, where `frame:3` is `request_intervention`. +fn cast_refusals(p: &Packet) -> Vec { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/interlingua/inventory.cc"); + let inv = weksa::InventoryStore::open(&path).unwrap().inventory().unwrap(); + validate(p, &inv).err().unwrap_or_default() +} + +fn requesting() -> Packet { + let mut p = proof_packet(); + p.predications[0].frame = iv("frame:3"); + p.discourse.addressee_consent = Some(nd("n4")); + p +} + +#[test] +fn a_request_frame_needs_consent_naming_its_predication() { + assert_eq!(cast_refusals(&requesting()), vec![]); + let mut p = requesting(); + p.discourse.addressee_consent = None; + assert_eq!( + cast_refusals(&p), + vec![Refusal::V18RequestWithoutConsent { predication: nd("n4") }] + ); + p.discourse.addressee_consent = Some(nd("n5")); + assert_eq!( + cast_refusals(&p), + vec![Refusal::V18RequestWithoutConsent { predication: nd("n4") }] + ); + let mut p = requesting(); + p.provenance.source = ProvenanceSource::GameTelemetry; + assert_eq!(cast_refusals(&p), vec![]); +} + +#[test] +fn model_source_consent_is_v15_alone() { + let mut p = requesting(); + p.provenance.source = ProvenanceSource::ModelDecomposition; + assert_eq!( + cast_refusals(&p), + vec![Refusal::V15ConsentSource { + source: ProvenanceSource::ModelDecomposition + }] + ); +} + // V17: the addressee is someone spoken to, a referent. #[test] diff --git a/docs/interlingua-standard.md b/docs/interlingua-standard.md index cfed562..a24b02c 100644 --- a/docs/interlingua-standard.md +++ b/docs/interlingua-standard.md @@ -164,6 +164,7 @@ names the rule. `GameTelemetry`; a decomposition of text or imported corpus text cannot mint it. - V17: an `addressee` names a referent, not a predication. +- V18: a predication whose frame is the inventory's `request_intervention` is named by `addressee_consent`. V9 is retired and not renumbered: the form of a key or a grade is its type, and an extension key's family is V6. `PacketStore::admit` returns the lowering view of a @@ -197,14 +198,16 @@ whatever the inventory grows). This is source-side communicative pressure, not target-language register. `addressee_consent` is no feature: it names the predication the addressee asked for or agreed to, a fact about what the addressee did. `V14` and `V15` check the field: it may be set only by authoring or game -telemetry, and it must name a predication of the packet. Open gap: `frame:3` -(`request_intervention`) is in the committed inventories (`data/interlingua`, -`data/original`, `data/daodejing`) and seeded at `inventory.rs:518`. No -committed packet uses it, but the encoder sees it, and `V14` and `V15` check only -the consent field, so a model-encoded `frame:3` predication with a null -`addressee_consent` reaches `Prompt::lower` and nothing refuses it. The retired -gate covered only the deleted `lower()`. Follow-up -`table-retirement-consent-gap` owns the gap. +telemetry, and it must name a predication of the packet. `V18` refuses a predication whose frame resolves, at the packet's revision, to +the gloss `request_intervention` (`frame:3` in the committed inventories) +unless `addressee_consent` names that predication. The encoder can never set +consent (`V15`), so an encoder-produced request frame is always refused. +Residual gaps: `V18` keys on that one gloss, so another frame that asserts the +addressee's act (for example a `frame:2` intervention on the original +inventory) or a renamed gloss is not covered; consent is one field, so two +request predications in one packet cannot validate; and the check that the +asker is the addressee was not kept. Follow-up +`weksa-lowering:follow_up:request-frame-consent-residual` owns them. ### `stance` From 1e06f003365788c5e93194d16d9a6c123817c265 Mon Sep 17 00:00:00 2001 From: Weksa Date: Sat, 14 Nov 2026 04:29:57 +0000 Subject: [PATCH 21/21] interlingua-standard: name V18 limits, drop false frame:2 gap (#95) --- docs/interlingua-standard.md | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/docs/interlingua-standard.md b/docs/interlingua-standard.md index a24b02c..ca8b860 100644 --- a/docs/interlingua-standard.md +++ b/docs/interlingua-standard.md @@ -201,13 +201,20 @@ addressee did. `V14` and `V15` check the field: it may be set only by authoring telemetry, and it must name a predication of the packet. `V18` refuses a predication whose frame resolves, at the packet's revision, to the gloss `request_intervention` (`frame:3` in the committed inventories) unless `addressee_consent` names that predication. The encoder can never set -consent (`V15`), so an encoder-produced request frame is always refused. -Residual gaps: `V18` keys on that one gloss, so another frame that asserts the -addressee's act (for example a `frame:2` intervention on the original -inventory) or a renamed gloss is not covered; consent is one field, so two -request predications in one packet cannot validate; and the check that the -asker is the addressee was not kept. Follow-up -`weksa-lowering:follow_up:request-frame-consent-residual` owns them. +consent (`V15`), so an encoder-produced `request_intervention` predication (`frame:3`) is always refused. +Limits, tracked by `weksa-lowering:follow_up:request-frame-consent-residual`: +`V18` keys on the gloss `request_intervention` and sees no other frame or +concept, so a request-shaped meaning built from another frame or concept (a +`frame:12` communication with `concept:6` request validates) or from a renamed +or newly minted frame is not seen by `validate`, which does not read meaning. +The inventory carries no marker for request shape, so classifying it would be +an inventory-shape decision for Imagination and language-ontology. `addressee_consent` +is one field, so a packet carries one request predication, and a compound +request is two packets until a packet schema revision makes consent a set. The +asker is not checked to be the addressee: a consented `frame:3` whose agent is +not the addressee validates, and no code outside `validate` and the encoder +prompt text reads `addressee_consent`. That check is an owed obligation with no +owner yet. ### `stance`