From 9f8ae13a121f7d98aa7228609e15e5bd4ef7bea6 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 18:04:37 +0000 Subject: [PATCH 01/80] original corpus: 22 new model-written items through the corpus door (Train 14, Validation 14) Items original:7..28, written for the non-memorised target; groups chosen so the V1 split derives Train or Validation. Test is not filled: a model-written item of a test group is in no split (split.rs), so none can be held out. Co-Authored-By: Claude Sonnet 5.5 --- data/original/corpus.cc | Bin 2050 -> 8614 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/data/original/corpus.cc b/data/original/corpus.cc index c11443faa710ffebcb55952272ae15eccff27fbf..57988dfaf5fe31057a344f805d956cfc1bcde77a 100644 GIT binary patch literal 8614 zcmd6sO>Y}T7{}>_9=O1P0}_W(D*74+d*i&+7ZkN11*sLHZ=$F|WAE5Ics*lg##vd2 z1K%J~DskwU1E{41^#fEW{SYH@;Q*+JcX2>m$}=--Z@ROQSDfaSIJ2A0FVFLT{{Lt9 zzv)d<)x*7ntVqQLS=njrefD)mh7zyzxEN352UL-f|H(t|3|#6T)7L*BHTf?yt8Fk^zQs@0!pX~X+YWKV6UcG(aJZ*;yI-m@vor))V z7&W8yMyu6mHLteXoz_ODxq0*8*E{7ab#n`SG@(SH5oIx=5~XxVkW@5HQHB)}h%$sx zOvo5@d6uqxPFUmGm3xFKs-~SGd``r@*`qYfk=;-4Ug9xHZwRUsVf$ZfF_j2DragVD zm*PoG8j>?cRAa5NhECrw{|0o0r2S6ECxiKdGuxRS3ml8_x6Jlrlybc8BUv>=4vNu-8}g5j(4ek?Ja}EJVlBbaABxR z;JFB*3c)XzD8^z~N$0`ruWIC11`B9WAcl#_BP6>Owecwg~H&U$0unFl_Uro z0Jb@=j2e8X$45j~^13_cg=G>ftEg@<{p_)d^tsEt%i}3PU`p^*+PX>z-T_Rw%Xf(V zpifB}S0egqj>w(~7Sd{6NWXcIR(10#G@BC`qX#2sdH}@{8I9G{0_tK3>ZCX*g5X&t zsY03(=-*&Lwd;cV-UHRvCk!Y-cc7~TkuX0}0EN*QD+)4=Bp#3jY=$&t!LnJa%jO@C z&6+;-PD(J05-f+dWb%4~GV0ItJR@oKSi3*(vJ}TL5rGie+kbZYzVNa)>DKEa^ZRaH zue?VjD0GZnlbQ0dJx5{YCQxH$#v=@FL8fG^Dl4lXC&z-aK_KLOOR!NF)DK>5ZRnF1 zIam^fQm{|P7<8eK=9+FlF{Y{#P$A%$?a**_X|pb$Cmx?oeflCdiVb9J4LMV38qm=) zWn{JdjVe!BxTXY)E}w+beSQs{ugmCPkI{L3_QqU>tlO4EsT53Q-!jV8D#KLCN=ii? z*;yD!pOqErl@sO2vdBHnBFp1r$IF=|(%Z$n%LpD4=8CL~+0djFCpk``=I+xS5uAfl zH;HDVCL^|>HSI_Z1gTecQSCJ28PKSy&uklq0Op*t0oe9D*+@vQFe%|D&~vf1@s(9q z^k$EmQ4y?nh-xRApUS-%we%TNXEVc)Ky@7wvsomy8M(5K3g>C6F4!_!)QR)V=c!i| z>9beBc~6%QYn8E8U=)n%e~2Nrb8uRdBMjCd^%4ZffiNno*YTIi{wk`SYdqT!t?HAP za%YtXKB?e)=(>!?w)cl~(O~E$ht~8@6 zaf*-GfQVDo^;JMpP>jJs`v37JT6b#&f(7u1X>1MrSP`p zw)czX_H&x21(4}CBOvUI=CggYtsbE(j}(qm|f_ZxjdN4 zE4_tTG%e?}o|b8~^#wx<<{lb=VC-VL1c$_}KGha%=kSYYcXJwzh2Ukwdrda$RX`zab{`HENG6&-=CunN!$c zAk9LodXyFb;l`l?Tr@~$01uv&`K{}7TPT^Z0Za(D_kwqD6DYfs!wWux6+_BqQyVTT zm8|fx_WDo})gGaLlo4&{C{Nkub(qWAGcrce^4Jaf*26-WmZhpPZWpBB?KKO>mX&0z z?)*IXy-aFTw=RHRg`DV2z8jO44`H!2thP_V&%(Om=;2s^vUdozq-rk=e<_1HuU|KM gpYJ4Q_SG%HEFrrCOEH)Xw*{;0ykaVv|r>&fwNp3G|uVH8R3W&r@P(F%G1 From 073b022c8a753d4103eea7d1c05cc43b9cfc606f Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 18:04:37 +0000 Subject: [PATCH 02/80] original_corpus tests: counts over 28 items, first six pinned, recall shows only live items Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/original_corpus.rs | 88 ++++++++++++++++++++++++--- 1 file changed, 78 insertions(+), 10 deletions(-) diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 91b635b..71cbf1a 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -25,7 +25,7 @@ fn stores() -> (CorpusStore, EntityStore) { fn every_original_item_records_its_author_and_none_is_public_domain() { let (corpus, entities) = stores(); let items = corpus.all().unwrap(); - assert_eq!(items.len(), 6); + assert_eq!(items.len(), 28); for item in &items { match &item.source { Source::ModelWritten { @@ -37,12 +37,18 @@ fn every_original_item_records_its_author_and_none_is_public_domain() { } other => panic!("{}: not model-written: {other:?}", item.id), } - assert!(entities.get(&item.speaker).unwrap().is_some(), "{}", item.id); + assert!( + entities.get(&item.speaker).unwrap().is_some(), + "{}", + item.id + ); } } #[test] fn the_original_items_spread_over_train_and_validation_and_none_is_in_test() { + // Model-written items cannot be in Test (`SplitScheme::split` excludes a + // model-written item of a test group), so this store holds none. let (corpus, _) = stores(); let splits: Vec> = corpus .all() @@ -51,10 +57,57 @@ fn the_original_items_spread_over_train_and_validation_and_none_is_in_test() { .map(|i| SplitScheme::V1.live(i)) .collect(); let count = |s| splits.iter().filter(|x| **x == Some(s)).count(); - assert_eq!((count(Split::Train), count(Split::Validation)), (3, 3)); + assert_eq!((count(Split::Train), count(Split::Validation)), (14, 14)); assert_eq!(count(Split::Test), 0); } +/// The first six items are the original golf-loop set; widening the store +/// adds items after them and never edits them. +#[test] +fn the_first_six_original_items_are_unchanged() { + let (corpus, _) = stores(); + let want = [ + ( + "item:original:1", + "A quiet mind is like still water in a deep bowl.", + "original-bowl", + ), + ( + "item:original:2", + "The river weakens the bank in spring and strengthens the roots in autumn.", + "original-river", + ), + ( + "item:original:3", + "The sage waits at the door until the water has settled.", + "original-door", + ), + ( + "item:original:4", + "The old gardener empties the jar and fills it again with rainwater.", + "original-jar", + ), + ( + "item:original:5", + "Under the same sky the earth keeps its patience, and the baker keeps hers.", + "original-sky", + ), + ( + "item:original:6", + "A hungry belly argues louder than any will.", + "original-belly", + ), + ]; + for (id, text, group) in want { + let item = corpus.get(id).unwrap().unwrap(); + assert_eq!( + (item.text.as_str(), item.group.as_str()), + (text, group), + "{id}" + ); + } +} + #[test] fn the_original_inventory_is_the_dao_revision_three() { let dao = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/daodejing/inventory.cc"); @@ -69,7 +122,7 @@ fn the_original_inventory_is_the_dao_revision_three() { /// The probe over the real original corpus: every item is offered, none names a /// work, and a model that answers with unrelated words reads at the noise floor. #[test] -fn the_recall_probe_covers_all_six_original_items_and_names_no_work() { +fn the_recall_probe_covers_every_live_original_item_and_names_no_work() { use weksa::model::{ModelPort, PortError}; use weksa::run::{RecallOutcome, RunPaths, RunStores}; struct Unrelated(Vec); @@ -107,9 +160,24 @@ fn the_recall_probe_covers_all_six_original_items_and_names_no_work() { let RecallOutcome::Read(r) = run.recall(&mut model).unwrap() else { panic!("answered"); }; - assert_eq!((r.n, r.answered), (6, 6)); + assert_eq!((r.n, r.answered), (28, 28)); assert!(r.chrf < 25.0, "{}", r.chrf); - assert_eq!(model.0.len(), 6); + assert_eq!(model.0.len(), 28); + // No item outside Train and Validation is shown to the model. + let all = CorpusStore::open(&c).unwrap().all().unwrap(); + for item in &all { + let live = SplitScheme::V1.live(item); + // A recall prompt shows the first half of the words. + let words: Vec<&str> = item.text.split_whitespace().collect(); + let opening = words[..words.len() / 2].join(" "); + let shown = model.0.iter().any(|t| t.contains(&opening)); + assert_eq!( + shown, + matches!(live, Some(Split::Train | Split::Validation)), + "{}", + item.id + ); + } assert!(model.0.iter().all(|t| t.contains("work: not published"))); assert!(model.0.iter().all(|t| !t.contains("translator:"))); } @@ -168,16 +236,16 @@ fn original_round(family: ModelFamily) -> (weksa::run::RunStores, weksa::run::Ro #[test] fn own_family_counts_and_labels() { - // The original corpus is six Anthropic-written items, all offered. + // The original corpus is Anthropic-written items, all offered. let (run, r) = original_round(ModelFamily::Anthropic); - assert_eq!(r.own_family(), 6); + assert_eq!(r.own_family(), 28); assert_eq!( r.parse_reasons().iter().map(|c| c.count).sum::(), - 6, + 28, "every item was refused at Parse and says why" ); let line = run.round_line(&r); - assert!(line.contains(" own-family 6"), "{line}"); + assert!(line.contains(" own-family 28"), "{line}"); // A different answerer family, or none recorded, labels nothing. for family in [ModelFamily::OpenAi, ModelFamily::Unrecorded] { From 32368a998e114bfa93eb22a564735d3fb73f166d Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 18:09:47 +0000 Subject: [PATCH 03/80] original_corpus: recall probes only live Train/Validation over a fixture with withdrawn, model-written-Test and public-domain-Test items Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/original_corpus.rs | 90 +++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 71cbf1a..79273f5 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -2,6 +2,8 @@ //! non-memorised golf-loop target. It holds no public-domain text, so a model //! cannot have it in pretraining, and every item says who wrote it. +mod common; + use std::path::Path; use weksa::corpus::{ModelFamily, Source}; @@ -254,3 +256,91 @@ fn own_family_counts_and_labels() { assert!(!run.round_line(&r).contains("own-family")); } } + +/// Over a fixture store whose items are not all live: recall probes the live +/// Train and Validation items and no others (a withdrawn item, a model-written +/// item of a test group, a public-domain Test item). +#[test] +fn recall_probes_only_live_train_and_validation_items() { + use common::{cast, entities_in, group_in_bucket, item_in, run_over}; + use weksa::model::{ModelPort, PortError}; + use weksa::run::RecallOutcome; + struct Record(Vec); + impl ModelPort for Record { + fn model(&self) -> &str { + "record" + } + fn answer(&mut self, p: &weksa::context::Prompt) -> Result { + self.0.push(p.text().to_string()); + Ok(r#"{"text":"nothing"}"#.to_string()) + } + } + let dir = tempfile::tempdir().unwrap(); + let entities = entities_in(dir.path()); + let mut store = CorpusStore::open(&dir.path().join("c.cc")).unwrap(); + let public = Source::PublicDomain { + work: "A Work".into(), + author: "An Author".into(), + edition_url: "https://example.org/e".into(), + licence: "CC0".into(), + jurisdiction: "US".into(), + translator: "A Translator".into(), + edition_year: 1900, + }; + let rows = [ + ( + "item:f:1", + "trainmark opens here and goes on", + 10, + cast(), + false, + ), + ( + "item:f:2", + "validmark opens here and goes on", + 50, + cast(), + false, + ), + ( + "item:f:3", + "gonemark opens here and goes on", + 10, + cast(), + true, + ), + ( + "item:f:4", + "modelmark opens here and goes on", + 90, + common::model(), + false, + ), + ( + "item:f:5", + "publicmark opens here and goes on", + 90, + public, + false, + ), + ]; + for (id, text, bucket, source, withdrawn) in rows { + let item = item_in(id, text, &group_in_bucket(bucket), source); + store.add(item, &entities).unwrap(); + if withdrawn { + store.withdraw(id).unwrap(); + } + } + drop(store); + let run = run_over(dir.path()); + let mut m = Record(vec![]); + let RecallOutcome::Read(r) = run.recall(&mut m).unwrap() else { + panic!("answered"); + }; + assert_eq!(r.n, 2); + let shown = |mark: &str| m.0.iter().any(|t| t.contains(mark)); + assert!(shown("trainmark") && shown("validmark")); + for hidden in ["gonemark", "modelmark", "publicmark"] { + assert!(!shown(hidden), "recall showed {hidden}"); + } +} From e7aef81309349a3f42739b6df8897e14b2c76819 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 18:24:37 +0000 Subject: [PATCH 04/80] data/original: 8 public-domain Test items with full provenance; Test held out of rounds and recall Eight one-sentence English excerpts from Dhammapada (Muller), Walden, Emerson, Bacon, Aesop (Townsend), Mill, Plato (Jowett), Darwin, each taken from the exact Project Gutenberg page named in edition_url and checked verbatim against the fetched text. Groups hash into Test (buckets 83-97); split.rs untouched, the ModelWritten-never-in-Test rule stands. Items 1..28 are byte-identical (digest pinned). Tests: counts Train/Validation 14/14, Test 8 all PublicDomain; no Test text in any round-1 or recall prompt or RoundRecord over data/original; a ModelWritten item in a Test group is read in no split. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/original_corpus.rs | 217 ++++++++++++++++++++++++-- data/original/corpus.cc | Bin 8614 -> 13160 bytes data/original/entity.cc | 2 +- 3 files changed, 203 insertions(+), 16 deletions(-) diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 79273f5..186cb2a 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -1,6 +1,7 @@ -//! The original corpus (`data/original`): passages written for the -//! non-memorised golf-loop target. It holds no public-domain text, so a model -//! cannot have it in pretraining, and every item says who wrote it. +//! The original corpus (`data/original`): 28 passages written for the +//! non-memorised golf-loop target (Train and Validation, all model-written) +//! and 8 public-domain Test items, each with its full provenance. Test is only +//! read at a checkpoint: no round or recall prompt over this store holds it. mod common; @@ -24,20 +25,58 @@ fn stores() -> (CorpusStore, EntityStore) { } #[test] -fn every_original_item_records_its_author_and_none_is_public_domain() { +fn every_original_item_records_its_author_and_the_public_domain_ones_their_provenance() { let (corpus, entities) = stores(); let items = corpus.all().unwrap(); - assert_eq!(items.len(), 28); + assert_eq!(items.len(), 36); + let (mut model, mut public) = (0, 0); for item in &items { match &item.source { Source::ModelWritten { family, generator, .. } => { + model += 1; + assert!(item.id.starts_with("item:original:"), "{}", item.id); assert_eq!(*family, ModelFamily::Anthropic, "{}", item.id); let generator = generator.as_deref().unwrap_or_default(); assert!(generator.contains("claude-sonnet-5-5"), "{}", item.id); } - other => panic!("{}: not model-written: {other:?}", item.id), + Source::PublicDomain { + work, + author, + edition_url, + licence, + jurisdiction, + translator, + edition_year, + } => { + public += 1; + assert!(item.id.starts_with("item:holdout:"), "{}", item.id); + // The edition is the page the text was taken from. + assert!( + edition_url.starts_with("https://www.gutenberg.org/cache/epub/") + && edition_url.ends_with(".txt"), + "{}", + item.id + ); + assert!(licence.contains("Public domain"), "{}", item.id); + assert!( + licence.contains("trademark or licence matter"), + "{}", + item.id + ); + assert!(jurisdiction.starts_with("United States"), "{}", item.id); + assert!( + [work, author, translator] + .iter() + .all(|s| !s.trim().is_empty()), + "{}", + item.id + ); + // Every edition and every author is out of copyright by date. + assert!((1600..=1900).contains(edition_year), "{}", item.id); + } + other => panic!("{}: not model-written or public-domain: {other:?}", item.id), } assert!( entities.get(&item.speaker).unwrap().is_some(), @@ -45,24 +84,67 @@ fn every_original_item_records_its_author_and_none_is_public_domain() { item.id ); } + assert_eq!((model, public), (28, 8)); } #[test] -fn the_original_items_spread_over_train_and_validation_and_none_is_in_test() { - // Model-written items cannot be in Test (`SplitScheme::split` excludes a - // model-written item of a test group), so this store holds none. +fn the_original_items_spread_over_train_and_validation_and_test_is_public_domain_only() { + // A model-written item of a Test group is in no split + // (`SplitScheme::split`), so Test holds the public-domain items alone. let (corpus, _) = stores(); - let splits: Vec> = corpus + let items = corpus.all().unwrap(); + let in_split = |s| { + items + .iter() + .filter(|i| SplitScheme::V1.live(i) == Some(s)) + .collect::>() + }; + assert_eq!(in_split(Split::Train).len(), 14); + assert_eq!(in_split(Split::Validation).len(), 14); + let test = in_split(Split::Test); + assert_eq!(test.len(), 8); + assert!( + test.iter() + .all(|i| matches!(i.source, Source::PublicDomain { .. })) + ); + // Train and Validation hold no public-domain item. + for s in [Split::Train, Split::Validation] { + assert!( + in_split(s) + .iter() + .all(|i| matches!(i.source, Source::ModelWritten { .. })) + ); + } +} + +/// The 28 model-written items are byte-for-byte what they were before Test +/// was added: id, text, group and source, folded in key order. +#[test] +fn the_28_model_written_items_are_unchanged() { + let (corpus, _) = stores(); + let mut items: Vec<_> = corpus .all() .unwrap() - .iter() - .map(|i| SplitScheme::V1.live(i)) + .into_iter() + .filter(|i| i.id.starts_with("item:original:")) .collect(); - let count = |s| splits.iter().filter(|x| **x == Some(s)).count(); - assert_eq!((count(Split::Train), count(Split::Validation)), (14, 14)); - assert_eq!(count(Split::Test), 0); + items.sort_by(|a, b| a.id.cmp(&b.id)); + assert_eq!(items.len(), 28); + let lines: Vec = items + .iter() + .map(|i| format!("{}\n{}\n{}\n{:?}", i.id, i.text, i.group, i.source)) + .collect(); + let parts: Vec<&[u8]> = lines.iter().map(|l| l.as_bytes()).collect(); + assert_eq!( + weksa::split::fnv1a64_of(&parts), + DIGEST_OF_THE_28, + "an original item changed" + ); } +/// Taken over the store at base da6bc7a, before any Test item existed. +const DIGEST_OF_THE_28: u64 = 3803723767212018190; + /// The first six items are the original golf-loop set; widening the store /// adds items after them and never edits them. #[test] @@ -344,3 +426,108 @@ fn recall_probes_only_live_train_and_validation_items() { assert!(!shown(hidden), "recall showed {hidden}"); } } + +/// Collects every prompt a round or recall shows the model; its answer is no +/// draft and no recall. +struct Record(Vec); + +impl weksa::model::ModelPort for Record { + fn model(&self) -> &str { + "nope" + } + fn answer(&mut self, p: &weksa::context::Prompt) -> Result { + self.0.push(p.text().to_string()); + Ok(r#"{"text":"nothing"}"#.to_string()) + } +} + +/// The public-domain Test items are held out: no prompt of round 1 over +/// `data/original`, no recall prompt, and no round record names one. +#[test] +fn no_test_item_is_offered_to_a_round_or_recall_or_recorded() { + use weksa::run::{LowererReach, RoundOutcome, RunPaths, RunStores, round}; + let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/original"); + let tmp = tempfile::tempdir().unwrap(); + for f in ["corpus.cc", "entity.cc", "inventory.cc"] { + std::fs::copy(src.join(f), tmp.path().join(f)).unwrap(); + } + let p = |f: &str| tmp.path().join(f); + let mut run = RunStores::create( + &p("run"), + "orig", + RunPaths { + corpus: &p("corpus.cc"), + inventory: &p("inventory.cc"), + entities: &p("entity.cc"), + }, + "nope", + "bag", + LowererReach::ToolLessRelay, + ModelFamily::Unrecorded, + ) + .unwrap(); + let items = CorpusStore::open(&p("corpus.cc")).unwrap().all().unwrap(); + let held: Vec<_> = items + .iter() + .filter(|i| SplitScheme::V1.live(i) == Some(Split::Test)) + .collect(); + assert_eq!(held.len(), 8); + // A Test item's text, or any run of five of its words, in a prompt. + let leaks = |prompts: &[String], item: &weksa::corpus::CorpusItem| { + let w: Vec<&str> = item.text.split_whitespace().collect(); + prompts + .iter() + .any(|t| t.contains(&item.text) || w.windows(5).any(|k| t.contains(&k.join(" ")))) + }; + let scorer = weksa::score::Scorer::new(Bag); + let mut model = Record(vec![]); + let RoundOutcome::Recorded(r) = round(&mut run, &mut model, &scorer).unwrap() else { + panic!("answered"); + }; + // The check bites: the round did show live items, 28 of them. + assert!(model.0.len() >= 28, "{}", model.0.len()); + assert!( + items + .iter() + .filter(|i| matches!(SplitScheme::V1.live(i), Some(Split::Train))) + .any(|i| leaks(&model.0, i)), + "the probe cannot see a shown Train item" + ); + for item in &held { + assert!(!leaks(&model.0, item), "round prompt held {}", item.id); + assert!(r.calls().iter().all(|c| !c.items.contains(&item.id))); + assert!(r.train().iter().all(|row| row.item != item.id)); + } + let mut recall = Record(vec![]); + let weksa::run::RecallOutcome::Read(rr) = run.recall(&mut recall).unwrap() else { + panic!("answered"); + }; + assert_eq!((rr.n, rr.answered), (28, 28)); + for item in &held { + assert!(!leaks(&recall.0, item), "recall prompt held {}", item.id); + } +} + +/// The exclusion stands: a model-written item in a Test group is read in no +/// split, and is not moved to Train or Validation. +#[test] +fn a_model_written_item_in_a_test_group_is_read_in_no_split() { + use common::{group_in_bucket, item_in, model}; + for bucket in [80, 99] { + let item = item_in("item:f:1", "x opens", &group_in_bucket(bucket), model()); + assert_eq!(SplitScheme::V1.split(&item), None, "bucket {bucket}"); + assert_eq!(SplitScheme::V1.live(&item), None); + } + // The same group holds a public-domain item in Test. + let public = Source::PublicDomain { + work: "w".into(), + author: "a".into(), + edition_url: "https://example.org/e".into(), + licence: "l".into(), + jurisdiction: "US".into(), + translator: "none".into(), + edition_year: 1900, + }; + let item = item_in("item:f:2", "x opens", &group_in_bucket(80), public); + assert_eq!(SplitScheme::V1.live(&item), Some(Split::Test)); +} diff --git a/data/original/corpus.cc b/data/original/corpus.cc index 57988dfaf5fe31057a344f805d956cfc1bcde77a..d778901e62568a8a52f6777b0c158717ccaf0df6 100644 GIT binary patch literal 13160 zcmd5@&yU;28Ld+^d+4P-^w8VDX=87^+LgH4U9AkXSnqlrr*&-DPK=~YgONBA>7_W# z3~5CuilXOU93(&v0eUHNXpq=W;sQM+K+xJp{{aMY$)OEW^f!7adP(0mLrGpzHi-fT zc!;%<$kFi4``-7yZzivQ&`*N6&-k7|rQ;?MGtINCj0xb5glcER&PX1;MytwyVPxz*`9PPf&3@$O^4d@FZn`t#G*M96v* z7JGsXco6vEkjViX^H`{ct7N1Z&)!8cX1-U>JAEF}h-Y%mP1-13|xM|5|-pgpKDe-4-@~Y}G~=C0 zT+DqrZF_MX>F(OvcsyPmCb0;yoWoVAhHHj_Ya&W|YfiJZwH6KOWi@^^PT#ao#a#1P z>^KkmK0slycr`Y^9O{6Bx@>=yo#U^vbHEyqx|sZSv$=edT~P8W9-N(-{-U{+B^omv z`>_Y2U;(sw2=G)LFboD9Mc5Z?#A7V?Nj4BMSnRSnd(zHxA;{*kOEDM$E`%=b8i*5H zO?EX=zIJ_sHc-n=>Be-XfB1-|dT;54cjm)>^84#K5<5kzKY+wXawK-rAD=bE91HLd ztKso$f@x6F@|O7_o5)15Q20ZyCzU6qyFwQZeO%Z}N>^#e4%=0<`lXyNogSl6b<-57@4cAYpsHmY@(;x-UL_ zT7+uC_V_j5#iiyqQl}!BHVl_G43)-tDkU^yLzT{^^9ZT3U70G`*?CA9E~7MoFSS=!mib9@k#DR+!dr5 z!+>jDWP=z%AY}OdlnvSY##WvVw@+)$C%Vh_eWhb|Nhn|F4=?f{@)!|CveR~`stU5* zZquscrF7Xl^;q`o0qwG5PG!s9dg!vZt1kOvU&}3fJAGvb>lz-@EaY*bugjgJ#6J zIhTjzx=H!fk z%@J&WLf+)E9D}QPS~4k}&ZA^dw=#nYlltw#Ip16Q!9%rYvnqLxq&f0zrZ1m@6@w9n)UW}lXzoZ%LJvfwf+bN zWwSCJ==m%^F`*JYS=xOlCAKP4;d(lrh-14d+mDha)k~Y)ly2tc zdT;6cL#ffJOpVWT)aaxyJqwM{kw7`9ZDqrw2>JkSpfsH@9msJoSz%_^_o29;~lMia0_eU`F}$3_zn z1$ts_1}Yg54guQ?F9bFTjp_O=#OvJ|UR-;%#j3pJlwtMf0#;6X;#t{~?gSu&sPqCP zZ1BnxXgwj=dJ>H#`e5LTz+DK@jTuDdn%aU?-dxE*`da~{^|W~z7+8U!Ud{+X51=Fy zqX=y<0;tC|Wh4njP;$HwsktpwREQE)cIYA&&^qs1>ALo zstH1`y?!{qeEnN*tZ7m)jsC`k0vWk#niTMLFKMK^=%_S}Po$ z(i}M*DfBLCg5322P~~-32Gk!5+}cW?+>;D;*>v@c9DztPO*d>KQ?-ye+b!BQRt=rb9Ua`lCtxv)&Q>!t@=G#J)<~%aFuqGJg73$khypbzB%G}emWVw4} zDPOZV>DhU{8w-9Q(0n)H$HN1vP_5)xg_^lfdx??~PMs>6Bu++QNobmsS`$bM(&bdH zW(o$>X{O6($sy1-gK0l!nNZ{W0Wu}NLCm2=2phkUb@SN7rUBJLwFRkiMe}(+Hl0?w zOgx(!1`q7IPt2wzsfpT!JeoUCVqI+!DvQKUH&gh6dO1$|)Fn7?^pEb-Dto}gd~%i3 zb{KXoaNJ-aME9u|HpShVV3b#{rElhKgHyS#DcA;QJ$>?A=B%=U4-I%a5Q6F%d&PLA z4IMu1t{qN_bHH#2qrSzlQ@Nb^vJiIdbmy#1ex@Te7TDo1L1tMtHjSxl2gAxf`H0xV zu?uQX6MPG~HV;Sg`!%6jQPnFKJ4IFBNLMmLirz{@J45E+O3%1bvqfkWpNB)CjxFZb z9HbaztSw0Ye|h6<+O~q(BV#TO%aDPO6rDG-=XR_O8=&8Sk{a!WVF{bSym4)Ts$Aau zs}P4w18~@j+eWd8x4sEO0#HM)bu^jRIO7 z?80SjC?iUlv|F_@9@TsdTA9snocn{dup~6s+}9?hZ;VeZ!4iGcx@64jd3#@$AE);I z&cE~Sem&hH*Oifu&Kzdv>0F6TfCMK!Z4=&xDeZOIifM6~J=Zh3y*A9t6+>m&a)#EI zd9>Q;E}@0Ihhwq|2MN)hp^}=m`-n&6z-Zr&Oqnqje284)>746vz+1AGeu zbq*i~t}B3LiF69!oiFnEZKfx7aHu34B12e*fjzhcvaqk%wq2b+Rg(w!>_#Jw=9s(+Iy!q+HQ?PS9;(xjjb;|NlW)A6u4_v8ws= z&foJ$b<)-@{3_~1Yw~THH2B~Mm@%v-PT^s4Jx^Hw^!h2WmTABfZah g5>xwXn_vc^9`(R*));ooG~O3R-aISP<#j>ne^E|8k^lez delta 13 VcmaEnw#<3Lc}AJdlZ37*001ss1_1y7 diff --git a/data/original/entity.cc b/data/original/entity.cc index c92e33d..f2af96f 100644 --- a/data/original/entity.cc +++ b/data/original/entity.cc @@ -1 +1 @@ -“²cultcache.store.v1‘—¬weksa.entity¬weksa.entity¯weksa.entity.v1¬weksa.entityÙL{"schemaName":"weksa.entity","schemaVersion":"weksa.entity.v1","members":[]}‘¬weksa.entity‘”­entity:keeper¬weksa.entity´2026-11-10T14:17:08ZÄ:“­entity:keeper‘“¥en-US‘ªthe keeperÀ¤Live‘®weksa-original \ No newline at end of file +“²cultcache.store.v1‘—¬weksa.entity¬weksa.entity¯weksa.entity.v1¬weksa.entityÙL{"schemaName":"weksa.entity","schemaVersion":"weksa.entity.v1","members":[]}‘¬weksa.entity™”­entity:keeper¬weksa.entity´2026-11-10T14:17:08ZÄ:“­entity:keeper‘“¥en-US‘ªthe keeperÀ¤Live‘®weksa-original”¯entity:pd-aesop¬weksa.entity´2026-11-10T18:22:10ZÄM“¯entity:pd-aesop‘“¥en-US‘¥AesopÀ¬PublicDomain‘¼Three Hundred Aesop's Fables”¯entity:pd-bacon¬weksa.entity´2026-11-10T18:22:10ZÄY“¯entity:pd-bacon‘“¥en-US‘¥BaconÀ¬PublicDomain‘Ù'The Essays or Counsels, Civil and Moral”°entity:pd-darwin¬weksa.entity´2026-11-10T18:22:10ZÄK“°entity:pd-darwin‘“¥en-US‘¦DarwinÀ¬PublicDomain‘¸On the Origin of Species”´entity:pd-dhammapada¬weksa.entity´2026-11-10T18:22:10ZĆ“´entity:pd-dhammapada‘“¥en-US‘®the DhammapadaÀ¬PublicDomain‘ÙFDhammapada, a Collection of Verses (Sacred Books of the East, vol. 10)”±entity:pd-emerson¬weksa.entity´2026-11-10T18:22:10ZÄI“±entity:pd-emerson‘“¥en-US‘§EmersonÀ¬PublicDomain‘´Essays: First Series”®entity:pd-mill¬weksa.entity´2026-11-10T18:22:10ZÄ9“®entity:pd-mill‘“¥en-US‘¤MillÀ¬PublicDomain‘ªOn Liberty”¯entity:pd-plato¬weksa.entity´2026-11-10T18:22:10ZÄ=“¯entity:pd-plato‘“¥en-US‘¥PlatoÀ¬PublicDomain‘¬The Republic”±entity:pd-thoreau¬weksa.entity´2026-11-10T18:22:10ZÄc“±entity:pd-thoreau‘“¥en-US‘§ThoreauÀ¬PublicDomain‘Ù-Walden, and On The Duty Of Civil Disobedience \ No newline at end of file From e0e3c93d979ab94f51429f779b8654dd564fdeb0 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 18:26:31 +0000 Subject: [PATCH 05/80] Stance ablation: replay, drop and permute a packet's stance beside recall RecordedAnswers is the one reader of .answer files (AnswerFiles reads through it). RunStores::stance_ablation asks a fresh lowerer the round's own lowering prompt again (replay), with the stance emptied (drop) and with the next item's stance in its split (permute), for live Train and Validation packets only, and reads how far the lowering moves and what the item's own stance adds to fidelity. A diagnostic in /ablation-: records nothing, never writes round-. `weksa loop ablate --round n`; exits 3 until answered. Touches no pinned prompt-source file. The source tripwire now allows the ablation as a second Prompt::lower call site. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/main.rs | 50 +++- crates/weksa/src/model.rs | 40 ++- crates/weksa/src/run.rs | 290 +++++++++++++++++++- crates/weksa/tests/cli.rs | 71 +++++ crates/weksa/tests/golf_loop.rs | 452 +++++++++++++++++++++++++++++++- 5 files changed, 892 insertions(+), 11 deletions(-) diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 5b88c14..a92c010 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -11,11 +11,14 @@ use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn}; use weksa::entity::{Entity, EntityOrigin, LanguageForms, Pronunciation}; use weksa::inventory::{proof_revision, seed}; use weksa::packet::proof_packet; -use weksa::run::{CycleOutcome, LowererReach, RecallOutcome, RoundOutcome, RunPaths, RunStores, cycle, round}; +use weksa::model::RecordedAnswers; +use weksa::run::{ + AblationOutcome, CycleOutcome, LowererReach, RecallOutcome, RoundOutcome, RunPaths, RunStores, cycle, round, +}; use weksa::score::ScoreInput; use weksa::{ AnswerFiles, CorpusStore, EntityId, EntityStore, InventoryStore, OllamaEmbedder, PacketStore, - Scorer, SplitScheme, StoreError, lower, pronounce, validate, + Scorer, Split, SplitScheme, StoreError, lower, pronounce, validate, }; #[derive(Parser)] @@ -112,6 +115,19 @@ enum LoopCommand { /// half). Prompts and answers live in `/recall`; the command exits 3 /// until every prompt has an answer. A diagnostic, never a gate. Recall { root: PathBuf }, + /// Ask whether the lowerer uses a packet's stance: for each live Train and + /// Validation packet of round ROUND, ask again with the stance as it is, + /// emptied, and swapped for the next item's, and read how far the lowering + /// moves from the round's own answer. Prompts and answers live in + /// `/ablation-`; the command exits 3 until every prompt has an + /// answer. A diagnostic, never a gate; Test is never touched. + Ablate { + root: PathBuf, + #[arg(long)] + round: u32, + #[arg(long, default_value = "http://127.0.0.1:11434")] + ollama_url: String, + }, /// Run one edit-accept cycle of the run at ROOT. The stores are read only /// from the run's record. The editor's prompt, then the candidate round's /// encode prompts, then its lowering prompts are written in turn; the @@ -785,6 +801,36 @@ fn main() -> Result<()> { RoundOutcome::Recorded(r) => println!("{}", run.round_line(&r)), } } + Command::Loop(LoopCommand::Ablate { + root, + round, + ollama_url, + }) => { + let run = RunStores::open(&root)?; + let model = run.record().model.clone(); + let mut recorded = + RecordedAnswers::new(root.join(format!("round-{round}")), model.clone()); + let mut files = AnswerFiles::new(run.ablation_dir(round), model); + let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder)); + match run.stance_ablation(round, &mut recorded, &mut files, &scorer)? { + AblationOutcome::Unanswered { prompts } => { + for digest in &prompts { + println!("unanswered {digest}"); + } + println!("unanswered {} prompts", prompts.len()); + std::process::exit(3); + } + AblationOutcome::Read(readings) => { + for (split, reading) in readings { + let name = match split { + Split::Train => "train", + _ => "validation", + }; + println!("{name} {}", reading.line()); + } + } + } + } Command::Loop(LoopCommand::Recall { root }) => { let run = RunStores::open(&root)?; let mut files = AnswerFiles::new(run.recall_dir(), run.record().model.clone()); diff --git a/crates/weksa/src/model.rs b/crates/weksa/src/model.rs index 2f83176..1a79449 100644 --- a/crates/weksa/src/model.rs +++ b/crates/weksa/src/model.rs @@ -48,6 +48,37 @@ impl AnswerFiles { } } +/// Answers read from one directory and nothing else: `.answer` is the +/// reply to the prompt whose digest is ``. A prompt with no file is +/// [`PortError::Unanswered`]; this port never creates or writes anything. The +/// one reader of answer files: [`AnswerFiles`] reads through it. +pub struct RecordedAnswers { + dir: PathBuf, + model: String, +} + +impl RecordedAnswers { + pub fn new(dir: PathBuf, model: String) -> Self { + Self { dir, model } + } +} + +impl ModelPort for RecordedAnswers { + fn model(&self) -> &str { + &self.model + } + + fn answer(&mut self, prompt: &Prompt) -> Result { + match std::fs::read(self.dir.join(format!("{}.answer", prompt.digest()))) { + Ok(bytes) => { + String::from_utf8(bytes).map_err(|_| PortError::Io(ErrorKind::InvalidData)) + } + Err(e) if e.kind() == ErrorKind::NotFound => Err(PortError::Unanswered), + Err(e) => Err(PortError::Io(e.kind())), + } + } +} + impl ModelPort for AnswerFiles { fn model(&self) -> &str { &self.model @@ -56,12 +87,9 @@ impl ModelPort for AnswerFiles { fn answer(&mut self, prompt: &Prompt) -> Result { let io = |e: std::io::Error| PortError::Io(e.kind()); let digest = prompt.digest(); - match std::fs::read(self.dir.join(format!("{digest}.answer"))) { - Ok(bytes) => { - return String::from_utf8(bytes).map_err(|_| PortError::Io(ErrorKind::InvalidData)); - } - Err(e) if e.kind() == ErrorKind::NotFound => {} - Err(e) => return Err(io(e)), + match RecordedAnswers::new(self.dir.clone(), self.model.clone()).answer(prompt) { + Err(PortError::Unanswered) => {} + read => return read, } std::fs::create_dir_all(&self.dir).map_err(io)?; let path = self.dir.join(format!("{digest}.prompt")); diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 943894d..958edf1 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -22,6 +22,9 @@ //! `ModelPort` (`Prompt::lower`), in two answer phases; the CLI's `lower` still //! uses the template table. The prompt source pin covers `context.rs`, //! `split.rs` and `lower.rs`. +//! +//! `RunStores::stance_ablation`, like `recall`, is a diagnostic in its own +//! directory: it records nothing and never writes its round's directory. use std::collections::{BTreeMap, BTreeSet}; use std::fmt; @@ -39,6 +42,7 @@ use crate::corpus::{CorpusItem, CorpusItemId, ModelFamily, Source}; use crate::encode::{EditDraft, LoweringDraft, PacketDraft, encoder_id}; use crate::form::{is_attestation, is_slug}; use crate::model::{ModelPort, PortError}; +use crate::packet::{Packet, Stance}; use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate}; use crate::split::{Split, SplitScheme, fnv1a64, fnv1a64_of, hex16}; use crate::store::{ @@ -950,6 +954,290 @@ impl RunStores { } } +/// What the lowerer's answer did to each variant of a packet's stance, per +/// split. Means only; no key, text or stance value. +#[derive(Clone, Debug, PartialEq)] +pub struct AblationReading { + /// Items whose recorded and replayed lowerings were both usable. + pub n: u32, + /// Answers dropped as unusable, replay or variant. + pub unusable: u32, + /// Mean agreement of the replay with the recorded lowering: the noise + /// floor a variant is read against. + pub replay: Agreement, + pub drop: VariantReading, + pub permute: VariantReading, +} + +#[derive(Clone, Debug, PartialEq, Default)] +pub struct Agreement { + pub chrf: f64, + pub meaning: f64, +} + +/// One variant's reading: agreement with the recorded lowering, pairs whose +/// chrF fell below the replay's, and the mean fidelity the item's own stance +/// adds (`score(item, replay) - score(item, variant)`). +#[derive(Clone, Debug, PartialEq, Default)] +pub struct VariantReading { + pub n: u32, + pub chrf: f64, + pub meaning: f64, + pub moved: u32, + pub ref_gain_chrf: f64, + pub ref_gain_meaning: f64, +} + +impl AblationReading { + pub fn line(&self) -> String { + let variant = |name: &str, v: &VariantReading| { + format!( + "{name} n={} chrf={:.2} meaning={:.3} moved={} ref_gain_chrf={:.2} ref_gain_meaning={:.3}", + v.n, v.chrf, v.meaning, v.moved, v.ref_gain_chrf, v.ref_gain_meaning + ) + }; + format!( + "ablation n={} unusable={} replay chrf={:.2} meaning={:.3} | {} | {}", + self.n, + self.unusable, + self.replay.chrf, + self.replay.meaning, + variant("drop", &self.drop), + variant("permute", &self.permute), + ) + } +} + +#[derive(Debug)] +pub enum AblationOutcome { + Unanswered { prompts: Vec }, + /// Train then Validation. + Read(Vec<(Split, AblationReading)>), +} + +/// What the port made of one lowering prompt. +enum Asked { + Text(String), + Unusable, + Waiting, +} + +fn asked(port: &mut dyn ModelPort, prompt: &Prompt) -> Result { + match port.answer(prompt) { + Ok(a) => Ok(LoweringDraft::parse(&a) + .ok() + .filter(LoweringDraft::usable) + .map_or(Asked::Unusable, |d| Asked::Text(d.text))), + Err(PortError::Io(ErrorKind::InvalidData)) => Ok(Asked::Unusable), + Err(PortError::Unanswered) => Ok(Asked::Waiting), + Err(PortError::Io(kind)) => Err(RunError::Io(kind)), + } +} + +fn mean(xs: impl Iterator) -> f64 { + let (sum, n) = xs.fold((0.0, 0u32), |(s, n), x| (s + x, n + 1)); + if n == 0 { 0.0 } else { sum / f64::from(n) } +} + +type Scored = (ItemScore, ItemScore); + +/// One item's reference text, recorded lowering, replay, and drop and permute +/// lowerings. +type Lowered = (String, String, String, [Option; 2]); + +impl RunStores { + /// `/ablation-`, where a stance ablation's prompts, answers and + /// variant packet stores live. + pub fn ablation_dir(&self, n: u32) -> PathBuf { + self.root.join(format!("ablation-{n}")) + } + + /// A diagnostic like [`recall`](Self::recall), never a gate: for each live + /// Train and Validation packet of plain round `n`, asks the lowerer again + /// (`model`, the replay), with the packet's stance emptied (drop) and with + /// the next item's stance in its split (permute), and reads how far the + /// lowering moves against the round's own answer (`recorded`). Test and + /// withdrawn items are never asked. Records nothing and writes nothing in + /// `round-`; the variant stores are rebuilt each call. An answer that + /// is not a usable lowering drops its pair and is counted in `unusable`. + pub fn stance_ablation( + &self, + n: u32, + recorded: &mut dyn ModelPort, + model: &mut dyn ModelPort, + scorer: &Scorer, + ) -> Result { + let store = |_| RunError::Store; + check_drift(self, model, scorer)?; + self.rounds() + .map_err(store)? + .into_iter() + .find(|r| r.n() == n && r.cycle().is_none()) + .ok_or(RunError::Malformed("round"))?; + let inventory = self.inventory.inventory().map_err(store)?; + + let dir = self.ablation_dir(n); + std::fs::create_dir_all(&dir).map_err(|e| RunError::Io(e.kind()))?; + for name in ["base", "drop", "permute"] { + for suffix in ["cc", "cc.lock"] { + let _ = std::fs::remove_file(dir.join(format!("{name}.{suffix}"))); + } + } + let mut stores = [ + PacketStore::open(&dir.join("base.cc")).map_err(store)?, + PacketStore::open(&dir.join("drop.cc")).map_err(store)?, + PacketStore::open(&dir.join("permute.cc")).map_err(store)?, + ]; + + // Live Train and Validation packets, in item key order within a split. + let round_packets = PacketStore::open(&self.root.join(format!("round-{n}")).join("packets.cc")) + .map_err(store)? + .all() + .map_err(store)?; + let mut splits: [(Split, Vec<(CorpusItem, Packet)>); 2] = + [(Split::Train, Vec::new()), (Split::Validation, Vec::new())]; + for packet in round_packets { + let item = self + .corpus + .get(packet.provenance.corpus_item.as_str()) + .map_err(store)? + .ok_or(RunError::Store)?; + for (split, held) in &mut splits { + if SplitScheme::V1.live(&item) == Some(*split) { + held.push((item.clone(), packet.clone())); + } + } + } + + let mut waiting: Vec = Vec::new(); + let mut readings = Vec::new(); + // Texts per split: (item text, recorded, replay, [drop, permute]). + let mut texts: Vec> = Vec::new(); + let mut unusable = [0u32; 2]; + for (at, (_, held)) in splits.iter_mut().enumerate() { + held.sort_by(|a, b| a.0.id.as_str().cmp(b.0.id.as_str())); + let mut rows = Vec::new(); + for (i, (item, packet)) in held.iter().enumerate() { + let packet_item = packet.provenance.corpus_item.clone(); + let mut prompt_of = |slot: usize, packet: Packet| -> Result { + let view = stores[slot] + .admit(packet, &self.inventory, &self.corpus, &self.entities) + .map_err(|_| RunError::Store)?; + Prompt::lower(&view, &packet_item, &inventory, &self.entities) + .map_err(|_| RunError::Store) + }; + let base = prompt_of(0, packet.clone())?; + let mut variants: [Option; 2] = [None, None]; + if !packet.stance.features.is_empty() { + let mut p = packet.clone(); + p.stance = Stance::default(); + variants[0] = Some(prompt_of(1, p)?); + } + if held.len() > 1 { + let mut p = packet.clone(); + p.stance = held[(i + 1) % held.len()].1.stance.clone(); + let prompt = prompt_of(2, p)?; + if prompt.digest() != base.digest() { + variants[1] = Some(prompt); + } + } + // The round's own answer must exist; a missing one is not a gap to ask for. + let recorded_text = match asked(recorded, &base)? { + Asked::Waiting => return Err(RunError::Store), + other => other, + }; + let replay = asked(model, &base)?; + let mut got: [Option; 2] = [None, None]; + for (slot, prompt) in got.iter_mut().zip(&variants) { + let Some(prompt) = prompt else { continue }; + match asked(model, prompt)? { + Asked::Text(t) => *slot = Some(t), + Asked::Unusable => unusable[at] += 1, + Asked::Waiting => { + let d = prompt.digest(); + if !waiting.contains(&d) { + waiting.push(d); + } + } + } + } + match (recorded_text, replay) { + (Asked::Text(b), Asked::Text(r)) => { + rows.push((item.text.clone(), b, r, got)); + } + (_, Asked::Waiting) => { + let d = base.digest(); + if !waiting.contains(&d) { + waiting.push(d); + } + } + _ => unusable[at] += 1, + } + } + texts.push(rows); + } + if !waiting.is_empty() { + return Ok(AblationOutcome::Unanswered { prompts: waiting }); + } + + // One score call: for each item, the replay then each variant, each + // against the recorded lowering (agreement) and the item (fidelity). + let mut inputs: Vec = Vec::new(); + for (reference, base, replay, got) in texts.iter().flatten() { + for output in std::iter::once(replay).chain(got.iter().flatten()) { + inputs.push(ScoreInput { reference: base, output }); + inputs.push(ScoreInput { reference, output }); + } + } + let mut scores = if inputs.is_empty() { + Vec::new() + } else { + scorer.score(&inputs).map_err(|_| RunError::Embed)? + } + .into_iter(); + let mut pair = || -> Result { + Ok((scores.next().ok_or(RunError::Store)?, scores.next().ok_or(RunError::Store)?)) + }; + for (at, ((split, _), rows)) in splits.iter().zip(&texts).enumerate() { + let mut replay: Vec = Vec::new(); + let mut variants: [Vec<(Scored, Scored)>; 2] = [Vec::new(), Vec::new()]; + for (_, _, _, got) in rows { + let r = pair()?; + for (k, g) in got.iter().enumerate() { + if g.is_some() { + variants[k].push((r.clone(), pair()?)); + } + } + replay.push(r); + } + let variant = |v: &[(Scored, Scored)]| VariantReading { + n: v.len() as u32, + chrf: mean(v.iter().map(|(_, x)| x.0.chrf)), + meaning: mean(v.iter().filter_map(|(_, x)| x.0.meaning)), + moved: v.iter().filter(|(r, x)| x.0.chrf < r.0.chrf).count() as u32, + ref_gain_chrf: mean(v.iter().map(|(r, x)| r.1.chrf - x.1.chrf)), + ref_gain_meaning: mean( + v.iter().filter_map(|(r, x)| Some(r.1.meaning? - x.1.meaning?)), + ), + }; + readings.push(( + *split, + AblationReading { + n: rows.len() as u32, + unusable: unusable[at], + replay: Agreement { + chrf: mean(replay.iter().map(|r| r.0.chrf)), + meaning: mean(replay.iter().filter_map(|r| r.0.meaning)), + }, + drop: variant(&variants[0]), + permute: variant(&variants[1]), + }, + )); + } + Ok(AblationOutcome::Read(readings)) + } +} + /// The distinct rule codes an admission refusal broke: validator rules by /// code, a store refusal by its variant name. Nothing from the packet or the /// stores is kept; an I/O failure is `io`. @@ -1008,7 +1296,7 @@ pub fn round( } /// The model and embedder the run pinned are the ones answering. -fn check_drift( +fn check_drift( run: &RunStores, model: &M, scorer: &Scorer, diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 4113a82..3d38bba 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -1633,3 +1633,74 @@ fn loop_open_requires_the_lowerer_reach_and_pins_it() { weksa::run::LowererReach::BriefedSubagent ); } + +/// Answers the Dao encode prompts with the Dao packets and every lowering with +/// one sentence. +struct DaoModel; + +impl weksa::model::ModelPort for DaoModel { + fn model(&self) -> &str { + "fake-model" + } + fn answer(&mut self, prompt: &weksa::context::Prompt) -> Result { + if prompt.text().starts_with(weksa::context::LOWER_INSTRUCTIONS) { + return Ok(r#"{"text":"The highest excellence is like the water."}"#.into()); + } + let n: usize = prompt.item_keys()[0].rsplit(':').next().unwrap().parse().unwrap(); + Ok(serde_json::to_string(&weksa::PacketDraft::of(&weksa::daodejing::packets()[n - 1])).unwrap()) + } +} + +#[test] +fn loop_ablate_waits_then_prints_one_line_per_split() { + let d = tempfile::tempdir().unwrap(); + open_dao_run(d.path()); + let url = fake_embedder(); + let root = d.path().join("canaryroot"); + + // Record round 1 the way `loop round` does, then put its lowering answers + // where the relay would have. + let mut run = weksa::run::RunStores::open(&root).unwrap(); + let scorer = weksa::Scorer::new(weksa::OllamaEmbedder::new(&url, &run.record().embedder)); + let weksa::run::RoundOutcome::Recorded(r) = weksa::run::round(&mut run, &mut DaoModel, &scorer).unwrap() + else { + panic!("answered") + }; + for call in &r.calls()[3..] { + std::fs::write( + root.join(format!("round-1/{}.answer", call.prompt)), + r#"{"text":"The highest excellence is like the water."}"#, + ) + .unwrap(); + } + + let ablate = || weksa(d.path(), &["loop", "ablate", "canaryroot", "--round", "1", "--ollama-url", &url]); + let first = ablate(); + assert_eq!(first.status.code(), Some(3), "{}", out(&first)); + let text = out(&first); + let digests: Vec<&str> = text + .lines() + .filter_map(|l| l.strip_prefix("unanswered ")) + .filter(|d| d.len() == 16) + .collect(); + // Three replays, three drops and the Validation permutes (2 items). + assert_eq!(digests.len(), 3 + 3 + 2, "{text}"); + for digest in &digests { + assert!(root.join(format!("ablation-1/{digest}.prompt")).is_file()); + std::fs::write( + root.join(format!("ablation-1/{digest}.answer")), + r#"{"text":"The highest excellence is like the water."}"#, + ) + .unwrap(); + } + let second = ablate(); + assert!(second.status.success(), "{}", out(&second)); + let text = out(&second); + let lines: Vec<&str> = text.lines().collect(); + assert_eq!(lines.len(), 2, "{text}"); + assert!(lines[0].starts_with("train ablation n=1 "), "{text}"); + assert!(lines[1].starts_with("validation ablation n=2 "), "{text}"); + for leaked in ["item:", "packet:", "highest excellence", "canaryroot"] { + assert!(!text.contains(leaked), "{text}"); + } +} diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 0929e53..c9c4bd6 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -2598,9 +2598,19 @@ fn source_tripwires() { let count = |needle: &str| code.iter().filter(|l| l.contains(needle)).count(); assert!(!run.contains("pub fn new")); // The round does not call the template table: the model lowers. - assert_eq!(count("lower(&"), 0, "run.rs calls the table"); + assert_eq!( + code.iter() + .filter(|l| l.contains("lower(&") && !l.contains("Prompt::lower(")) + .count(), + 0, + "run.rs calls the table" + ); assert_eq!(count("crate::lower::lower"), 0, "run.rs imports the table"); - assert_eq!(count("Prompt::lower("), 1, "one lowering call site"); + assert_eq!( + count("Prompt::lower("), + 2, + "two lowering call sites: the round's and the stance ablation's replay of it" + ); // Prompt::lower takes the view, the item's key, the inventory and the entity // port: no corpus item, train set, offer or store. let start = context.find("pub fn lower(").unwrap(); @@ -3560,3 +3570,441 @@ fn own_family_counts_offered_items_of_the_answerer_family() { assert_eq!(mixed_family_round(ModelFamily::Google), 0); assert_eq!(mixed_family_round(ModelFamily::Unrecorded), 0); } + +// ---- stance ablation (stance-to-lowerer) ---- + +/// A lowerer by function: the text it answers for an item key and the prompt +/// text, recording every prompt it was shown. +struct Lowerer { + f: fn(&str, &str) -> String, + seen: Vec<(String, String, String)>, +} + +impl Lowerer { + fn new(f: fn(&str, &str) -> String) -> Self { + Self { f, seen: vec![] } + } + /// The digests asked, once each, in the order first asked. + fn digests(&self) -> Vec { + let mut out: Vec = Vec::new(); + for (d, ..) in &self.seen { + if !out.contains(d) { + out.push(d.clone()); + } + } + out + } +} + +impl ModelPort for Lowerer { + fn model(&self) -> &str { + "fake-model" + } + fn answer(&mut self, prompt: &Prompt) -> Result { + let key = prompt.item_keys()[0].clone(); + self.seen.push((prompt.digest(), key.clone(), prompt.text().to_string())); + Ok(serde_json::json!({ "text": (self.f)(&key, prompt.text()) }).to_string()) + } +} + +const WRITTEN: &str = "item:written:1"; + +fn english(key: &str) -> String { + let mut table = table_english(); + table.insert( + WRITTEN.into(), + "Heaven endures through the long years without end.".into(), + ); + table[key].clone() +} + +/// The packet line's stance, as the prompt prints it: the packet line is one +/// JSON object ending in the stance. +fn stance_of(text: &str) -> String { + let line = text.lines().find(|l| l.starts_with("packet: ")).unwrap(); + line[line.find("\"stance\":").unwrap()..].to_string() +} + +/// Says the same thing whatever the stance. +fn blind(key: &str, _text: &str) -> String { + english(key) +} + +/// Appends the stance it was shown. +fn reader(key: &str, text: &str) -> String { + format!("{} {}", english(key), stance_of(text)) +} + +/// Does better with the item's own stance: the real sentence for it, nonsense +/// for any other, empty included. +fn helped(key: &str, text: &str) -> String { + let r800 = r#""stance":{"features":[{"feature":"feature:18","value":{"Grade":800}}]}}"#; + let own = match key { + "item:daodejing:3" => r#""stance":{"features":[{"feature":"feature:18","value":{"Grade":800}},{"feature":"feature:16","value":{"Grade":300}}]}}"#, + WRITTEN => r#""stance":{"features":[{"feature":"feature:18","value":{"Grade":500}}]}}"#, + _ => r800, + }; + if stance_of(text) == own { english(key) } else { "zzz qqq xxx".into() } +} + +fn grade_stance(features: &[(&str, u16)]) -> weksa::packet::Stance { + weksa::packet::Stance { + features: features + .iter() + .map(|(f, v)| weksa::packet::FeatureValue { + feature: f.parse().unwrap(), + value: weksa::packet::FeatValue::Grade(common::grade(*v)), + }) + .collect(), + } +} + +/// A Dao run with a third Validation item (`item:written:1`, key order after +/// items 2 and 3) and round 1 recorded over it, every item encoded to its Dao +/// packet. Stances: item 1 restraint 800; item 2 restraint 800; item 3 +/// restraint 800 and affection 300; the written item restraint 500. +fn recorded_dao(edit: impl FnOnce(&mut HashMap)) -> (Dao, RoundRecord) { + let mut dao = dao_over(|corpus, entities| { + let entities = weksa::EntityStore::open(entities).unwrap(); + let mut store = CorpusStore::open(corpus).unwrap(); + let mut written = store.get("item:daodejing:2").unwrap().unwrap(); + written.id = WRITTEN.into(); + written.text = "Heaven endures through the long years without end.".into(); + written.source = weksa::corpus::Source::OperatorWritten; + store.add(written, &entities).unwrap(); + }); + let mut model = fake(&good_answers()); + let mut extra = draft_of(2); + extra.referents.truncate(1); + extra.predications.truncate(1); + extra.stance = grade_stance(&[("feature:18", 500)]); + model.answers.insert(WRITTEN.into(), json(&extra)); + edit(&mut model.answers); + model.lowered.insert(WRITTEN.into(), english(WRITTEN)); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + (dao, r) +} + +fn ablate( + dao: &Dao, + recorded: &mut dyn ModelPort, + model: &mut dyn ModelPort, +) -> Result { + dao.run.stance_ablation(1, recorded, model, &Scorer::new(Bag)) +} + +fn read( + outcome: weksa::run::AblationOutcome, +) -> Vec<(weksa::Split, weksa::run::AblationReading)> { + match outcome { + weksa::run::AblationOutcome::Read(r) => r, + weksa::run::AblationOutcome::Unanswered { prompts } => panic!("unanswered {prompts:?}"), + } +} + +#[test] +fn recorded_answers_read_and_never_write() { + use std::io::ErrorKind; + use weksa::model::RecordedAnswers; + let p = prompt(); + let dir = tempfile::tempdir().unwrap(); + let mut port = RecordedAnswers::new(dir.path().to_path_buf(), "answerer".into()); + assert_eq!(port.model(), "answerer"); + // A miss is Unanswered and the directory is as it was. + assert_eq!(port.answer(&p), Err(PortError::Unanswered)); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0); + // A directory that does not exist is a miss too, and is not made. + let mut absent = RecordedAnswers::new(dir.path().join("nope"), "m".into()); + assert_eq!(absent.answer(&p), Err(PortError::Unanswered)); + assert!(!dir.path().join("nope").exists()); + // A hit is the file's text as written. + let file = dir.path().join(format!("{}.answer", p.digest())); + std::fs::write(&file, " {\"x\": 1}\n").unwrap(); + assert_eq!(port.answer(&p).unwrap(), " {\"x\": 1}\n"); + std::fs::write(&file, [0xff, 0xfe]).unwrap(); + assert_eq!(port.answer(&p), Err(PortError::Io(ErrorKind::InvalidData))); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1); +} + +#[test] +fn a_stance_ablation_asks_replay_drop_and_permute_for_train_and_validation_only() { + let (dao, r) = recorded_dao(|_| {}); + let round_digests: Vec = r.calls()[4..].iter().map(|c| c.prompt.clone()).collect(); + assert_eq!(round_digests.len(), 4, "the round lowered four items"); + let mut recorded_port = Lowerer::new(reader); + let mut model = Lowerer::new(reader); + read(ablate(&dao, &mut recorded_port, &mut model).unwrap()); + + // The base prompts are the round's own, byte for byte. + let recorded_digests = recorded_port.digests(); + assert_eq!(recorded_digests, round_digests); + let asked = model.digests(); + for d in &round_digests { + assert!(asked.contains(d), "the replay asks the round's prompt {d}"); + } + // Per item: replay and drop for each; permute for each Validation item, and + // none for the lone Train item. + let per_key = |key: &str| model.seen.iter().filter(|s| s.1 == key).count(); + assert_eq!(per_key("item:daodejing:1"), 2); + for key in ["item:daodejing:2", "item:daodejing:3", WRITTEN] { + assert_eq!(per_key(key), 3, "{key}"); + } + // Test items are never asked. + for (_, key, text) in &model.seen { + assert!(key != "item:daodejing:4" && key != "item:daodejing:5", "{key}"); + assert!(!text.contains("item:daodejing:4")); + } + // A drop prompt prints the empty stance, a replay the item's own. + let empty = model + .seen + .iter() + .filter(|s| stance_of(&s.2) == "\"stance\":{\"features\":[]}}") + .count(); + assert_eq!(empty, 4, "one drop per item"); +} + +#[test] +fn a_lowerer_blind_to_stance_reads_no_movement_and_one_that_reads_it_does() { + let (dao, _) = recorded_dao(|_| {}); + let one = |f: fn(&str, &str) -> String| { + let (mut rec, mut model) = (Lowerer::new(f), Lowerer::new(f)); + read(ablate(&dao, &mut rec, &mut model).unwrap()) + }; + for (split, r) in one(blind) { + assert_eq!(r.unusable, 0); + assert_eq!((r.drop.moved, r.permute.moved), (0, 0), "{split:?}: {}", r.line()); + assert!((r.replay.chrf - 100.0).abs() < 1e-9); + assert!((r.drop.chrf - r.replay.chrf).abs() < 1e-9); + if r.permute.n > 0 { + assert!((r.permute.chrf - r.replay.chrf).abs() < 1e-9); + } + assert_eq!(r.drop.ref_gain_chrf, 0.0); + } + let reading = one(reader); + let (train, validation) = (&reading[0].1, &reading[1].1); + assert_eq!((reading[0].0, reading[1].0), (weksa::Split::Train, weksa::Split::Validation)); + assert_eq!((train.n, validation.n), (1, 3)); + assert_eq!((train.drop.n, train.drop.moved), (1, 1)); + assert_eq!((train.permute.n, train.permute.moved), (0, 0)); + assert_eq!((validation.drop.n, validation.drop.moved), (3, 3)); + assert_eq!((validation.permute.n, validation.permute.moved), (3, 3)); + assert!((validation.replay.chrf - 100.0).abs() < 1e-9); + assert!(validation.drop.chrf < validation.replay.chrf); + assert!(validation.permute.chrf < validation.replay.chrf); +} + +#[test] +fn the_item_s_own_stance_helps_fidelity_only_for_a_lowerer_that_uses_it() { + let (dao, _) = recorded_dao(|_| {}); + let one = |f: fn(&str, &str) -> String| { + let (mut rec, mut model) = (Lowerer::new(f), Lowerer::new(f)); + read(ablate(&dao, &mut rec, &mut model).unwrap()) + }; + for (split, r) in one(helped) { + assert!(r.drop.ref_gain_chrf > 10.0, "{split:?}: {}", r.line()); + assert!(r.drop.ref_gain_meaning > 0.0, "{split:?}: {}", r.line()); + if r.permute.n > 0 { + assert!(r.permute.ref_gain_chrf > 10.0, "{split:?}: {}", r.line()); + } + assert_eq!(r.drop.moved, r.drop.n); + } + for (split, r) in one(blind) { + assert_eq!((r.drop.ref_gain_chrf, r.drop.ref_gain_meaning), (0.0, 0.0), "{split:?}"); + } +} + +#[test] +fn a_packet_without_stance_asks_no_drop_and_a_test_packet_in_the_round_is_never_asked() { + let (dao, _) = recorded_dao(|answers| { + let mut d = draft_of(3); + d.stance = Default::default(); + answers.insert("item:daodejing:3".into(), json(&d)); + }); + // A packet for a Test item, put into the round's own store the way no round + // would: the ablation reads splits from the corpus, not from the store. + { + let corpus = CorpusStore::open(&dao.corpus).unwrap(); + let inv = dao.run.inventory(); + let mut packet = packets().remove(3); + packet.inventory_rev = 3; + PacketStore::open(&dao.root.join("round-1/packets.cc")) + .unwrap() + .admit(packet, inv, &corpus, dao.run.entities()) + .unwrap(); + } + let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(reader)); + read(ablate(&dao, &mut rec, &mut model).unwrap()); + let bare = "\"stance\":{\"features\":[]}}"; + let drops_of_3 = model + .seen + .iter() + .filter(|s| s.1 == "item:daodejing:3" && stance_of(&s.2) == bare) + .count(); + // Its replay already shows the empty stance; there is no drop beside it. + assert_eq!(drops_of_3, 1); + assert!(model.seen.iter().all(|s| s.1 != "item:daodejing:4"), "a Test item is never asked"); + assert!(rec.seen.iter().all(|s| s.1 != "item:daodejing:4")); +} + +#[test] +fn permute_takes_the_next_items_stance_in_its_split_and_skips_equal_ones() { + let (dao, _) = recorded_dao(|_| {}); + let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(reader)); + read(ablate(&dao, &mut rec, &mut model).unwrap()); + // The stance each Validation item's permute prompt shows: the next item's + // in key order (2, 3, written:1), cyclic; item 1 is alone in Train. + let stance = |text: &str| stance_of(text); + let of = |key: &str| -> Vec { + model + .seen + .iter() + .filter(|s| s.1 == key) + .map(|s| stance(&s.2)) + .collect() + }; + let (s2, s3, sw) = ( + stance(&rec.seen.iter().find(|s| s.1 == "item:daodejing:2").unwrap().2), + stance(&rec.seen.iter().find(|s| s.1 == "item:daodejing:3").unwrap().2), + stance(&rec.seen.iter().find(|s| s.1 == WRITTEN).unwrap().2), + ); + assert!(s2 != s3 && s3 != sw && sw != s2); + assert!(of("item:daodejing:2").contains(&s3)); + assert!(of("item:daodejing:3").contains(&sw)); + assert!(of(WRITTEN).contains(&s2), "the last wraps to the first"); + assert!(!of("item:daodejing:2").contains(&sw), "not the previous or the other"); + // Item 1 is alone in Train: it never takes a stance across splits. + assert_eq!(of("item:daodejing:1").len(), 2, "replay and drop only"); + + // Items 2 and 3 with equal stances: item 2's permute is its own prompt, so + // it asks none; the others still do. + let (dao, _) = recorded_dao(|answers| { + let mut d = draft_of(3); + d.stance = draft_of(2).stance; + answers.insert("item:daodejing:3".into(), json(&d)); + }); + let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(reader)); + let r = read(ablate(&dao, &mut rec, &mut model).unwrap()); + let per_key = |key: &str| model.seen.iter().filter(|s| s.1 == key).count(); + assert_eq!(per_key("item:daodejing:2"), 2, "replay and drop: its next holds the stance it has"); + assert_eq!(per_key("item:daodejing:3"), 3); + assert_eq!(per_key(WRITTEN), 3); + assert_eq!(r[1].1.permute.n, 2, "{}", r[1].1.line()); +} + +#[test] +fn a_stance_ablation_records_nothing_and_writes_nothing_in_its_round() { + use weksa::model::RecordedAnswers; + let (dao, r) = recorded_dao(|_| {}); + let round1 = dao.root.join("round-1"); + for c in &r.calls()[4..] { + std::fs::write( + round1.join(format!("{}.answer", c.prompt)), + serde_json::json!({ "text": english(&c.items[0]) }).to_string(), + ) + .unwrap(); + } + let snapshot = |dir: &Path| -> Vec<(String, Vec)> { + let mut files: Vec<_> = std::fs::read_dir(dir) + .unwrap() + .map(|e| { + let e = e.unwrap(); + let bytes = if e.path().is_dir() { vec![] } else { std::fs::read(e.path()).unwrap() }; + (e.file_name().to_string_lossy().into_owned(), bytes) + }) + .collect(); + files.sort(); + files.retain(|f| f.0 != "ablation-1"); + files + }; + let before = (snapshot(&round1), snapshot(&dao.root), dao.run.rounds().unwrap(), dao.run.decisions().unwrap()); + let go = || { + let mut rec = RecordedAnswers::new(round1.clone(), "fake-model".into()); + let mut files = weksa::model::AnswerFiles::new(dao.run.ablation_dir(1), "fake-model".into()); + // Nothing answers yet: the replay and variant prompts are written. + dao.run.stance_ablation(1, &mut rec, &mut files, &Scorer::new(Bag)) + }; + let weksa::run::AblationOutcome::Unanswered { prompts } = go().unwrap() else { + panic!("nobody answered") + }; + // Four replays (the round's own prompts), four drops, three permutes (item 1 + // is alone in Train). + assert_eq!(prompts.len(), 4 + 4 + 3, "{prompts:?}"); + for d in &prompts { + assert!(dao.run.ablation_dir(1).join(format!("{d}.prompt")).is_file()); + } + // Answer every ask from a fake and read; then again over the same files. + for d in &prompts { + let text = std::fs::read_to_string(dao.run.ablation_dir(1).join(format!("{d}.prompt"))).unwrap(); + std::fs::write( + dao.run.ablation_dir(1).join(format!("{d}.answer")), + serde_json::json!({ "text": format!("answer {}", &text[..8]) }).to_string(), + ) + .unwrap(); + } + let first = read(go().unwrap()); + let again = read(go().unwrap()); + assert_eq!(first, again); + let after = (snapshot(&round1), snapshot(&dao.root), dao.run.rounds().unwrap(), dao.run.decisions().unwrap()); + assert!(before.0 == after.0, "round-1 is unchanged byte for byte"); + assert!(before.1 == after.1, "the run root's files are unchanged"); + assert_eq!(before.2, after.2); + assert_eq!(before.3, after.3); +} + +#[test] +fn a_stance_ablation_refuses_a_cycle_round_and_missing_round_answers() { + use weksa::model::RecordedAnswers; + let (dao, _) = recorded_dao(|_| {}); + // No round-2, and round 1 holds no answer files. + let mut model = Lowerer::new(reader); + let mut rec = RecordedAnswers::new(dao.root.join("round-1"), "fake-model".into()); + assert!(matches!( + dao.run.stance_ablation(2, &mut rec, &mut model, &Scorer::new(Bag)), + Err(RunError::Malformed("round")) + )); + assert!(matches!( + ablate(&dao, &mut rec, &mut model), + Err(RunError::Store) + )); + assert!( + std::fs::read_dir(dao.root.join("round-1")) + .unwrap() + .all(|e| !e.unwrap().file_name().to_string_lossy().ends_with(".prompt")), + "no prompt is written into the round" + ); + // Drift is refused as a round refuses it. + struct Named(Lowerer); + impl ModelPort for Named { + fn model(&self) -> &str { + "other-model" + } + fn answer(&mut self, p: &Prompt) -> Result { + self.0.answer(p) + } + } + let mut named = Named(Lowerer::new(reader)); + let mut rec2 = Lowerer::new(reader); + assert!(matches!( + ablate(&dao, &mut rec2, &mut named), + Err(RunError::Drift(DriftOf::Model)) + )); + assert!(matches!( + dao.run.stance_ablation(1, &mut rec2, &mut Lowerer::new(reader), &Scorer::new(OtherEmbedder)), + Err(RunError::Drift(DriftOf::Embedder)) + )); +} + +#[test] +fn a_stance_ablation_refuses_a_cycle_round() { + let mut dao = dao(); + two_rounds(&mut dao); + run_cycle(&mut dao, &mut cycle_fake(ONE_CONCEPT)); + let rounds = dao.run.rounds().unwrap(); + let candidate = rounds.iter().find(|r| r.cycle().is_some()).expect("the cycle's round"); + let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(reader)); + assert!(matches!( + dao.run.stance_ablation(candidate.n(), &mut rec, &mut model, &Scorer::new(Bag)), + Err(RunError::Malformed("round")) + )); + assert!(model.seen.is_empty() && rec.seen.is_empty()); +} From 633bf831dc3e4cfaeb3549b6653d760187f72551 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 18:27:23 +0000 Subject: [PATCH 06/80] Pin that the lowering prompt moves with the stance alone One dao packet admitted with its own stance, an anger-and-restraint stance and none: three digests, each packet line ending in exactly that stance's JSON, the rest of the text byte-equal, the stance features glossed in the one inventory listing. No prompt change; the lowerer already reads the stance. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/context.rs | 44 +++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs index d6e2efe..32d3306 100644 --- a/crates/weksa/tests/context.rs +++ b/crates/weksa/tests/context.rs @@ -642,6 +642,50 @@ mod lowering { assert!(!prompt.text().contains(item.as_str())); } + #[test] + fn lower_prompt_moves_with_the_stance_alone() { + use weksa::packet::{FeatValue, FeatureValue, Grade, Stance}; + let (_, inv, entities, item) = dao(2); + let store = InventoryStore::open(&data("daodejing/inventory.cc")).unwrap(); + let own = packets().remove(2).stance; + let grade = |feature: &str, v: u16| FeatureValue { + feature: feature.parse().unwrap(), + value: FeatValue::Grade(Grade::new(v).unwrap()), + }; + let angry = Stance { + features: vec![grade("feature:9", 800), grade("feature:18", 900)], + }; + let prompt_of = |stance: Stance| { + let mut packet = packets().remove(2); + packet.stance = stance.clone(); + let view = admitted(packet, &store, "daodejing/corpus.cc", &entities); + let prompt = Prompt::lower(&view, &item, &inv, &entities).unwrap(); + (stance, prompt) + }; + let three = [prompt_of(own), prompt_of(angry), prompt_of(Stance::default())]; + + // Three stances, three prompts. + let digests: std::collections::BTreeSet = + three.iter().map(|(_, p)| p.digest()).collect(); + assert_eq!(digests.len(), 3); + // Each packet line ends with exactly that stance's JSON, and the prompt + // with the packet line taken out is the same text for all three. + let without_packet_line = |text: &str| -> String { + text.lines().filter(|l| !l.starts_with("packet: ")).collect::>().join("\n") + }; + let bare = without_packet_line(three[0].1.text()); + for (stance, prompt) in &three { + let line = prompt.text().lines().find(|l| l.starts_with("packet: ")).unwrap(); + let json = serde_json::to_string(stance).unwrap(); + assert!(line.ends_with(&format!(r#","stance":{json}}}"#)), "{line}"); + assert_eq!(without_packet_line(prompt.text()), bare); + } + // The stance's feature ids are glossed in the one inventory listing. + let text = three[1].1.text(); + assert!(text.contains("\nfeature:9: anger [grade; slots Stance]\n")); + assert!(text.contains("\nfeature:18: restraint [grade; slots Stance]\n")); + } + #[test] fn an_entity_referent_prints_its_en_us_forms_citation_first() { let cast = InventoryStore::open(&data("interlingua/inventory.cc")).unwrap(); From 00feb1d31071fcebfdacdbee55a48a335091a8f1 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 18:28:14 +0000 Subject: [PATCH 07/80] Lowering brief states the grade scale LOWER_INSTRUCTIONS gains 'a grade is permille, 0 to 1000' (the encode brief already states it). The pinned lowering test requires the sentence and asserts the item's speaker entity id and group are in no prompt text. This edits context.rs, a PROMPT_SOURCE_DIGEST input: the digest moves and runs opened before this commit stop opening; open a fresh run to ablate under the new brief. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/context.rs | 2 +- crates/weksa/tests/context.rs | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index 1a648a0..8e2cbfc 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -66,7 +66,7 @@ A feature is {feature, value}; value is {\"Grade\": 0 to 1000}, {\"Option\": lab /// Fixed per run by `PROMPT_SOURCE_DIGEST`. pub const LOWER_INSTRUCTIONS: &str = "\ Say the packet below in English (en-US), as one utterance of its speaker: all of its meaning and nothing it does not carry. -Every id is the inventory's listed below and every feature value is as listed. The stance is the speaker's felt affect and restraint: it shapes how the utterance is said, not what it says. +Every id is the inventory's listed below and every feature value is as listed; a grade is permille, 0 to 1000. The stance is the speaker's felt affect and restraint: it shapes how the utterance is said, not what it says. A referent with an entity is named by one of that entity's forms on the entities line. Answer with one JSON object and nothing else: {\"text\": \"...\"}. No code fence, no commentary."; diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs index 32d3306..5979a26 100644 --- a/crates/weksa/tests/context.rs +++ b/crates/weksa/tests/context.rs @@ -630,6 +630,7 @@ mod lowering { for needed in [ "en-US", "all of its meaning and nothing it does not carry", + "a grade is permille, 0 to 1000", "felt affect and restraint", "shapes how the utterance is said, not what it says", "forms on the entities line", @@ -640,6 +641,17 @@ mod lowering { // The one call it makes is its item; the key is in no text. assert_eq!(prompt.item_keys(), [item.as_str()]); assert!(!prompt.text().contains(item.as_str())); + // The view's speaker and group are private: the item's speaker entity + // id and its group are in no text. + let held = CorpusStore::open(&data("daodejing/corpus.cc")) + .unwrap() + .get(item.as_str()) + .unwrap() + .unwrap(); + assert!(!held.group.is_empty()); + for private in [held.speaker.as_str(), held.group.as_str()] { + assert!(!prompt.text().contains(private), "{private}"); + } } #[test] From e8a9bbc5abce05a336890a87cf18e6b13eb92569 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 19:01:24 +0000 Subject: [PATCH 08/80] Ablation reads none for an empty set, pins its line, names a missing round answer Soul s1 findings: an empty mean printed 0.00 beside n=0 (now Option, printed none, like SplitMean); the printed line and whole ablate output are pinned by exact string, with drop and permute each carrying their own n; a missing recorded round answer is RunError::Unrecorded and says the prompt digest drifted or the head inventory moved, not that a store would not read; the unusable count is pinned for variant and item sides. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/run.rs | 45 +++++++---- crates/weksa/tests/cli.rs | 24 +++++- crates/weksa/tests/golf_loop.rs | 131 ++++++++++++++++++++++++++++---- 3 files changed, 167 insertions(+), 33 deletions(-) diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 958edf1..dbbf6ad 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -455,6 +455,9 @@ pub enum RunError { NoFloor, /// A round has no Validation reading. NoValidation, + /// The round holds no answer to a lowering prompt the ablation rebuilt: the + /// prompt's digest drifted or the head inventory moved since the round. + Unrecorded, } impl fmt::Display for RunError { @@ -470,6 +473,9 @@ impl fmt::Display for RunError { RunError::NoInventory => f.write_str("the inventory holds no revision"), RunError::Io(kind) => write!(f, "io error {kind:?}"), RunError::Store => f.write_str("a store would not read or write"), + RunError::Unrecorded => f.write_str( + "the round holds no answer to a lowering prompt rebuilt now: the prompt digest drifted or the head inventory moved since the round", + ), RunError::Embed => f.write_str("the embedder failed"), RunError::NoFloor => f.write_str("the run has no noise floor: record two rounds first"), RunError::NoValidation => f.write_str("a reading has no validation item"), @@ -969,10 +975,12 @@ pub struct AblationReading { pub permute: VariantReading, } +/// Means over an empty set are `None`, printed `none`: no measurement, never +/// a zero. #[derive(Clone, Debug, PartialEq, Default)] pub struct Agreement { - pub chrf: f64, - pub meaning: f64, + pub chrf: Option, + pub meaning: Option, } /// One variant's reading: agreement with the recorded lowering, pairs whose @@ -981,27 +989,36 @@ pub struct Agreement { #[derive(Clone, Debug, PartialEq, Default)] pub struct VariantReading { pub n: u32, - pub chrf: f64, - pub meaning: f64, + pub chrf: Option, + pub meaning: Option, pub moved: u32, - pub ref_gain_chrf: f64, - pub ref_gain_meaning: f64, + pub ref_gain_chrf: Option, + pub ref_gain_meaning: Option, +} + +fn show(x: Option, places: usize) -> String { + x.map_or("none".to_string(), |x| format!("{x:.places$}")) } impl AblationReading { pub fn line(&self) -> String { let variant = |name: &str, v: &VariantReading| { format!( - "{name} n={} chrf={:.2} meaning={:.3} moved={} ref_gain_chrf={:.2} ref_gain_meaning={:.3}", - v.n, v.chrf, v.meaning, v.moved, v.ref_gain_chrf, v.ref_gain_meaning + "{name} n={} chrf={} meaning={} moved={} ref_gain_chrf={} ref_gain_meaning={}", + v.n, + show(v.chrf, 2), + show(v.meaning, 3), + v.moved, + show(v.ref_gain_chrf, 2), + show(v.ref_gain_meaning, 3) ) }; format!( - "ablation n={} unusable={} replay chrf={:.2} meaning={:.3} | {} | {}", + "ablation n={} unusable={} replay chrf={} meaning={} | {} | {}", self.n, self.unusable, - self.replay.chrf, - self.replay.meaning, + show(self.replay.chrf, 2), + show(self.replay.meaning, 3), variant("drop", &self.drop), variant("permute", &self.permute), ) @@ -1034,9 +1051,9 @@ fn asked(port: &mut dyn ModelPort, prompt: &Prompt) -> Result { } } -fn mean(xs: impl Iterator) -> f64 { +fn mean(xs: impl Iterator) -> Option { let (sum, n) = xs.fold((0.0, 0u32), |(s, n), x| (s + x, n + 1)); - if n == 0 { 0.0 } else { sum / f64::from(n) } + (n > 0).then(|| sum / f64::from(n)) } type Scored = (ItemScore, ItemScore); @@ -1143,7 +1160,7 @@ impl RunStores { } // The round's own answer must exist; a missing one is not a gap to ask for. let recorded_text = match asked(recorded, &base)? { - Asked::Waiting => return Err(RunError::Store), + Asked::Waiting => return Err(RunError::Unrecorded), other => other, }; let replay = asked(model, &base)?; diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 3d38bba..f5562cd 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -1696,11 +1696,27 @@ fn loop_ablate_waits_then_prints_one_line_per_split() { let second = ablate(); assert!(second.status.success(), "{}", out(&second)); let text = out(&second); - let lines: Vec<&str> = text.lines().collect(); - assert_eq!(lines.len(), 2, "{text}"); - assert!(lines[0].starts_with("train ablation n=1 "), "{text}"); - assert!(lines[1].starts_with("validation ablation n=2 "), "{text}"); + assert_eq!( + text, + "train ablation n=1 unusable=0 replay chrf=100.00 meaning=1.000 | drop n=1 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000 | permute n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none\nvalidation ablation n=2 unusable=0 replay chrf=100.00 meaning=1.000 | drop n=2 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000 | permute n=2 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000\n" + ); for leaked in ["item:", "packet:", "highest excellence", "canaryroot"] { assert!(!text.contains(leaked), "{text}"); } } + +#[test] +fn loop_ablate_without_the_rounds_answers_says_so() { + let d = tempfile::tempdir().unwrap(); + open_dao_run(d.path()); + let url = fake_embedder(); + let root = d.path().join("canaryroot"); + let mut run = weksa::run::RunStores::open(&root).unwrap(); + let scorer = weksa::Scorer::new(weksa::OllamaEmbedder::new(&url, &run.record().embedder)); + weksa::run::round(&mut run, &mut DaoModel, &scorer).unwrap(); + let o = weksa(d.path(), &["loop", "ablate", "canaryroot", "--round", "1", "--ollama-url", &url]); + assert!(!o.status.success()); + let text = out(&o); + assert!(text.contains("holds no answer") && text.contains("head inventory"), "{text}"); + assert!(!text.contains("would not read"), "{text}"); +} diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index c9c4bd6..4a1c7ee 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -3773,12 +3773,12 @@ fn a_lowerer_blind_to_stance_reads_no_movement_and_one_that_reads_it_does() { for (split, r) in one(blind) { assert_eq!(r.unusable, 0); assert_eq!((r.drop.moved, r.permute.moved), (0, 0), "{split:?}: {}", r.line()); - assert!((r.replay.chrf - 100.0).abs() < 1e-9); - assert!((r.drop.chrf - r.replay.chrf).abs() < 1e-9); + assert!((r.replay.chrf.unwrap() - 100.0).abs() < 1e-9); + assert!((r.drop.chrf.unwrap() - r.replay.chrf.unwrap()).abs() < 1e-9); if r.permute.n > 0 { - assert!((r.permute.chrf - r.replay.chrf).abs() < 1e-9); + assert!((r.permute.chrf.unwrap() - r.replay.chrf.unwrap()).abs() < 1e-9); } - assert_eq!(r.drop.ref_gain_chrf, 0.0); + assert_eq!(r.drop.ref_gain_chrf, Some(0.0)); } let reading = one(reader); let (train, validation) = (&reading[0].1, &reading[1].1); @@ -3788,9 +3788,23 @@ fn a_lowerer_blind_to_stance_reads_no_movement_and_one_that_reads_it_does() { assert_eq!((train.permute.n, train.permute.moved), (0, 0)); assert_eq!((validation.drop.n, validation.drop.moved), (3, 3)); assert_eq!((validation.permute.n, validation.permute.moved), (3, 3)); - assert!((validation.replay.chrf - 100.0).abs() < 1e-9); - assert!(validation.drop.chrf < validation.replay.chrf); - assert!(validation.permute.chrf < validation.replay.chrf); + // A lone Train item has no permute: no measurement, not total disagreement. + assert_eq!( + train.permute, + weksa::run::VariantReading::default(), + "{}", + train.line() + ); + assert!( + train.line().contains( + "| permute n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none" + ), + "{}", + train.line() + ); + assert!((validation.replay.chrf.unwrap() - 100.0).abs() < 1e-9); + assert!(validation.drop.chrf.unwrap() < validation.replay.chrf.unwrap()); + assert!(validation.permute.chrf.unwrap() < validation.replay.chrf.unwrap()); } #[test] @@ -3801,15 +3815,15 @@ fn the_item_s_own_stance_helps_fidelity_only_for_a_lowerer_that_uses_it() { read(ablate(&dao, &mut rec, &mut model).unwrap()) }; for (split, r) in one(helped) { - assert!(r.drop.ref_gain_chrf > 10.0, "{split:?}: {}", r.line()); - assert!(r.drop.ref_gain_meaning > 0.0, "{split:?}: {}", r.line()); + assert!(r.drop.ref_gain_chrf.unwrap() > 10.0, "{split:?}: {}", r.line()); + assert!(r.drop.ref_gain_meaning.unwrap() > 0.0, "{split:?}: {}", r.line()); if r.permute.n > 0 { - assert!(r.permute.ref_gain_chrf > 10.0, "{split:?}: {}", r.line()); + assert!(r.permute.ref_gain_chrf.unwrap() > 10.0, "{split:?}: {}", r.line()); } assert_eq!(r.drop.moved, r.drop.n); } for (split, r) in one(blind) { - assert_eq!((r.drop.ref_gain_chrf, r.drop.ref_gain_meaning), (0.0, 0.0), "{split:?}"); + assert_eq!((r.drop.ref_gain_chrf, r.drop.ref_gain_meaning), (Some(0.0), Some(0.0)), "{split:?}"); } } @@ -3962,10 +3976,14 @@ fn a_stance_ablation_refuses_a_cycle_round_and_missing_round_answers() { dao.run.stance_ablation(2, &mut rec, &mut model, &Scorer::new(Bag)), Err(RunError::Malformed("round")) )); - assert!(matches!( - ablate(&dao, &mut rec, &mut model), - Err(RunError::Store) - )); + let Err(missing) = ablate(&dao, &mut rec, &mut model) else { + panic!("a missing round answer is refused") + }; + assert!(matches!(missing, RunError::Unrecorded)); + // It says what happened, not that a store would not read. + let said = missing.to_string(); + assert!(said.contains("holds no answer") && said.contains("digest") && said.contains("head inventory"), "{said}"); + assert!(!said.contains("would not read"), "{said}"); assert!( std::fs::read_dir(dao.root.join("round-1")) .unwrap() @@ -4008,3 +4026,86 @@ fn a_stance_ablation_refuses_a_cycle_round() { )); assert!(model.seen.is_empty() && rec.seen.is_empty()); } + +/// The real sentence for the item's own stance, an empty text (not a usable +/// lowering) for any other. +fn mute_unless_own(key: &str, text: &str) -> String { + let said = helped(key, text); + if said == "zzz qqq xxx" { String::new() } else { said } +} + +#[test] +fn unusable_variant_answers_are_counted_and_read_as_none_not_zero() { + let (dao, _) = recorded_dao(|_| {}); + let (mut rec, mut model) = (Lowerer::new(mute_unless_own), Lowerer::new(mute_unless_own)); + let reading = read(ablate(&dao, &mut rec, &mut model).unwrap()); + let (train, validation) = (&reading[0].1, &reading[1].1); + // Every variant answer is unusable: Train has one drop, Validation three + // drops and three permutes. The items themselves (replay) still read. + assert_eq!((train.n, train.unusable), (1, 1)); + assert_eq!((validation.n, validation.unusable), (3, 6)); + for r in [train, validation] { + assert!((r.replay.chrf.unwrap() - 100.0).abs() < 1e-9); + assert_eq!(r.drop.n, 0); + assert_eq!(r.permute.n, 0); + assert_eq!(r.drop, weksa::run::VariantReading::default()); + assert_eq!(r.permute, weksa::run::VariantReading::default()); + } + assert_eq!( + train.line(), + "ablation n=1 unusable=1 replay chrf=100.00 meaning=1.000 | drop n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none | permute n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none" + ); +} + +#[test] +fn the_line_names_each_variant_with_its_own_n_and_the_splits_n() { + use weksa::run::{AblationReading, Agreement, VariantReading}; + let reading = AblationReading { + n: 7, + unusable: 2, + replay: Agreement { chrf: Some(91.5), meaning: Some(0.9) }, + drop: VariantReading { + n: 5, + chrf: Some(80.25), + meaning: Some(0.8), + moved: 4, + ref_gain_chrf: Some(3.5), + ref_gain_meaning: Some(-0.125), + }, + permute: VariantReading { + n: 3, + chrf: Some(70.0), + meaning: None, + moved: 1, + ref_gain_chrf: None, + ref_gain_meaning: Some(0.5), + }, + }; + assert_eq!( + reading.line(), + "ablation n=7 unusable=2 replay chrf=91.50 meaning=0.900 | drop n=5 chrf=80.25 meaning=0.800 moved=4 ref_gain_chrf=3.50 ref_gain_meaning=-0.125 | permute n=3 chrf=70.00 meaning=none moved=1 ref_gain_chrf=none ref_gain_meaning=0.500" + ); +} + +fn nothing_usable(_: &str, _: &str) -> String { + String::new() +} + +#[test] +fn a_split_where_no_replay_is_usable_reads_none_throughout() { + let (dao, _) = recorded_dao(|_| {}); + let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(nothing_usable)); + let reading = read(ablate(&dao, &mut rec, &mut model).unwrap()); + // Train: the item and its drop; Validation: three items, three drops, three permutes. + assert_eq!((reading[0].1.n, reading[0].1.unusable), (0, 2)); + assert_eq!((reading[1].1.n, reading[1].1.unusable), (0, 9)); + for (_, r) in &reading { + assert_eq!(r.replay, weksa::run::Agreement::default()); + assert_eq!(r.drop, weksa::run::VariantReading::default()); + assert_eq!(r.permute, weksa::run::VariantReading::default()); + } + assert_eq!( + reading[0].1.line(), + "ablation n=0 unusable=2 replay chrf=none meaning=none | drop n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none | permute n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none" + ); +} From 707e0204bf54559d1ff6c137abe9be2eb89fdcbf Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 19:27:48 +0000 Subject: [PATCH 09/80] Encode prompt: participants are node ids, never an entity id Both bad-form refusals in orig-5 put the speaker entity (entity:keeper) in context.participants. One sentence, pinned by a plain test; no new rule. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/context.rs | 2 +- crates/weksa/tests/context.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index 8e2cbfc..28c43db 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -59,7 +59,7 @@ Fields: - predications: a list of {id, concept, frame, roles, features}; roles is a list of {role, filler}, filler a node id. - relations: a list of {relation, from, to, features}. - stance: {features}; the speaker's felt affect and restraint, never the affect the line displays. -Node ids are n1, n2, and so on, one per referent and predication. Every other id comes from the inventory listed below and must be live at its revision. Write each id in full, exactly as the inventory lines spell it, family prefix included: concept:7, never 7. An entity comes only from the entities line. A referent's concept is never null, even when it carries an entity: give the nearest concept id. +Node ids are n1, n2, and so on, one per referent and predication. Every other id comes from the inventory listed below and must be live at its revision. Write each id in full, exactly as the inventory lines spell it, family prefix included: concept:7, never 7. An entity comes only from the entities line. A referent's concept is never null, even when it carries an entity: give the nearest concept id. Participants and audience list the node ids of referents, never an entity id: an entity speaker is a referent whose entity field holds it, and participants lists that referent's node id. A feature is {feature, value}; value is {\"Grade\": 0 to 1000}, {\"Option\": label} or {\"Flag\": true or false}, as the inventory lists the feature's kind."; /// The instructions a lowering prompt starts with: the lowerer's whole brief. diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs index 5979a26..b908dcb 100644 --- a/crates/weksa/tests/context.rs +++ b/crates/weksa/tests/context.rs @@ -751,6 +751,7 @@ mod lowering { fn the_encode_prompt_states_the_id_form_the_inventory_lines_use() { assert!(ENCODE.contains("family prefix included: concept:7, never 7")); assert!(ENCODE.contains("A referent's concept is never null, even when it carries an entity: give the nearest concept id.")); + assert!(ENCODE.contains("Participants and audience list the node ids of referents, never an entity id: an entity speaker is a referent whose entity field holds it, and participants lists that referent's node id.")); let w = world(); let run = run_over(w.dir.path()); let inv = run.inventory().inventory().unwrap(); From ff3261a50f1423bbe0f08b8f63b4add93576cbce Mon Sep 17 00:00:00 2001 From: Weksa Date: Thu, 12 Nov 2026 09:46:50 +0000 Subject: [PATCH 10/80] Lows: show-working publishes by allowlist and prints Display, edition_year says first appearance The split guard failed open on any new split name; it now allows Train and split-less items only. The page printed Debug of Refusal/StoreError and the EntityRead store message; it prints Display and a fixed line. edition_year held the first-appearance year of the work or translation, not the page read: the doc comment and published schema say so. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/examples/show_working.rs | 6 +++--- crates/weksa/src/corpus.rs | 6 +++++- schemas/cultnet/weksa.corpus_item.schema.json | 3 ++- tools/show-working/build.py | 4 +++- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/crates/weksa/examples/show_working.rs b/crates/weksa/examples/show_working.rs index dae9ca9..8c23fb2 100644 --- a/crates/weksa/examples/show_working.rs +++ b/crates/weksa/examples/show_working.rs @@ -258,7 +258,7 @@ fn lower_error(e: &LowerError) -> Value { LowerError::DanglingStress { entity, at } => ("DanglingStress", format!("the ruled pronunciation of {entity} has a stress mark at {at} with no vowel after it")), LowerError::ConsentMissing { predication } => ("ConsentMissing", format!("{predication} is a request the packet does not carry the addressee's consent to")), LowerError::AskerNotAddressee { predication } => ("AskerNotAddressee", format!("the agent of {predication} is not the addressee")), - LowerError::EntityRead { entity, why } => ("EntityRead", format!("reading {entity}: {why}")), + LowerError::EntityRead { entity, .. } => ("EntityRead", format!("the entity store failed reading {entity}")), }; json!({ "variant": variant, "reason": reason }) } @@ -279,11 +279,11 @@ fn run(w: &World, dir: &Path, packet: &Packet) -> Result<(String, Value), Value> Err(StoreError::Invalid(rs)) => { return Err(json!({ "kind": "admission_refused", - "refusals": rs.iter().map(|r| json!({ "rule": r.rule(), "detail": format!("{r:?}") })).collect::>(), + "refusals": rs.iter().map(|r| json!({ "rule": r.rule(), "detail": r.to_string() })).collect::>(), })); } Err(e) => { - return Err(json!({ "kind": "admission_refused", "refusals": [{ "rule": "store", "detail": format!("{e:?}") }] })); + return Err(json!({ "kind": "admission_refused", "refusals": [{ "rule": "store", "detail": e.to_string() }] })); } }; let text = lower(&view, &w.inv, &w.entities).map_err(|e| { diff --git a/crates/weksa/src/corpus.rs b/crates/weksa/src/corpus.rs index a2f94ae..4afa662 100644 --- a/crates/weksa/src/corpus.rs +++ b/crates/weksa/src/corpus.rs @@ -32,7 +32,7 @@ pub enum Source { /// A public-domain work, with everything a reader needs to check the claim: /// the edition's licence and the jurisdiction the public-domain status is /// asserted in, who translated it (the translation is the text's copyright, - /// not the original's) and when the edition appeared. A work missing any + /// not the original's) and when the work or translation first appeared. A work missing any /// is refused at admission. PublicDomain { work: String, @@ -41,6 +41,10 @@ pub enum Source { licence: String, jurisdiction: String, translator: String, + /// The year the work, or for a translation this translation, first + /// appeared. It is not the year of the `edition_url` file's source + /// print: Project Gutenberg's text may be a later or modernised + /// edition, and the claim rests on the date of first appearance. edition_year: u16, }, /// Labelled model output; never in the test split. The rule that it is diff --git a/schemas/cultnet/weksa.corpus_item.schema.json b/schemas/cultnet/weksa.corpus_item.schema.json index b5e5bc9..1bd6083 100644 --- a/schemas/cultnet/weksa.corpus_item.schema.json +++ b/schemas/cultnet/weksa.corpus_item.schema.json @@ -170,7 +170,7 @@ ] }, { - "description": "A public-domain work, with everything a reader needs to check the claim:\nthe edition's licence and the jurisdiction the public-domain status is\nasserted in, who translated it (the translation is the text's copyright,\nnot the original's) and when the edition appeared. A work missing any\nis refused at admission.", + "description": "A public-domain work, with everything a reader needs to check the claim:\nthe edition's licence and the jurisdiction the public-domain status is\nasserted in, who translated it (the translation is the text's copyright,\nnot the original's) and when the work or translation first appeared. A work missing any\nis refused at admission.", "type": "object", "properties": { "PublicDomain": { @@ -183,6 +183,7 @@ "type": "string" }, "edition_year": { + "description": "The year the work, or for a translation this translation, first\nappeared. It is not the year of the `edition_url` file's source\nprint: Project Gutenberg's text may be a later or modernised\nedition, and the claim rests on the date of first appearance.", "type": "integer", "format": "uint16", "maximum": 65535, diff --git a/tools/show-working/build.py b/tools/show-working/build.py index 84e5189..ede5458 100755 --- a/tools/show-working/build.py +++ b/tools/show-working/build.py @@ -59,7 +59,9 @@ for m in re.finditer(r'^(item:\S+) \[(.*?)\]\n "(.*)"$', run([str(weksa_bin), " if set(corpus) != cited: sys.exit(f"corpus dump lacks cited items {sorted(cited - set(corpus))}; unexpected {sorted(set(corpus) - cited)}") # The dump's meta ends with the split (`... group=g Validation`, `... excluded`). -held_out = {k: v["meta"].split()[-1] for k, v in corpus.items() if v["meta"].split()[-1] in ("Validation", "Test")} +# Only Train and split-less items publish; any other name, including a split +# added later, is held out until it is allowed here. +held_out = {k: v["meta"].split()[-1] for k, v in corpus.items() if v["meta"].split()[-1] not in ("Train", "excluded")} if held_out: sys.exit(f"refusing to publish held-out items (nothing written): {held_out}") model = "qwen3-embedding:0.6b" From b83929d4d295caab46282c5784737878103c65df Mon Sep 17 00:00:00 2001 From: Weksa Date: Thu, 12 Nov 2026 12:16:19 +0000 Subject: [PATCH 11/80] Map: cut 9 ja-panel specced as ja-panel-loop and ja-panel-corpus; panel rows in Loop round and decision Co-Authored-By: Claude Sonnet 5.5 --- docs/targets/interlingua-lowering-map.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 4846466..50c685b 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -40,8 +40,8 @@ means no store holds it and nothing writes it. | Scene (cut `speaker-scene`) | Key `scene:@`, the item's split group; type `weksa.scene`: participants (speaker slugs) and, per ordered pair, relation features of slot `Scene` (relative status, intimacy, in-group, audience and whatever the loop adds), at an inventory revision, and an `Annotator` | As Speaker. Keyed by the group, so a scene never straddles the split | As Speaker | | Corpus item | Key `item::`, `n` canonical decimal >= 1 (no sign, no leading zero, from cut 2b); type `weksa.corpus_item` with text, language, speaker, split group, prior turns, source. Source `ModelWritten` carries a `ModelFamily` enum, the generator string as the fixture wrote it, and a prompt digest (cut 2b). The item carries no speaker state of its own: its speaker slug and group name the Speaker and Scene documents | Admitted once; the whole item is immutable (re-adding an identical item is a no-op, any difference is refused; a correction is a new item that `supersedes` the old); text already held under another key is refused unless the new item supersedes that key (cut 2b); withdrawn, never deleted, so old scores stay explainable | Importers and `weksa corpus add`, under `corpus-cast-plus-public-domain`; the source enum has no variant for Discord or any training-forbidden source (`corpus-clean`) | | Split assignment | Derived: `split(scheme, item.group)` | Fixed for a scheme; a new scheme is a new constant, never a reassignment. A model-written item whose group derives test is in no split; a superseding item keeps its predecessor's group | The hash function. Nobody assigns by hand, so there is no writer to forbid | -| Loop round | Key `run: