From 9f8ae13a121f7d98aa7228609e15e5bd4ef7bea6 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 18:04:37 +0000 Subject: [PATCH 1/3] original corpus: 22 new model-written items through the corpus door (Train 14, Validation 14) Items original:7..28, written for the non-memorised target; groups chosen so the V1 split derives Train or Validation. Test is not filled: a model-written item of a test group is in no split (split.rs), so none can be held out. Co-Authored-By: Claude Sonnet 5.5 --- data/original/corpus.cc | Bin 2050 -> 8614 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/data/original/corpus.cc b/data/original/corpus.cc index c11443faa710ffebcb55952272ae15eccff27fbf..57988dfaf5fe31057a344f805d956cfc1bcde77a 100644 GIT binary patch literal 8614 zcmd6sO>Y}T7{}>_9=O1P0}_W(D*74+d*i&+7ZkN11*sLHZ=$F|WAE5Ics*lg##vd2 z1K%J~DskwU1E{41^#fEW{SYH@;Q*+JcX2>m$}=--Z@ROQSDfaSIJ2A0FVFLT{{Lt9 zzv)d<)x*7ntVqQLS=njrefD)mh7zyzxEN352UL-f|H(t|3|#6T)7L*BHTf?yt8Fk^zQs@0!pX~X+YWKV6UcG(aJZ*;yI-m@vor))V z7&W8yMyu6mHLteXoz_ODxq0*8*E{7ab#n`SG@(SH5oIx=5~XxVkW@5HQHB)}h%$sx zOvo5@d6uqxPFUmGm3xFKs-~SGd``r@*`qYfk=;-4Ug9xHZwRUsVf$ZfF_j2DragVD zm*PoG8j>?cRAa5NhECrw{|0o0r2S6ECxiKdGuxRS3ml8_x6Jlrlybc8BUv>=4vNu-8}g5j(4ek?Ja}EJVlBbaABxR z;JFB*3c)XzD8^z~N$0`ruWIC11`B9WAcl#_BP6>Owecwg~H&U$0unFl_Uro z0Jb@=j2e8X$45j~^13_cg=G>ftEg@<{p_)d^tsEt%i}3PU`p^*+PX>z-T_Rw%Xf(V zpifB}S0egqj>w(~7Sd{6NWXcIR(10#G@BC`qX#2sdH}@{8I9G{0_tK3>ZCX*g5X&t zsY03(=-*&Lwd;cV-UHRvCk!Y-cc7~TkuX0}0EN*QD+)4=Bp#3jY=$&t!LnJa%jO@C z&6+;-PD(J05-f+dWb%4~GV0ItJR@oKSi3*(vJ}TL5rGie+kbZYzVNa)>DKEa^ZRaH zue?VjD0GZnlbQ0dJx5{YCQxH$#v=@FL8fG^Dl4lXC&z-aK_KLOOR!NF)DK>5ZRnF1 zIam^fQm{|P7<8eK=9+FlF{Y{#P$A%$?a**_X|pb$Cmx?oeflCdiVb9J4LMV38qm=) zWn{JdjVe!BxTXY)E}w+beSQs{ugmCPkI{L3_QqU>tlO4EsT53Q-!jV8D#KLCN=ii? z*;yD!pOqErl@sO2vdBHnBFp1r$IF=|(%Z$n%LpD4=8CL~+0djFCpk``=I+xS5uAfl zH;HDVCL^|>HSI_Z1gTecQSCJ28PKSy&uklq0Op*t0oe9D*+@vQFe%|D&~vf1@s(9q z^k$EmQ4y?nh-xRApUS-%we%TNXEVc)Ky@7wvsomy8M(5K3g>C6F4!_!)QR)V=c!i| z>9beBc~6%QYn8E8U=)n%e~2Nrb8uRdBMjCd^%4ZffiNno*YTIi{wk`SYdqT!t?HAP za%YtXKB?e)=(>!?w)cl~(O~E$ht~8@6 zaf*-GfQVDo^;JMpP>jJs`v37JT6b#&f(7u1X>1MrSP`p zw)czX_H&x21(4}CBOvUI=CggYtsbE(j}(qm|f_ZxjdN4 zE4_tTG%e?}o|b8~^#wx<<{lb=VC-VL1c$_}KGha%=kSYYcXJwzh2Ukwdrda$RX`zab{`HENG6&-=CunN!$c zAk9LodXyFb;l`l?Tr@~$01uv&`K{}7TPT^Z0Za(D_kwqD6DYfs!wWux6+_BqQyVTT zm8|fx_WDo})gGaLlo4&{C{Nkub(qWAGcrce^4Jaf*26-WmZhpPZWpBB?KKO>mX&0z z?)*IXy-aFTw=RHRg`DV2z8jO44`H!2thP_V&%(Om=;2s^vUdozq-rk=e<_1HuU|KM gpYJ4Q_SG%HEFrrCOEH)Xw*{;0ykaVv|r>&fwNp3G|uVH8R3W&r@P(F%G1 -- 2.47.3 From 073b022c8a753d4103eea7d1c05cc43b9cfc606f Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 18:04:37 +0000 Subject: [PATCH 2/3] original_corpus tests: counts over 28 items, first six pinned, recall shows only live items Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/original_corpus.rs | 88 ++++++++++++++++++++++++--- 1 file changed, 78 insertions(+), 10 deletions(-) diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 91b635b..71cbf1a 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -25,7 +25,7 @@ fn stores() -> (CorpusStore, EntityStore) { fn every_original_item_records_its_author_and_none_is_public_domain() { let (corpus, entities) = stores(); let items = corpus.all().unwrap(); - assert_eq!(items.len(), 6); + assert_eq!(items.len(), 28); for item in &items { match &item.source { Source::ModelWritten { @@ -37,12 +37,18 @@ fn every_original_item_records_its_author_and_none_is_public_domain() { } other => panic!("{}: not model-written: {other:?}", item.id), } - assert!(entities.get(&item.speaker).unwrap().is_some(), "{}", item.id); + assert!( + entities.get(&item.speaker).unwrap().is_some(), + "{}", + item.id + ); } } #[test] fn the_original_items_spread_over_train_and_validation_and_none_is_in_test() { + // Model-written items cannot be in Test (`SplitScheme::split` excludes a + // model-written item of a test group), so this store holds none. let (corpus, _) = stores(); let splits: Vec> = corpus .all() @@ -51,10 +57,57 @@ fn the_original_items_spread_over_train_and_validation_and_none_is_in_test() { .map(|i| SplitScheme::V1.live(i)) .collect(); let count = |s| splits.iter().filter(|x| **x == Some(s)).count(); - assert_eq!((count(Split::Train), count(Split::Validation)), (3, 3)); + assert_eq!((count(Split::Train), count(Split::Validation)), (14, 14)); assert_eq!(count(Split::Test), 0); } +/// The first six items are the original golf-loop set; widening the store +/// adds items after them and never edits them. +#[test] +fn the_first_six_original_items_are_unchanged() { + let (corpus, _) = stores(); + let want = [ + ( + "item:original:1", + "A quiet mind is like still water in a deep bowl.", + "original-bowl", + ), + ( + "item:original:2", + "The river weakens the bank in spring and strengthens the roots in autumn.", + "original-river", + ), + ( + "item:original:3", + "The sage waits at the door until the water has settled.", + "original-door", + ), + ( + "item:original:4", + "The old gardener empties the jar and fills it again with rainwater.", + "original-jar", + ), + ( + "item:original:5", + "Under the same sky the earth keeps its patience, and the baker keeps hers.", + "original-sky", + ), + ( + "item:original:6", + "A hungry belly argues louder than any will.", + "original-belly", + ), + ]; + for (id, text, group) in want { + let item = corpus.get(id).unwrap().unwrap(); + assert_eq!( + (item.text.as_str(), item.group.as_str()), + (text, group), + "{id}" + ); + } +} + #[test] fn the_original_inventory_is_the_dao_revision_three() { let dao = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/daodejing/inventory.cc"); @@ -69,7 +122,7 @@ fn the_original_inventory_is_the_dao_revision_three() { /// The probe over the real original corpus: every item is offered, none names a /// work, and a model that answers with unrelated words reads at the noise floor. #[test] -fn the_recall_probe_covers_all_six_original_items_and_names_no_work() { +fn the_recall_probe_covers_every_live_original_item_and_names_no_work() { use weksa::model::{ModelPort, PortError}; use weksa::run::{RecallOutcome, RunPaths, RunStores}; struct Unrelated(Vec); @@ -107,9 +160,24 @@ fn the_recall_probe_covers_all_six_original_items_and_names_no_work() { let RecallOutcome::Read(r) = run.recall(&mut model).unwrap() else { panic!("answered"); }; - assert_eq!((r.n, r.answered), (6, 6)); + assert_eq!((r.n, r.answered), (28, 28)); assert!(r.chrf < 25.0, "{}", r.chrf); - assert_eq!(model.0.len(), 6); + assert_eq!(model.0.len(), 28); + // No item outside Train and Validation is shown to the model. + let all = CorpusStore::open(&c).unwrap().all().unwrap(); + for item in &all { + let live = SplitScheme::V1.live(item); + // A recall prompt shows the first half of the words. + let words: Vec<&str> = item.text.split_whitespace().collect(); + let opening = words[..words.len() / 2].join(" "); + let shown = model.0.iter().any(|t| t.contains(&opening)); + assert_eq!( + shown, + matches!(live, Some(Split::Train | Split::Validation)), + "{}", + item.id + ); + } assert!(model.0.iter().all(|t| t.contains("work: not published"))); assert!(model.0.iter().all(|t| !t.contains("translator:"))); } @@ -168,16 +236,16 @@ fn original_round(family: ModelFamily) -> (weksa::run::RunStores, weksa::run::Ro #[test] fn own_family_counts_and_labels() { - // The original corpus is six Anthropic-written items, all offered. + // The original corpus is Anthropic-written items, all offered. let (run, r) = original_round(ModelFamily::Anthropic); - assert_eq!(r.own_family(), 6); + assert_eq!(r.own_family(), 28); assert_eq!( r.parse_reasons().iter().map(|c| c.count).sum::(), - 6, + 28, "every item was refused at Parse and says why" ); let line = run.round_line(&r); - assert!(line.contains(" own-family 6"), "{line}"); + assert!(line.contains(" own-family 28"), "{line}"); // A different answerer family, or none recorded, labels nothing. for family in [ModelFamily::OpenAi, ModelFamily::Unrecorded] { -- 2.47.3 From 32368a998e114bfa93eb22a564735d3fb73f166d Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 18:09:47 +0000 Subject: [PATCH 3/3] original_corpus: recall probes only live Train/Validation over a fixture with withdrawn, model-written-Test and public-domain-Test items Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/original_corpus.rs | 90 +++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 71cbf1a..79273f5 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -2,6 +2,8 @@ //! non-memorised golf-loop target. It holds no public-domain text, so a model //! cannot have it in pretraining, and every item says who wrote it. +mod common; + use std::path::Path; use weksa::corpus::{ModelFamily, Source}; @@ -254,3 +256,91 @@ fn own_family_counts_and_labels() { assert!(!run.round_line(&r).contains("own-family")); } } + +/// Over a fixture store whose items are not all live: recall probes the live +/// Train and Validation items and no others (a withdrawn item, a model-written +/// item of a test group, a public-domain Test item). +#[test] +fn recall_probes_only_live_train_and_validation_items() { + use common::{cast, entities_in, group_in_bucket, item_in, run_over}; + use weksa::model::{ModelPort, PortError}; + use weksa::run::RecallOutcome; + struct Record(Vec); + impl ModelPort for Record { + fn model(&self) -> &str { + "record" + } + fn answer(&mut self, p: &weksa::context::Prompt) -> Result { + self.0.push(p.text().to_string()); + Ok(r#"{"text":"nothing"}"#.to_string()) + } + } + let dir = tempfile::tempdir().unwrap(); + let entities = entities_in(dir.path()); + let mut store = CorpusStore::open(&dir.path().join("c.cc")).unwrap(); + let public = Source::PublicDomain { + work: "A Work".into(), + author: "An Author".into(), + edition_url: "https://example.org/e".into(), + licence: "CC0".into(), + jurisdiction: "US".into(), + translator: "A Translator".into(), + edition_year: 1900, + }; + let rows = [ + ( + "item:f:1", + "trainmark opens here and goes on", + 10, + cast(), + false, + ), + ( + "item:f:2", + "validmark opens here and goes on", + 50, + cast(), + false, + ), + ( + "item:f:3", + "gonemark opens here and goes on", + 10, + cast(), + true, + ), + ( + "item:f:4", + "modelmark opens here and goes on", + 90, + common::model(), + false, + ), + ( + "item:f:5", + "publicmark opens here and goes on", + 90, + public, + false, + ), + ]; + for (id, text, bucket, source, withdrawn) in rows { + let item = item_in(id, text, &group_in_bucket(bucket), source); + store.add(item, &entities).unwrap(); + if withdrawn { + store.withdraw(id).unwrap(); + } + } + drop(store); + let run = run_over(dir.path()); + let mut m = Record(vec![]); + let RecallOutcome::Read(r) = run.recall(&mut m).unwrap() else { + panic!("answered"); + }; + assert_eq!(r.n, 2); + let shown = |mark: &str| m.0.iter().any(|t| t.contains(mark)); + assert!(shown("trainmark") && shown("validmark")); + for hidden in ["gonemark", "modelmark", "publicmark"] { + assert!(!shown(hidden), "recall showed {hidden}"); + } +} -- 2.47.3