From e7aef81309349a3f42739b6df8897e14b2c76819 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 18:24:37 +0000 Subject: [PATCH] data/original: 8 public-domain Test items with full provenance; Test held out of rounds and recall Eight one-sentence English excerpts from Dhammapada (Muller), Walden, Emerson, Bacon, Aesop (Townsend), Mill, Plato (Jowett), Darwin, each taken from the exact Project Gutenberg page named in edition_url and checked verbatim against the fetched text. Groups hash into Test (buckets 83-97); split.rs untouched, the ModelWritten-never-in-Test rule stands. Items 1..28 are byte-identical (digest pinned). Tests: counts Train/Validation 14/14, Test 8 all PublicDomain; no Test text in any round-1 or recall prompt or RoundRecord over data/original; a ModelWritten item in a Test group is read in no split. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/original_corpus.rs | 217 ++++++++++++++++++++++++-- data/original/corpus.cc | Bin 8614 -> 13160 bytes data/original/entity.cc | 2 +- 3 files changed, 203 insertions(+), 16 deletions(-) diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 79273f5..186cb2a 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -1,6 +1,7 @@ -//! The original corpus (`data/original`): passages written for the -//! non-memorised golf-loop target. It holds no public-domain text, so a model -//! cannot have it in pretraining, and every item says who wrote it. +//! The original corpus (`data/original`): 28 passages written for the +//! non-memorised golf-loop target (Train and Validation, all model-written) +//! and 8 public-domain Test items, each with its full provenance. Test is only +//! read at a checkpoint: no round or recall prompt over this store holds it. mod common; @@ -24,20 +25,58 @@ fn stores() -> (CorpusStore, EntityStore) { } #[test] -fn every_original_item_records_its_author_and_none_is_public_domain() { +fn every_original_item_records_its_author_and_the_public_domain_ones_their_provenance() { let (corpus, entities) = stores(); let items = corpus.all().unwrap(); - assert_eq!(items.len(), 28); + assert_eq!(items.len(), 36); + let (mut model, mut public) = (0, 0); for item in &items { match &item.source { Source::ModelWritten { family, generator, .. } => { + model += 1; + assert!(item.id.starts_with("item:original:"), "{}", item.id); assert_eq!(*family, ModelFamily::Anthropic, "{}", item.id); let generator = generator.as_deref().unwrap_or_default(); assert!(generator.contains("claude-sonnet-5-5"), "{}", item.id); } - other => panic!("{}: not model-written: {other:?}", item.id), + Source::PublicDomain { + work, + author, + edition_url, + licence, + jurisdiction, + translator, + edition_year, + } => { + public += 1; + assert!(item.id.starts_with("item:holdout:"), "{}", item.id); + // The edition is the page the text was taken from. + assert!( + edition_url.starts_with("https://www.gutenberg.org/cache/epub/") + && edition_url.ends_with(".txt"), + "{}", + item.id + ); + assert!(licence.contains("Public domain"), "{}", item.id); + assert!( + licence.contains("trademark or licence matter"), + "{}", + item.id + ); + assert!(jurisdiction.starts_with("United States"), "{}", item.id); + assert!( + [work, author, translator] + .iter() + .all(|s| !s.trim().is_empty()), + "{}", + item.id + ); + // Every edition and every author is out of copyright by date. + assert!((1600..=1900).contains(edition_year), "{}", item.id); + } + other => panic!("{}: not model-written or public-domain: {other:?}", item.id), } assert!( entities.get(&item.speaker).unwrap().is_some(), @@ -45,24 +84,67 @@ fn every_original_item_records_its_author_and_none_is_public_domain() { item.id ); } + assert_eq!((model, public), (28, 8)); } #[test] -fn the_original_items_spread_over_train_and_validation_and_none_is_in_test() { - // Model-written items cannot be in Test (`SplitScheme::split` excludes a - // model-written item of a test group), so this store holds none. +fn the_original_items_spread_over_train_and_validation_and_test_is_public_domain_only() { + // A model-written item of a Test group is in no split + // (`SplitScheme::split`), so Test holds the public-domain items alone. let (corpus, _) = stores(); - let splits: Vec> = corpus + let items = corpus.all().unwrap(); + let in_split = |s| { + items + .iter() + .filter(|i| SplitScheme::V1.live(i) == Some(s)) + .collect::>() + }; + assert_eq!(in_split(Split::Train).len(), 14); + assert_eq!(in_split(Split::Validation).len(), 14); + let test = in_split(Split::Test); + assert_eq!(test.len(), 8); + assert!( + test.iter() + .all(|i| matches!(i.source, Source::PublicDomain { .. })) + ); + // Train and Validation hold no public-domain item. + for s in [Split::Train, Split::Validation] { + assert!( + in_split(s) + .iter() + .all(|i| matches!(i.source, Source::ModelWritten { .. })) + ); + } +} + +/// The 28 model-written items are byte-for-byte what they were before Test +/// was added: id, text, group and source, folded in key order. +#[test] +fn the_28_model_written_items_are_unchanged() { + let (corpus, _) = stores(); + let mut items: Vec<_> = corpus .all() .unwrap() - .iter() - .map(|i| SplitScheme::V1.live(i)) + .into_iter() + .filter(|i| i.id.starts_with("item:original:")) .collect(); - let count = |s| splits.iter().filter(|x| **x == Some(s)).count(); - assert_eq!((count(Split::Train), count(Split::Validation)), (14, 14)); - assert_eq!(count(Split::Test), 0); + items.sort_by(|a, b| a.id.cmp(&b.id)); + assert_eq!(items.len(), 28); + let lines: Vec = items + .iter() + .map(|i| format!("{}\n{}\n{}\n{:?}", i.id, i.text, i.group, i.source)) + .collect(); + let parts: Vec<&[u8]> = lines.iter().map(|l| l.as_bytes()).collect(); + assert_eq!( + weksa::split::fnv1a64_of(&parts), + DIGEST_OF_THE_28, + "an original item changed" + ); } +/// Taken over the store at base da6bc7a, before any Test item existed. +const DIGEST_OF_THE_28: u64 = 3803723767212018190; + /// The first six items are the original golf-loop set; widening the store /// adds items after them and never edits them. #[test] @@ -344,3 +426,108 @@ fn recall_probes_only_live_train_and_validation_items() { assert!(!shown(hidden), "recall showed {hidden}"); } } + +/// Collects every prompt a round or recall shows the model; its answer is no +/// draft and no recall. +struct Record(Vec); + +impl weksa::model::ModelPort for Record { + fn model(&self) -> &str { + "nope" + } + fn answer(&mut self, p: &weksa::context::Prompt) -> Result { + self.0.push(p.text().to_string()); + Ok(r#"{"text":"nothing"}"#.to_string()) + } +} + +/// The public-domain Test items are held out: no prompt of round 1 over +/// `data/original`, no recall prompt, and no round record names one. +#[test] +fn no_test_item_is_offered_to_a_round_or_recall_or_recorded() { + use weksa::run::{LowererReach, RoundOutcome, RunPaths, RunStores, round}; + let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/original"); + let tmp = tempfile::tempdir().unwrap(); + for f in ["corpus.cc", "entity.cc", "inventory.cc"] { + std::fs::copy(src.join(f), tmp.path().join(f)).unwrap(); + } + let p = |f: &str| tmp.path().join(f); + let mut run = RunStores::create( + &p("run"), + "orig", + RunPaths { + corpus: &p("corpus.cc"), + inventory: &p("inventory.cc"), + entities: &p("entity.cc"), + }, + "nope", + "bag", + LowererReach::ToolLessRelay, + ModelFamily::Unrecorded, + ) + .unwrap(); + let items = CorpusStore::open(&p("corpus.cc")).unwrap().all().unwrap(); + let held: Vec<_> = items + .iter() + .filter(|i| SplitScheme::V1.live(i) == Some(Split::Test)) + .collect(); + assert_eq!(held.len(), 8); + // A Test item's text, or any run of five of its words, in a prompt. + let leaks = |prompts: &[String], item: &weksa::corpus::CorpusItem| { + let w: Vec<&str> = item.text.split_whitespace().collect(); + prompts + .iter() + .any(|t| t.contains(&item.text) || w.windows(5).any(|k| t.contains(&k.join(" ")))) + }; + let scorer = weksa::score::Scorer::new(Bag); + let mut model = Record(vec![]); + let RoundOutcome::Recorded(r) = round(&mut run, &mut model, &scorer).unwrap() else { + panic!("answered"); + }; + // The check bites: the round did show live items, 28 of them. + assert!(model.0.len() >= 28, "{}", model.0.len()); + assert!( + items + .iter() + .filter(|i| matches!(SplitScheme::V1.live(i), Some(Split::Train))) + .any(|i| leaks(&model.0, i)), + "the probe cannot see a shown Train item" + ); + for item in &held { + assert!(!leaks(&model.0, item), "round prompt held {}", item.id); + assert!(r.calls().iter().all(|c| !c.items.contains(&item.id))); + assert!(r.train().iter().all(|row| row.item != item.id)); + } + let mut recall = Record(vec![]); + let weksa::run::RecallOutcome::Read(rr) = run.recall(&mut recall).unwrap() else { + panic!("answered"); + }; + assert_eq!((rr.n, rr.answered), (28, 28)); + for item in &held { + assert!(!leaks(&recall.0, item), "recall prompt held {}", item.id); + } +} + +/// The exclusion stands: a model-written item in a Test group is read in no +/// split, and is not moved to Train or Validation. +#[test] +fn a_model_written_item_in_a_test_group_is_read_in_no_split() { + use common::{group_in_bucket, item_in, model}; + for bucket in [80, 99] { + let item = item_in("item:f:1", "x opens", &group_in_bucket(bucket), model()); + assert_eq!(SplitScheme::V1.split(&item), None, "bucket {bucket}"); + assert_eq!(SplitScheme::V1.live(&item), None); + } + // The same group holds a public-domain item in Test. + let public = Source::PublicDomain { + work: "w".into(), + author: "a".into(), + edition_url: "https://example.org/e".into(), + licence: "l".into(), + jurisdiction: "US".into(), + translator: "none".into(), + edition_year: 1900, + }; + let item = item_in("item:f:2", "x opens", &group_in_bucket(80), public); + assert_eq!(SplitScheme::V1.live(&item), Some(Split::Test)); +} diff --git a/data/original/corpus.cc b/data/original/corpus.cc index 57988dfaf5fe31057a344f805d956cfc1bcde77a..d778901e62568a8a52f6777b0c158717ccaf0df6 100644 GIT binary patch literal 13160 zcmd5@&yU;28Ld+^d+4P-^w8VDX=87^+LgH4U9AkXSnqlrr*&-DPK=~YgONBA>7_W# z3~5CuilXOU93(&v0eUHNXpq=W;sQM+K+xJp{{aMY$)OEW^f!7adP(0mLrGpzHi-fT zc!;%<$kFi4``-7yZzivQ&`*N6&-k7|rQ;?MGtINCj0xb5glcER&PX1;MytwyVPxz*`9PPf&3@$O^4d@FZn`t#G*M96v* z7JGsXco6vEkjViX^H`{ct7N1Z&)!8cX1-U>JAEF}h-Y%mP1-13|xM|5|-pgpKDe-4-@~Y}G~=C0 zT+DqrZF_MX>F(OvcsyPmCb0;yoWoVAhHHj_Ya&W|YfiJZwH6KOWi@^^PT#ao#a#1P z>^KkmK0slycr`Y^9O{6Bx@>=yo#U^vbHEyqx|sZSv$=edT~P8W9-N(-{-U{+B^omv z`>_Y2U;(sw2=G)LFboD9Mc5Z?#A7V?Nj4BMSnRSnd(zHxA;{*kOEDM$E`%=b8i*5H zO?EX=zIJ_sHc-n=>Be-XfB1-|dT;54cjm)>^84#K5<5kzKY+wXawK-rAD=bE91HLd ztKso$f@x6F@|O7_o5)15Q20ZyCzU6qyFwQZeO%Z}N>^#e4%=0<`lXyNogSl6b<-57@4cAYpsHmY@(;x-UL_ zT7+uC_V_j5#iiyqQl}!BHVl_G43)-tDkU^yLzT{^^9ZT3U70G`*?CA9E~7MoFSS=!mib9@k#DR+!dr5 z!+>jDWP=z%AY}OdlnvSY##WvVw@+)$C%Vh_eWhb|Nhn|F4=?f{@)!|CveR~`stU5* zZquscrF7Xl^;q`o0qwG5PG!s9dg!vZt1kOvU&}3fJAGvb>lz-@EaY*bugjgJ#6J zIhTjzx=H!fk z%@J&WLf+)E9D}QPS~4k}&ZA^dw=#nYlltw#Ip16Q!9%rYvnqLxq&f0zrZ1m@6@w9n)UW}lXzoZ%LJvfwf+bN zWwSCJ==m%^F`*JYS=xOlCAKP4;d(lrh-14d+mDha)k~Y)ly2tc zdT;6cL#ffJOpVWT)aaxyJqwM{kw7`9ZDqrw2>JkSpfsH@9msJoSz%_^_o29;~lMia0_eU`F}$3_zn z1$ts_1}Yg54guQ?F9bFTjp_O=#OvJ|UR-;%#j3pJlwtMf0#;6X;#t{~?gSu&sPqCP zZ1BnxXgwj=dJ>H#`e5LTz+DK@jTuDdn%aU?-dxE*`da~{^|W~z7+8U!Ud{+X51=Fy zqX=y<0;tC|Wh4njP;$HwsktpwREQE)cIYA&&^qs1>ALo zstH1`y?!{qeEnN*tZ7m)jsC`k0vWk#niTMLFKMK^=%_S}Po$ z(i}M*DfBLCg5322P~~-32Gk!5+}cW?+>;D;*>v@c9DztPO*d>KQ?-ye+b!BQRt=rb9Ua`lCtxv)&Q>!t@=G#J)<~%aFuqGJg73$khypbzB%G}emWVw4} zDPOZV>DhU{8w-9Q(0n)H$HN1vP_5)xg_^lfdx??~PMs>6Bu++QNobmsS`$bM(&bdH zW(o$>X{O6($sy1-gK0l!nNZ{W0Wu}NLCm2=2phkUb@SN7rUBJLwFRkiMe}(+Hl0?w zOgx(!1`q7IPt2wzsfpT!JeoUCVqI+!DvQKUH&gh6dO1$|)Fn7?^pEb-Dto}gd~%i3 zb{KXoaNJ-aME9u|HpShVV3b#{rElhKgHyS#DcA;QJ$>?A=B%=U4-I%a5Q6F%d&PLA z4IMu1t{qN_bHH#2qrSzlQ@Nb^vJiIdbmy#1ex@Te7TDo1L1tMtHjSxl2gAxf`H0xV zu?uQX6MPG~HV;Sg`!%6jQPnFKJ4IFBNLMmLirz{@J45E+O3%1bvqfkWpNB)CjxFZb z9HbaztSw0Ye|h6<+O~q(BV#TO%aDPO6rDG-=XR_O8=&8Sk{a!WVF{bSym4)Ts$Aau zs}P4w18~@j+eWd8x4sEO0#HM)bu^jRIO7 z?80SjC?iUlv|F_@9@TsdTA9snocn{dup~6s+}9?hZ;VeZ!4iGcx@64jd3#@$AE);I z&cE~Sem&hH*Oifu&Kzdv>0F6TfCMK!Z4=&xDeZOIifM6~J=Zh3y*A9t6+>m&a)#EI zd9>Q;E}@0Ihhwq|2MN)hp^}=m`-n&6z-Zr&Oqnqje284)>746vz+1AGeu zbq*i~t}B3LiF69!oiFnEZKfx7aHu34B12e*fjzhcvaqk%wq2b+Rg(w!>_#Jw=9s(+Iy!q+HQ?PS9;(xjjb;|NlW)A6u4_v8ws= z&foJ$b<)-@{3_~1Yw~THH2B~Mm@%v-PT^s4Jx^Hw^!h2WmTABfZah g5>xwXn_vc^9`(R*));ooG~O3R-aISP<#j>ne^E|8k^lez delta 13 VcmaEnw#<3Lc}AJdlZ37*001ss1_1y7 diff --git a/data/original/entity.cc b/data/original/entity.cc index c92e33d..f2af96f 100644 --- a/data/original/entity.cc +++ b/data/original/entity.cc @@ -1 +1 @@ -“²cultcache.store.v1‘—¬weksa.entity¬weksa.entity¯weksa.entity.v1¬weksa.entityÙL{"schemaName":"weksa.entity","schemaVersion":"weksa.entity.v1","members":[]}‘¬weksa.entity‘”­entity:keeper¬weksa.entity´2026-11-10T14:17:08ZÄ:“­entity:keeper‘“¥en-US‘ªthe keeperÀ¤Live‘®weksa-original \ No newline at end of file +“²cultcache.store.v1‘—¬weksa.entity¬weksa.entity¯weksa.entity.v1¬weksa.entityÙL{"schemaName":"weksa.entity","schemaVersion":"weksa.entity.v1","members":[]}‘¬weksa.entity™”­entity:keeper¬weksa.entity´2026-11-10T14:17:08ZÄ:“­entity:keeper‘“¥en-US‘ªthe keeperÀ¤Live‘®weksa-original”¯entity:pd-aesop¬weksa.entity´2026-11-10T18:22:10ZÄM“¯entity:pd-aesop‘“¥en-US‘¥AesopÀ¬PublicDomain‘¼Three Hundred Aesop's Fables”¯entity:pd-bacon¬weksa.entity´2026-11-10T18:22:10ZÄY“¯entity:pd-bacon‘“¥en-US‘¥BaconÀ¬PublicDomain‘Ù'The Essays or Counsels, Civil and Moral”°entity:pd-darwin¬weksa.entity´2026-11-10T18:22:10ZÄK“°entity:pd-darwin‘“¥en-US‘¦DarwinÀ¬PublicDomain‘¸On the Origin of Species”´entity:pd-dhammapada¬weksa.entity´2026-11-10T18:22:10ZĆ“´entity:pd-dhammapada‘“¥en-US‘®the DhammapadaÀ¬PublicDomain‘ÙFDhammapada, a Collection of Verses (Sacred Books of the East, vol. 10)”±entity:pd-emerson¬weksa.entity´2026-11-10T18:22:10ZÄI“±entity:pd-emerson‘“¥en-US‘§EmersonÀ¬PublicDomain‘´Essays: First Series”®entity:pd-mill¬weksa.entity´2026-11-10T18:22:10ZÄ9“®entity:pd-mill‘“¥en-US‘¤MillÀ¬PublicDomain‘ªOn Liberty”¯entity:pd-plato¬weksa.entity´2026-11-10T18:22:10ZÄ=“¯entity:pd-plato‘“¥en-US‘¥PlatoÀ¬PublicDomain‘¬The Republic”±entity:pd-thoreau¬weksa.entity´2026-11-10T18:22:10ZÄc“±entity:pd-thoreau‘“¥en-US‘§ThoreauÀ¬PublicDomain‘Ù-Walden, and On The Duty Of Civil Disobedience \ No newline at end of file -- 2.47.3