diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 99e7e30..a3eae63 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -1,6 +1,6 @@ -//! The original corpus (`data/original`), in en-US and ja-JP: per language, 28 +//! The original corpus (`data/original`), in en-US and ja-JP: per language, 49 //! passages written for the non-memorised golf-loop target (Train and -//! Validation, all model-written) and 8 public-domain Test sentences, each with +//! Validation, all model-written: 28, then 21 more Validation items grown blind) and 8 public-domain Test sentences, each with //! its full provenance. Test is only read at a checkpoint: no round or recall //! prompt over this store holds it. The ja-JP passages are written in Japanese, //! not translated; the ja-JP Test sentences are Aozora Bunko excerpts. @@ -32,7 +32,7 @@ fn stores() -> (CorpusStore, EntityStore) { fn every_original_item_records_its_author_and_the_public_domain_ones_their_provenance() { let (corpus, entities) = stores(); let items = corpus.all().unwrap(); - assert_eq!(items.len(), 72); + assert_eq!(items.len(), 114); let (mut model, mut public) = ([0; 2], [0; 2]); for item in &items { let lang = PANEL.iter().position(|l| *l == item.language).unwrap(); @@ -43,7 +43,8 @@ fn every_original_item_records_its_author_and_the_public_domain_ones_their_prove model[lang] += 1; assert!( item.id - .starts_with(["item:original:", "item:original-ja:"][lang]), + .starts_with(["item:original:", "item:original-ja:"][lang]) + || item.id.starts_with(["item:vgrow:", "item:vgrow-ja:"][lang]), "{}", item.id ); @@ -116,7 +117,7 @@ fn every_original_item_records_its_author_and_the_public_domain_ones_their_prove item.id ); } - assert_eq!((model, public), ([28, 28], [8, 8])); + assert_eq!((model, public), ([49, 49], [8, 8])); } #[test] @@ -140,7 +141,7 @@ fn the_original_items_spread_over_train_and_validation_and_test_is_public_domain }; assert_eq!( (of(Split::Train), of(Split::Validation), of(Split::Test)), - (14, 14, 8), + (14, 35, 8), "{lang}" ); } @@ -165,6 +166,85 @@ fn the_original_items_spread_over_train_and_validation_and_test_is_public_domain } } +/// The 42 Validation items grown after the 56: written blind, one new group +/// each, every group bucketed Validation by its name alone, none sharing text +/// with a Test item in either direction. +#[test] +fn grown_items_hold_no_test_text_and_every_group_buckets_validation() { + let (corpus, _) = stores(); + let all = corpus.all().unwrap(); + let mut grown: Vec<_> = all + .iter() + .filter(|i| i.id.starts_with("item:vgrow")) + .collect(); + grown.sort_by(|a, b| a.id.cmp(&b.id)); + assert_eq!(grown.len(), 42); + let test: Vec<_> = all + .iter() + .filter(|i| SplitScheme::V1.live(i) == Some(Split::Test)) + .collect(); + assert_eq!(test.len(), 16); + let mut groups = std::collections::BTreeSet::new(); + for item in &grown { + match &item.source { + Source::ModelWritten { + family, generator, .. + } => { + assert_eq!(*family, ModelFamily::Anthropic, "{}", item.id); + assert!( + generator + .as_deref() + .is_some_and(|g| g.contains("claude-sonnet-5-5")), + "{}", + item.id + ); + } + other => panic!("{}: {other:?}", item.id), + } + assert_eq!(item.speaker.as_str(), "entity:keeper", "{}", item.id); + assert_eq!( + SplitScheme::V1.live(item), + Some(Split::Validation), + "{}", + item.id + ); + assert!(groups.insert(item.group.clone()), "{}", item.group); + assert!( + all.iter() + .filter(|o| o.id != item.id) + .all(|o| o.group != item.group), + "{} shares its group", + item.group + ); + for t in &test { + assert!( + !leaks(std::slice::from_ref(&item.text), t), + "{} holds Test text", + item.id + ); + // The reverse: the Test item contains the new text. + assert!( + !t.text.contains(&item.text), + "{} is inside Test text", + item.id + ); + } + } + let lines: Vec = grown + .iter() + .map(|i| format!("{}\n{}\n{}\n{:?}", i.id, i.text, i.group, i.source)) + .collect(); + let parts: Vec<&[u8]> = lines.iter().map(|l| l.as_bytes()).collect(); + assert_eq!( + weksa::split::fnv1a64_of(&parts), + DIGEST_OF_THE_42, + "a grown item changed" + ); +} + +/// Taken over the 42 grown items at admission, 2026-11-13. +const DIGEST_OF_THE_42: u64 = 15237292705431734093; + /// The 28 model-written items are byte-for-byte what they were before Test /// was added: id, text, group and source, folded in key order. #[test] @@ -460,10 +540,10 @@ fn the_recall_probe_covers_every_live_original_item_and_names_no_work() { }; assert_eq!(reading.languages.len(), 2); for r in &reading.languages { - assert_eq!((r.n, r.answered), (28, 28), "{}", r.language); + assert_eq!((r.n, r.answered), (49, 49), "{}", r.language); assert!(r.chrf < 25.0, "{} {}", r.language, r.chrf); } - assert_eq!(model.0.len(), 56); + assert_eq!(model.0.len(), 98); // No item outside Train and Validation is shown to the model. let all = CorpusStore::open(&c).unwrap().all().unwrap(); for item in &all { @@ -544,14 +624,14 @@ fn original_round(family: ModelFamily) -> (weksa::run::RunStores, weksa::run::Ro fn own_family_counts_and_labels() { // The original corpus is Anthropic-written items, all offered. let (run, r) = original_round(ModelFamily::Anthropic); - assert_eq!(r.own_family(), 56); + assert_eq!(r.own_family(), 98); assert_eq!( r.parse_reasons().iter().map(|c| c.count).sum::(), - 56, + 98, "every item was refused at Parse and says why" ); let line = run.round_line(&r); - assert!(line.contains(" own-family 56"), "{line}"); + assert!(line.contains(" own-family 98"), "{line}"); // A different answerer family, or none recorded, labels nothing. for family in [ModelFamily::OpenAi, ModelFamily::Unrecorded] { @@ -664,6 +744,20 @@ impl weksa::model::ModelPort for Record { } } +/// A Test item's text, or any run of five of its words (twelve characters of a +/// text written without spaces), in a prompt. +fn leaks(prompts: &[String], item: &weksa::corpus::CorpusItem) -> bool { + let w: Vec<&str> = item.text.split_whitespace().collect(); + let c: Vec = item.text.chars().collect(); + prompts.iter().any(|t| { + t.contains(&item.text) + || w.windows(5).any(|k| t.contains(&k.join(" "))) + || (w.len() == 1 + && c.windows(12) + .any(|k| t.contains(&k.iter().collect::()))) + }) +} + /// The public-domain Test items are held out: no prompt of round 1 over /// `data/original`, no recall prompt, and no round record names one. #[test] @@ -717,9 +811,9 @@ fn no_test_item_is_offered_to_a_round_or_recall_or_recorded() { let RoundOutcome::Recorded(r) = round(&mut run, &mut model, &scorer).unwrap() else { panic!("answered"); }; - // The check bites: the round did show live items, 56 of them, and the + // The check bites: the round did show live items, 98 of them, and the // probe sees a shown Train item of each language. - assert!(model.0.len() >= 56, "{}", model.0.len()); + assert!(model.0.len() >= 98, "{}", model.0.len()); for lang in PANEL { assert!( items @@ -740,7 +834,7 @@ fn no_test_item_is_offered_to_a_round_or_recall_or_recorded() { panic!("answered"); }; for rr in &reading.languages { - assert_eq!((rr.n, rr.answered), (28, 28), "{}", rr.language); + assert_eq!((rr.n, rr.answered), (49, 49), "{}", rr.language); } for item in &held { assert!(!leaks(&recall.0, item), "recall prompt held {}", item.id); @@ -783,7 +877,7 @@ fn ja_test_text_carries_no_aozora_markup() { .into_iter() .filter(|i| i.language == "ja-JP") .collect(); - assert_eq!(ja.len(), 36); + assert_eq!(ja.len(), 57); for item in &ja { for bad in ["《", "》", "｜", "［＃", "※"] { assert!(!item.text.contains(bad), "{} holds {bad}", item.id); @@ -810,7 +904,7 @@ fn every_cited_speaker_has_a_form_in_its_items_language() { } /// The panel [en-US, ja-JP] over `data/original`: both languages are read, each -/// with 14 Validation items, and no Test item is in any call. +/// with 35 Validation items, and no Test item is in any call. #[test] fn a_panel_round_reads_both_languages() { let (run, r) = original_round(ModelFamily::Unrecorded); @@ -821,7 +915,7 @@ fn a_panel_round_reads_both_languages() { let refused = &l.validation_refused; assert_eq!( refused.parse + refused.admit + refused.lower, - 14, + 35, "{}", l.language ); @@ -941,24 +1035,46 @@ fn committed_scenes_cover_original_groups() { .filter(|i| matches!(SplitScheme::V1.live(i), Some(Split::Train | Split::Validation))) .map(|i| i.group) .collect(); - assert_eq!(live.len(), 56); + assert_eq!(live.len(), 98); let held: BTreeSet = scenes.iter().map(|s| s.group.clone()).collect(); assert_eq!(held, live, "a scene for each live Train or Validation group, none other"); // Revisions append: a group's later revisions re-derived its relation. assert!(scenes.iter().all(|s| s.n <= 2)); + const S7: &str = "claude-sonnet-5-5 (Weksa Hands, model-annotated original scenes)"; + const GROWTH: &str = "claude-sonnet-5-5 (Weksa Hands, model-annotated validation-growth scenes; annotator is not the writer)"; + let grown: BTreeSet = CorpusStore::open(&data.join("corpus.cc")) + .unwrap() + .all() + .unwrap() + .into_iter() + .filter(|i| i.id.starts_with("item:vgrow")) + .map(|i| i.group) + .collect(); + assert_eq!(grown.len(), 42); + let mut by_growth = BTreeSet::new(); for s in &scenes { match &s.annotator { Annotator::Model { family, generator } => { assert_eq!(*family, ModelFamily::Anthropic, "{}", s.group); - assert_eq!( - generator, "claude-sonnet-5-5 (Weksa Hands, model-annotated original scenes)", - "{}", - s.group - ); + assert!(generator == S7 || generator == GROWTH, "{}", s.group); + if generator == GROWTH { + by_growth.insert(s.group.clone()); + // One relation, keeper to unnamed, the three Scene features. + assert_eq!(s.n, 1, "{}", s.group); + assert_eq!(s.relations.len(), 1, "{}", s.group); + let rel = &s.relations[0]; + assert_eq!(rel.from.as_str(), "entity:keeper", "{}", s.group); + assert_eq!(rel.to, weksa::speaker::Party::Unnamed, "{}", s.group); + assert_eq!(rel.features.len(), 3, "{}", s.group); + } } other => panic!("{} is {other:?}", s.group), } } + assert_eq!( + by_growth, grown, + "the growth scenes are exactly the vgrow groups" + ); } /// The sweep both seed-pair tests share: over a grid of grades per feature (each diff --git a/data/original/corpus.cc b/data/original/corpus.cc index 9c53141..6e8b471 100644 Binary files a/data/original/corpus.cc and b/data/original/corpus.cc differ diff --git a/data/original/speaker-state.cc b/data/original/speaker-state.cc index c4f563d..2503674 100644 Binary files a/data/original/speaker-state.cc and b/data/original/speaker-state.cc differ diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index d226184..0182cc0 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -406,37 +406,44 @@ of 1120 trials, NoGain 71%, Regressed 0%: a net gain of 0.079 against a mean noise term of 0.079. At 14 items per language the rule is therefore a low-power filter for a gain of this size; by 1/sqrt(n), about 35 or more items per language would bring the noise term near 0.05 (an extrapolation, not measured). -### Validation growth plan (`validation-growth`, not built) +### Validation growth (`validation-growth`, built; recalibration not run) -Power needs about 35 or more Validation items per language (14 now). The -corpus is `data/original`: model-written original prose for Train and -Validation (written to be non-memorised; public domain text was rejected for -them), eight public-domain sentences per language for Test with full -provenance. The split is derived from an item's group alone -(`SplitScheme::V1`: FNV-1a of `weksa-split-v1:` mod 100, under 40 Train, -under 80 Validation, else Test; a model-written item in a Test group is in no -split), and every item so far is its own group (`original-`). Growth is -therefore: about 21 new model-written items per language, each its own new -group, the group name chosen before the text is written (the bucket is then a -property of the name, not of the text), keeping names whose bucket is -Validation, with source `ModelWritten` (family, generator, prompt digest), -a speaker and a Scene as the existing items have, and the inventory and -ontology beds unchanged for the measure. No Test text is read by the writer: -the writer's prompt holds a noun list, the register note and the language, -never a Test sentence or `weksa corpus dump` output. -Costs and hazards, all real: (a) the corpus digest changes, so a run record -pins a new corpus and every calibration number above is redone on it (eight -prompted draws of 35 items per language, about 2.5 times the 14-item cost); -(b) the inventory was drafted with Validation informing it (ja inventory rev -5), so fresh Validation items also measure inventory coverage, and some -will refuse at encode (counted 0) until concepts are added, which is a result, -not a defect; (c) the writer and the lowerers are one model family, so a -lowerer may be favoured by its own prose (self-preference, Panickssery et al. -2024); a cross-family writer for the new Validation items removes that -(question `validation-growth-writer`). -Not built here because the writer fork is open, the scenes and entities for -~42 items are a real authoring cut, and the digest change invalidates the -runs in flight. +Power needs about 35 or more Validation items per language. `data/original` +now holds 35 per language (it held 14; Train 14 and Test 8 are unchanged): +42 model-written items, `item:vgrow:1..21` (en-US) and `item:vgrow-ja:1..21` +(ja-JP), live Train+Validation 56 to 98. The slug is new on purpose: the +`item:original:` prefix filter of `DIGEST_OF_THE_28` still selects the first 28 +and passes; `DIGEST_OF_THE_42` freezes the new prose. + +The split is derived from an item's group alone (`SplitScheme::V1`: FNV-1a of +`weksa-split-v1:` mod 100, under 40 Train, under 80 Validation, else +Test). The names were chosen before any text existed: 90 candidate nouns per +language, dropped by script when the group already existed or the noun +occurred in a Test text, then the first 24 whose bucket fell in 40..=79 (the +bucket came from `SplitScheme::V1` itself, cross-checked on the 72 existing +items). Each language was written by three blind writer agents (8 nouns each, +a brief of language, register note, noun and speech act, no Test text and no +existing item), 24 texts per language written, the first 21 in name order +admitted through `weksa corpus add`; each group is its own group. The scenes +are `weksa scene put` documents (keeper to `unnamed`, inventory rev 4), +annotated by a separate agent; a second annotator measured agreement only and +is not committed. + +The writer is Claude (family Anthropic recorded): no cross-family generator was +reachable from the body that ran this cut. The self-preference effect (a Claude +lowerer favouring Claude prose, Panickssery et al. 2024) is therefore not +measured. The writers' blindness is by brief content, not sandbox. + +Expect encode refusals: the existing 56 were written onto the Dao r3 concepts +and the inventory was drafted from them, while the new items were written free +of the inventory (question `validation-growth-vocabulary` stays open). Some new +packets will be refused at encode (inventory coverage) and score 0 as the means +do; they stay in the corpus. That is a result about inventory generalization, +not a defect. + +Every calibration number above (92/0/8 prompted rates, the 29% power, the 0.079 +noise term) is for the 14-item corpus; the corpus digest changed, and the +numbers are redone on the grown corpus in a separate recalibration step. ### Smallest measurable distil slice (`distil-incontext`, not built)