validation-growth: 42 blind-written Validation items and scenes (35 per language) (#75)

crates/weksa/tests/original_corpus.rs

@@ -1,6 +1,6 @@
-//! The original corpus (`data/original`), in en-US and ja-JP: per language, 28
+//! The original corpus (`data/original`), in en-US and ja-JP: per language, 49
 //! passages written for the non-memorised golf-loop target (Train and
-//! Validation, all model-written) and 8 public-domain Test sentences, each with
+//! Validation, all model-written: 28, then 21 more Validation items grown blind) and 8 public-domain Test sentences, each with
 //! its full provenance. Test is only read at a checkpoint: no round or recall
 //! prompt over this store holds it. The ja-JP passages are written in Japanese,
 //! not translated; the ja-JP Test sentences are Aozora Bunko excerpts.
@@ -32,7 +32,7 @@ fn stores() -> (CorpusStore, EntityStore) {
 fn every_original_item_records_its_author_and_the_public_domain_ones_their_provenance() {
     let (corpus, entities) = stores();
     let items = corpus.all().unwrap();
-    assert_eq!(items.len(), 72);
+    assert_eq!(items.len(), 114);
     let (mut model, mut public) = ([0; 2], [0; 2]);
     for item in &items {
         let lang = PANEL.iter().position(|l| *l == item.language).unwrap();
@@ -43,7 +43,8 @@ fn every_original_item_records_its_author_and_the_public_domain_ones_their_prove
                 model[lang] += 1;
                 assert!(
                     item.id
-                        .starts_with(["item:original:", "item:original-ja:"][lang]),
+                        .starts_with(["item:original:", "item:original-ja:"][lang])
+                        || item.id.starts_with(["item:vgrow:", "item:vgrow-ja:"][lang]),
                     "{}",
                     item.id
                 );
@@ -116,7 +117,7 @@ fn every_original_item_records_its_author_and_the_public_domain_ones_their_prove
             item.id
         );
     }
-    assert_eq!((model, public), ([28, 28], [8, 8]));
+    assert_eq!((model, public), ([49, 49], [8, 8]));
 }
 
 #[test]
@@ -140,7 +141,7 @@ fn the_original_items_spread_over_train_and_validation_and_test_is_public_domain
         };
         assert_eq!(
             (of(Split::Train), of(Split::Validation), of(Split::Test)),
-            (14, 14, 8),
+            (14, 35, 8),
             "{lang}"
         );
     }
@@ -165,6 +166,85 @@ fn the_original_items_spread_over_train_and_validation_and_test_is_public_domain
     }
 }
 
+/// The 42 Validation items grown after the 56: written blind, one new group
+/// each, every group bucketed Validation by its name alone, none sharing text
+/// with a Test item in either direction.
+#[test]
+fn grown_items_hold_no_test_text_and_every_group_buckets_validation() {
+    let (corpus, _) = stores();
+    let all = corpus.all().unwrap();
+    let mut grown: Vec<_> = all
+        .iter()
+        .filter(|i| i.id.starts_with("item:vgrow"))
+        .collect();
+    grown.sort_by(|a, b| a.id.cmp(&b.id));
+    assert_eq!(grown.len(), 42);
+    let test: Vec<_> = all
+        .iter()
+        .filter(|i| SplitScheme::V1.live(i) == Some(Split::Test))
+        .collect();
+    assert_eq!(test.len(), 16);
+    let mut groups = std::collections::BTreeSet::new();
+    for item in &grown {
+        match &item.source {
+            Source::ModelWritten {
+                family, generator, ..
+            } => {
+                assert_eq!(*family, ModelFamily::Anthropic, "{}", item.id);
+                assert!(
+                    generator
+                        .as_deref()
+                        .is_some_and(|g| g.contains("claude-sonnet-5-5")),
+                    "{}",
+                    item.id
+                );
+            }
+            other => panic!("{}: {other:?}", item.id),
+        }
+        assert_eq!(item.speaker.as_str(), "entity:keeper", "{}", item.id);
+        assert_eq!(
+            SplitScheme::V1.live(item),
+            Some(Split::Validation),
+            "{}",
+            item.id
+        );
+        assert!(groups.insert(item.group.clone()), "{}", item.group);
+        assert!(
+            all.iter()
+                .filter(|o| o.id != item.id)
+                .all(|o| o.group != item.group),
+            "{} shares its group",
+            item.group
+        );
+        for t in &test {
+            assert!(
+                !leaks(std::slice::from_ref(&item.text), t),
+                "{} holds Test text",
+                item.id
+            );
+            // The reverse: the Test item contains the new text.
+            assert!(
+                !t.text.contains(&item.text),
+                "{} is inside Test text",
+                item.id
+            );
+        }
+    }
+    let lines: Vec<String> = grown
+        .iter()
+        .map(|i| format!("{}\n{}\n{}\n{:?}", i.id, i.text, i.group, i.source))
+        .collect();
+    let parts: Vec<&[u8]> = lines.iter().map(|l| l.as_bytes()).collect();
+    assert_eq!(
+        weksa::split::fnv1a64_of(&parts),
+        DIGEST_OF_THE_42,
+        "a grown item changed"
+    );
+}
+
+/// Taken over the 42 grown items at admission, 2026-11-13.
+const DIGEST_OF_THE_42: u64 = 15237292705431734093;
+
 /// The 28 model-written items are byte-for-byte what they were before Test
 /// was added: id, text, group and source, folded in key order.
 #[test]
@@ -460,10 +540,10 @@ fn the_recall_probe_covers_every_live_original_item_and_names_no_work() {
     };
     assert_eq!(reading.languages.len(), 2);
     for r in &reading.languages {
-        assert_eq!((r.n, r.answered), (28, 28), "{}", r.language);
+        assert_eq!((r.n, r.answered), (49, 49), "{}", r.language);
         assert!(r.chrf < 25.0, "{} {}", r.language, r.chrf);
     }
-    assert_eq!(model.0.len(), 56);
+    assert_eq!(model.0.len(), 98);
     // No item outside Train and Validation is shown to the model.
     let all = CorpusStore::open(&c).unwrap().all().unwrap();
     for item in &all {
@@ -544,14 +624,14 @@ fn original_round(family: ModelFamily) -> (weksa::run::RunStores, weksa::run::Ro
 fn own_family_counts_and_labels() {
     // The original corpus is Anthropic-written items, all offered.
     let (run, r) = original_round(ModelFamily::Anthropic);
-    assert_eq!(r.own_family(), 56);
+    assert_eq!(r.own_family(), 98);
     assert_eq!(
         r.parse_reasons().iter().map(|c| c.count).sum::<u32>(),
-        56,
+        98,
         "every item was refused at Parse and says why"
     );
     let line = run.round_line(&r);
-    assert!(line.contains(" own-family 56"), "{line}");
+    assert!(line.contains(" own-family 98"), "{line}");
 
     // A different answerer family, or none recorded, labels nothing.
     for family in [ModelFamily::OpenAi, ModelFamily::Unrecorded] {
@@ -664,6 +744,20 @@ impl weksa::model::ModelPort for Record {
     }
 }
 
+/// A Test item's text, or any run of five of its words (twelve characters of a
+/// text written without spaces), in a prompt.
+fn leaks(prompts: &[String], item: &weksa::corpus::CorpusItem) -> bool {
+    let w: Vec<&str> = item.text.split_whitespace().collect();
+    let c: Vec<char> = item.text.chars().collect();
+    prompts.iter().any(|t| {
+        t.contains(&item.text)
+            || w.windows(5).any(|k| t.contains(&k.join(" ")))
+            || (w.len() == 1
+                && c.windows(12)
+                    .any(|k| t.contains(&k.iter().collect::<String>())))
+    })
+}
+
 /// The public-domain Test items are held out: no prompt of round 1 over
 /// `data/original`, no recall prompt, and no round record names one.
 #[test]
@@ -717,9 +811,9 @@ fn no_test_item_is_offered_to_a_round_or_recall_or_recorded() {
     let RoundOutcome::Recorded(r) = round(&mut run, &mut model, &scorer).unwrap() else {
         panic!("answered");
     };
-    // The check bites: the round did show live items, 56 of them, and the
+    // The check bites: the round did show live items, 98 of them, and the
     // probe sees a shown Train item of each language.
-    assert!(model.0.len() >= 56, "{}", model.0.len());
+    assert!(model.0.len() >= 98, "{}", model.0.len());
     for lang in PANEL {
         assert!(
             items
@@ -740,7 +834,7 @@ fn no_test_item_is_offered_to_a_round_or_recall_or_recorded() {
         panic!("answered");
     };
     for rr in &reading.languages {
-        assert_eq!((rr.n, rr.answered), (28, 28), "{}", rr.language);
+        assert_eq!((rr.n, rr.answered), (49, 49), "{}", rr.language);
     }
     for item in &held {
         assert!(!leaks(&recall.0, item), "recall prompt held {}", item.id);
@@ -783,7 +877,7 @@ fn ja_test_text_carries_no_aozora_markup() {
         .into_iter()
         .filter(|i| i.language == "ja-JP")
         .collect();
-    assert_eq!(ja.len(), 36);
+    assert_eq!(ja.len(), 57);
     for item in &ja {
         for bad in ["《", "》", "｜", "［＃", "※"] {
             assert!(!item.text.contains(bad), "{} holds {bad}", item.id);
@@ -810,7 +904,7 @@ fn every_cited_speaker_has_a_form_in_its_items_language() {
 }
 
 /// The panel [en-US, ja-JP] over `data/original`: both languages are read, each
-/// with 14 Validation items, and no Test item is in any call.
+/// with 35 Validation items, and no Test item is in any call.
 #[test]
 fn a_panel_round_reads_both_languages() {
     let (run, r) = original_round(ModelFamily::Unrecorded);
@@ -821,7 +915,7 @@ fn a_panel_round_reads_both_languages() {
         let refused = &l.validation_refused;
         assert_eq!(
             refused.parse + refused.admit + refused.lower,
-            14,
+            35,
             "{}",
             l.language
         );
@@ -941,24 +1035,46 @@ fn committed_scenes_cover_original_groups() {
         .filter(|i| matches!(SplitScheme::V1.live(i), Some(Split::Train | Split::Validation)))
         .map(|i| i.group)
         .collect();
-    assert_eq!(live.len(), 56);
+    assert_eq!(live.len(), 98);
     let held: BTreeSet<String> = scenes.iter().map(|s| s.group.clone()).collect();
     assert_eq!(held, live, "a scene for each live Train or Validation group, none other");
     // Revisions append: a group's later revisions re-derived its relation.
     assert!(scenes.iter().all(|s| s.n <= 2));
+    const S7: &str = "claude-sonnet-5-5 (Weksa Hands, model-annotated original scenes)";
+    const GROWTH: &str = "claude-sonnet-5-5 (Weksa Hands, model-annotated validation-growth scenes; annotator is not the writer)";
+    let grown: BTreeSet<String> = CorpusStore::open(&data.join("corpus.cc"))
+        .unwrap()
+        .all()
+        .unwrap()
+        .into_iter()
+        .filter(|i| i.id.starts_with("item:vgrow"))
+        .map(|i| i.group)
+        .collect();
+    assert_eq!(grown.len(), 42);
+    let mut by_growth = BTreeSet::new();
     for s in &scenes {
         match &s.annotator {
             Annotator::Model { family, generator } => {
                 assert_eq!(*family, ModelFamily::Anthropic, "{}", s.group);
-                assert_eq!(
-                    generator, "claude-sonnet-5-5 (Weksa Hands, model-annotated original scenes)",
-                    "{}",
-                    s.group
-                );
+                assert!(generator == S7 || generator == GROWTH, "{}", s.group);
+                if generator == GROWTH {
+                    by_growth.insert(s.group.clone());
+                    // One relation, keeper to unnamed, the three Scene features.
+                    assert_eq!(s.n, 1, "{}", s.group);
+                    assert_eq!(s.relations.len(), 1, "{}", s.group);
+                    let rel = &s.relations[0];
+                    assert_eq!(rel.from.as_str(), "entity:keeper", "{}", s.group);
+                    assert_eq!(rel.to, weksa::speaker::Party::Unnamed, "{}", s.group);
+                    assert_eq!(rel.features.len(), 3, "{}", s.group);
+                }
             }
             other => panic!("{} is {other:?}", s.group),
         }
     }
+    assert_eq!(
+        by_growth, grown,
+        "the growth scenes are exactly the vgrow groups"
+    );
 }
 
 /// The sweep both seed-pair tests share: over a grid of grades per feature (each

docs/targets/interlingua-lowering-map.md

@@ -406,37 +406,44 @@ of 1120 trials, NoGain 71%, Regressed 0%: a net gain of 0.079 against a mean
 noise term of 0.079. At 14 items per language the rule is therefore a low-power
 filter for a gain of this size; by 1/sqrt(n), about 35 or more items per
 language would bring the noise term near 0.05 (an extrapolation, not measured).
-### Validation growth plan (`validation-growth`, not built)
+### Validation growth (`validation-growth`, built; recalibration not run)
 
-Power needs about 35 or more Validation items per language (14 now). The
-corpus is `data/original`: model-written original prose for Train and
-Validation (written to be non-memorised; public domain text was rejected for
-them), eight public-domain sentences per language for Test with full
-provenance. The split is derived from an item's group alone
-(`SplitScheme::V1`: FNV-1a of `weksa-split-v1:<group>` mod 100, under 40 Train,
-under 80 Validation, else Test; a model-written item in a Test group is in no
-split), and every item so far is its own group (`original-<noun>`). Growth is
-therefore: about 21 new model-written items per language, each its own new
-group, the group name chosen before the text is written (the bucket is then a
-property of the name, not of the text), keeping names whose bucket is
-Validation, with source `ModelWritten` (family, generator, prompt digest),
-a speaker and a Scene as the existing items have, and the inventory and
-ontology beds unchanged for the measure. No Test text is read by the writer:
-the writer's prompt holds a noun list, the register note and the language,
-never a Test sentence or `weksa corpus dump` output.
-Costs and hazards, all real: (a) the corpus digest changes, so a run record
-pins a new corpus and every calibration number above is redone on it (eight
-prompted draws of 35 items per language, about 2.5 times the 14-item cost);
-(b) the inventory was drafted with Validation informing it (ja inventory rev
-5), so fresh Validation items also measure inventory coverage, and some
-will refuse at encode (counted 0) until concepts are added, which is a result,
-not a defect; (c) the writer and the lowerers are one model family, so a
-lowerer may be favoured by its own prose (self-preference, Panickssery et al.
-2024); a cross-family writer for the new Validation items removes that
-(question `validation-growth-writer`).
-Not built here because the writer fork is open, the scenes and entities for
-~42 items are a real authoring cut, and the digest change invalidates the
-runs in flight.
+Power needs about 35 or more Validation items per language. `data/original`
+now holds 35 per language (it held 14; Train 14 and Test 8 are unchanged):
+42 model-written items, `item:vgrow:1..21` (en-US) and `item:vgrow-ja:1..21`
+(ja-JP), live Train+Validation 56 to 98. The slug is new on purpose: the
+`item:original:` prefix filter of `DIGEST_OF_THE_28` still selects the first 28
+and passes; `DIGEST_OF_THE_42` freezes the new prose.
+
+The split is derived from an item's group alone (`SplitScheme::V1`: FNV-1a of
+`weksa-split-v1:<group>` mod 100, under 40 Train, under 80 Validation, else
+Test). The names were chosen before any text existed: 90 candidate nouns per
+language, dropped by script when the group already existed or the noun
+occurred in a Test text, then the first 24 whose bucket fell in 40..=79 (the
+bucket came from `SplitScheme::V1` itself, cross-checked on the 72 existing
+items). Each language was written by three blind writer agents (8 nouns each,
+a brief of language, register note, noun and speech act, no Test text and no
+existing item), 24 texts per language written, the first 21 in name order
+admitted through `weksa corpus add`; each group is its own group. The scenes
+are `weksa scene put` documents (keeper to `unnamed`, inventory rev 4),
+annotated by a separate agent; a second annotator measured agreement only and
+is not committed.
+
+The writer is Claude (family Anthropic recorded): no cross-family generator was
+reachable from the body that ran this cut. The self-preference effect (a Claude
+lowerer favouring Claude prose, Panickssery et al. 2024) is therefore not
+measured. The writers' blindness is by brief content, not sandbox.
+
+Expect encode refusals: the existing 56 were written onto the Dao r3 concepts
+and the inventory was drafted from them, while the new items were written free
+of the inventory (question `validation-growth-vocabulary` stays open). Some new
+packets will be refused at encode (inventory coverage) and score 0 as the means
+do; they stay in the corpus. That is a result about inventory generalization,
+not a defect.
+
+Every calibration number above (92/0/8 prompted rates, the 29% power, the 0.079
+noise term) is for the 14-item corpus; the corpus digest changed, and the
+numbers are redone on the grown corpus in a separate recalibration step.
 
 ### Smallest measurable distil slice (`distil-incontext`, not built)