diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 91b635b..79273f5 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -2,6 +2,8 @@ //! non-memorised golf-loop target. It holds no public-domain text, so a model //! cannot have it in pretraining, and every item says who wrote it. +mod common; + use std::path::Path; use weksa::corpus::{ModelFamily, Source}; @@ -25,7 +27,7 @@ fn stores() -> (CorpusStore, EntityStore) { fn every_original_item_records_its_author_and_none_is_public_domain() { let (corpus, entities) = stores(); let items = corpus.all().unwrap(); - assert_eq!(items.len(), 6); + assert_eq!(items.len(), 28); for item in &items { match &item.source { Source::ModelWritten { @@ -37,12 +39,18 @@ fn every_original_item_records_its_author_and_none_is_public_domain() { } other => panic!("{}: not model-written: {other:?}", item.id), } - assert!(entities.get(&item.speaker).unwrap().is_some(), "{}", item.id); + assert!( + entities.get(&item.speaker).unwrap().is_some(), + "{}", + item.id + ); } } #[test] fn the_original_items_spread_over_train_and_validation_and_none_is_in_test() { + // Model-written items cannot be in Test (`SplitScheme::split` excludes a + // model-written item of a test group), so this store holds none. let (corpus, _) = stores(); let splits: Vec> = corpus .all() @@ -51,10 +59,57 @@ fn the_original_items_spread_over_train_and_validation_and_none_is_in_test() { .map(|i| SplitScheme::V1.live(i)) .collect(); let count = |s| splits.iter().filter(|x| **x == Some(s)).count(); - assert_eq!((count(Split::Train), count(Split::Validation)), (3, 3)); + assert_eq!((count(Split::Train), count(Split::Validation)), (14, 14)); assert_eq!(count(Split::Test), 0); } +/// The first six items are the original golf-loop set; widening the store +/// adds items after them and never edits them. +#[test] +fn the_first_six_original_items_are_unchanged() { + let (corpus, _) = stores(); + let want = [ + ( + "item:original:1", + "A quiet mind is like still water in a deep bowl.", + "original-bowl", + ), + ( + "item:original:2", + "The river weakens the bank in spring and strengthens the roots in autumn.", + "original-river", + ), + ( + "item:original:3", + "The sage waits at the door until the water has settled.", + "original-door", + ), + ( + "item:original:4", + "The old gardener empties the jar and fills it again with rainwater.", + "original-jar", + ), + ( + "item:original:5", + "Under the same sky the earth keeps its patience, and the baker keeps hers.", + "original-sky", + ), + ( + "item:original:6", + "A hungry belly argues louder than any will.", + "original-belly", + ), + ]; + for (id, text, group) in want { + let item = corpus.get(id).unwrap().unwrap(); + assert_eq!( + (item.text.as_str(), item.group.as_str()), + (text, group), + "{id}" + ); + } +} + #[test] fn the_original_inventory_is_the_dao_revision_three() { let dao = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/daodejing/inventory.cc"); @@ -69,7 +124,7 @@ fn the_original_inventory_is_the_dao_revision_three() { /// The probe over the real original corpus: every item is offered, none names a /// work, and a model that answers with unrelated words reads at the noise floor. #[test] -fn the_recall_probe_covers_all_six_original_items_and_names_no_work() { +fn the_recall_probe_covers_every_live_original_item_and_names_no_work() { use weksa::model::{ModelPort, PortError}; use weksa::run::{RecallOutcome, RunPaths, RunStores}; struct Unrelated(Vec); @@ -107,9 +162,24 @@ fn the_recall_probe_covers_all_six_original_items_and_names_no_work() { let RecallOutcome::Read(r) = run.recall(&mut model).unwrap() else { panic!("answered"); }; - assert_eq!((r.n, r.answered), (6, 6)); + assert_eq!((r.n, r.answered), (28, 28)); assert!(r.chrf < 25.0, "{}", r.chrf); - assert_eq!(model.0.len(), 6); + assert_eq!(model.0.len(), 28); + // No item outside Train and Validation is shown to the model. + let all = CorpusStore::open(&c).unwrap().all().unwrap(); + for item in &all { + let live = SplitScheme::V1.live(item); + // A recall prompt shows the first half of the words. + let words: Vec<&str> = item.text.split_whitespace().collect(); + let opening = words[..words.len() / 2].join(" "); + let shown = model.0.iter().any(|t| t.contains(&opening)); + assert_eq!( + shown, + matches!(live, Some(Split::Train | Split::Validation)), + "{}", + item.id + ); + } assert!(model.0.iter().all(|t| t.contains("work: not published"))); assert!(model.0.iter().all(|t| !t.contains("translator:"))); } @@ -168,16 +238,16 @@ fn original_round(family: ModelFamily) -> (weksa::run::RunStores, weksa::run::Ro #[test] fn own_family_counts_and_labels() { - // The original corpus is six Anthropic-written items, all offered. + // The original corpus is Anthropic-written items, all offered. let (run, r) = original_round(ModelFamily::Anthropic); - assert_eq!(r.own_family(), 6); + assert_eq!(r.own_family(), 28); assert_eq!( r.parse_reasons().iter().map(|c| c.count).sum::(), - 6, + 28, "every item was refused at Parse and says why" ); let line = run.round_line(&r); - assert!(line.contains(" own-family 6"), "{line}"); + assert!(line.contains(" own-family 28"), "{line}"); // A different answerer family, or none recorded, labels nothing. for family in [ModelFamily::OpenAi, ModelFamily::Unrecorded] { @@ -186,3 +256,91 @@ fn own_family_counts_and_labels() { assert!(!run.round_line(&r).contains("own-family")); } } + +/// Over a fixture store whose items are not all live: recall probes the live +/// Train and Validation items and no others (a withdrawn item, a model-written +/// item of a test group, a public-domain Test item). +#[test] +fn recall_probes_only_live_train_and_validation_items() { + use common::{cast, entities_in, group_in_bucket, item_in, run_over}; + use weksa::model::{ModelPort, PortError}; + use weksa::run::RecallOutcome; + struct Record(Vec); + impl ModelPort for Record { + fn model(&self) -> &str { + "record" + } + fn answer(&mut self, p: &weksa::context::Prompt) -> Result { + self.0.push(p.text().to_string()); + Ok(r#"{"text":"nothing"}"#.to_string()) + } + } + let dir = tempfile::tempdir().unwrap(); + let entities = entities_in(dir.path()); + let mut store = CorpusStore::open(&dir.path().join("c.cc")).unwrap(); + let public = Source::PublicDomain { + work: "A Work".into(), + author: "An Author".into(), + edition_url: "https://example.org/e".into(), + licence: "CC0".into(), + jurisdiction: "US".into(), + translator: "A Translator".into(), + edition_year: 1900, + }; + let rows = [ + ( + "item:f:1", + "trainmark opens here and goes on", + 10, + cast(), + false, + ), + ( + "item:f:2", + "validmark opens here and goes on", + 50, + cast(), + false, + ), + ( + "item:f:3", + "gonemark opens here and goes on", + 10, + cast(), + true, + ), + ( + "item:f:4", + "modelmark opens here and goes on", + 90, + common::model(), + false, + ), + ( + "item:f:5", + "publicmark opens here and goes on", + 90, + public, + false, + ), + ]; + for (id, text, bucket, source, withdrawn) in rows { + let item = item_in(id, text, &group_in_bucket(bucket), source); + store.add(item, &entities).unwrap(); + if withdrawn { + store.withdraw(id).unwrap(); + } + } + drop(store); + let run = run_over(dir.path()); + let mut m = Record(vec![]); + let RecallOutcome::Read(r) = run.recall(&mut m).unwrap() else { + panic!("answered"); + }; + assert_eq!(r.n, 2); + let shown = |mark: &str| m.0.iter().any(|t| t.contains(mark)); + assert!(shown("trainmark") && shown("validmark")); + for hidden in ["gonemark", "modelmark", "publicmark"] { + assert!(!shown(hidden), "recall showed {hidden}"); + } +} diff --git a/data/original/corpus.cc b/data/original/corpus.cc index c11443f..57988df 100644 Binary files a/data/original/corpus.cc and b/data/original/corpus.cc differ