crates/weksa/tests/original_corpus.rs

@@ -2,6 +2,8 @@
 //! non-memorised golf-loop target. It holds no public-domain text, so a model
 //! cannot have it in pretraining, and every item says who wrote it.
 
+mod common;
+
 use std::path::Path;
 
 use weksa::corpus::{ModelFamily, Source};
@@ -25,7 +27,7 @@ fn stores() -> (CorpusStore, EntityStore) {
 fn every_original_item_records_its_author_and_none_is_public_domain() {
     let (corpus, entities) = stores();
     let items = corpus.all().unwrap();
-    assert_eq!(items.len(), 6);
+    assert_eq!(items.len(), 28);
     for item in &items {
         match &item.source {
             Source::ModelWritten {
@@ -37,12 +39,18 @@ fn every_original_item_records_its_author_and_none_is_public_domain() {
             }
             other => panic!("{}: not model-written: {other:?}", item.id),
         }
-        assert!(entities.get(&item.speaker).unwrap().is_some(), "{}", item.id);
+        assert!(
+            entities.get(&item.speaker).unwrap().is_some(),
+            "{}",
+            item.id
+        );
     }
 }
 
 #[test]
 fn the_original_items_spread_over_train_and_validation_and_none_is_in_test() {
+    // Model-written items cannot be in Test (`SplitScheme::split` excludes a
+    // model-written item of a test group), so this store holds none.
     let (corpus, _) = stores();
     let splits: Vec<Option<Split>> = corpus
         .all()
@@ -51,10 +59,57 @@ fn the_original_items_spread_over_train_and_validation_and_none_is_in_test() {
         .map(|i| SplitScheme::V1.live(i))
         .collect();
     let count = |s| splits.iter().filter(|x| **x == Some(s)).count();
-    assert_eq!((count(Split::Train), count(Split::Validation)), (3, 3));
+    assert_eq!((count(Split::Train), count(Split::Validation)), (14, 14));
     assert_eq!(count(Split::Test), 0);
 }
 
+/// The first six items are the original golf-loop set; widening the store
+/// adds items after them and never edits them.
+#[test]
+fn the_first_six_original_items_are_unchanged() {
+    let (corpus, _) = stores();
+    let want = [
+        (
+            "item:original:1",
+            "A quiet mind is like still water in a deep bowl.",
+            "original-bowl",
+        ),
+        (
+            "item:original:2",
+            "The river weakens the bank in spring and strengthens the roots in autumn.",
+            "original-river",
+        ),
+        (
+            "item:original:3",
+            "The sage waits at the door until the water has settled.",
+            "original-door",
+        ),
+        (
+            "item:original:4",
+            "The old gardener empties the jar and fills it again with rainwater.",
+            "original-jar",
+        ),
+        (
+            "item:original:5",
+            "Under the same sky the earth keeps its patience, and the baker keeps hers.",
+            "original-sky",
+        ),
+        (
+            "item:original:6",
+            "A hungry belly argues louder than any will.",
+            "original-belly",
+        ),
+    ];
+    for (id, text, group) in want {
+        let item = corpus.get(id).unwrap().unwrap();
+        assert_eq!(
+            (item.text.as_str(), item.group.as_str()),
+            (text, group),
+            "{id}"
+        );
+    }
+}
+
 #[test]
 fn the_original_inventory_is_the_dao_revision_three() {
     let dao = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/daodejing/inventory.cc");
@@ -69,7 +124,7 @@ fn the_original_inventory_is_the_dao_revision_three() {
 /// The probe over the real original corpus: every item is offered, none names a
 /// work, and a model that answers with unrelated words reads at the noise floor.
 #[test]
-fn the_recall_probe_covers_all_six_original_items_and_names_no_work() {
+fn the_recall_probe_covers_every_live_original_item_and_names_no_work() {
     use weksa::model::{ModelPort, PortError};
     use weksa::run::{RecallOutcome, RunPaths, RunStores};
     struct Unrelated(Vec<String>);
@@ -107,9 +162,24 @@ fn the_recall_probe_covers_all_six_original_items_and_names_no_work() {
     let RecallOutcome::Read(r) = run.recall(&mut model).unwrap() else {
         panic!("answered");
     };
-    assert_eq!((r.n, r.answered), (6, 6));
+    assert_eq!((r.n, r.answered), (28, 28));
     assert!(r.chrf < 25.0, "{}", r.chrf);
-    assert_eq!(model.0.len(), 6);
+    assert_eq!(model.0.len(), 28);
+    // No item outside Train and Validation is shown to the model.
+    let all = CorpusStore::open(&c).unwrap().all().unwrap();
+    for item in &all {
+        let live = SplitScheme::V1.live(item);
+        // A recall prompt shows the first half of the words.
+        let words: Vec<&str> = item.text.split_whitespace().collect();
+        let opening = words[..words.len() / 2].join(" ");
+        let shown = model.0.iter().any(|t| t.contains(&opening));
+        assert_eq!(
+            shown,
+            matches!(live, Some(Split::Train | Split::Validation)),
+            "{}",
+            item.id
+        );
+    }
     assert!(model.0.iter().all(|t| t.contains("work: not published")));
     assert!(model.0.iter().all(|t| !t.contains("translator:")));
 }
@@ -168,16 +238,16 @@ fn original_round(family: ModelFamily) -> (weksa::run::RunStores, weksa::run::Ro
 
 #[test]
 fn own_family_counts_and_labels() {
-    // The original corpus is six Anthropic-written items, all offered.
+    // The original corpus is Anthropic-written items, all offered.
     let (run, r) = original_round(ModelFamily::Anthropic);
-    assert_eq!(r.own_family(), 6);
+    assert_eq!(r.own_family(), 28);
     assert_eq!(
         r.parse_reasons().iter().map(|c| c.count).sum::<u32>(),
-        6,
+        28,
         "every item was refused at Parse and says why"
     );
     let line = run.round_line(&r);
-    assert!(line.contains(" own-family 6"), "{line}");
+    assert!(line.contains(" own-family 28"), "{line}");
 
     // A different answerer family, or none recorded, labels nothing.
     for family in [ModelFamily::OpenAi, ModelFamily::Unrecorded] {
@@ -186,3 +256,91 @@ fn own_family_counts_and_labels() {
         assert!(!run.round_line(&r).contains("own-family"));
     }
 }
+
+/// Over a fixture store whose items are not all live: recall probes the live
+/// Train and Validation items and no others (a withdrawn item, a model-written
+/// item of a test group, a public-domain Test item).
+#[test]
+fn recall_probes_only_live_train_and_validation_items() {
+    use common::{cast, entities_in, group_in_bucket, item_in, run_over};
+    use weksa::model::{ModelPort, PortError};
+    use weksa::run::RecallOutcome;
+    struct Record(Vec<String>);
+    impl ModelPort for Record {
+        fn model(&self) -> &str {
+            "record"
+        }
+        fn answer(&mut self, p: &weksa::context::Prompt) -> Result<String, PortError> {
+            self.0.push(p.text().to_string());
+            Ok(r#"{"text":"nothing"}"#.to_string())
+        }
+    }
+    let dir = tempfile::tempdir().unwrap();
+    let entities = entities_in(dir.path());
+    let mut store = CorpusStore::open(&dir.path().join("c.cc")).unwrap();
+    let public = Source::PublicDomain {
+        work: "A Work".into(),
+        author: "An Author".into(),
+        edition_url: "https://example.org/e".into(),
+        licence: "CC0".into(),
+        jurisdiction: "US".into(),
+        translator: "A Translator".into(),
+        edition_year: 1900,
+    };
+    let rows = [
+        (
+            "item:f:1",
+            "trainmark opens here and goes on",
+            10,
+            cast(),
+            false,
+        ),
+        (
+            "item:f:2",
+            "validmark opens here and goes on",
+            50,
+            cast(),
+            false,
+        ),
+        (
+            "item:f:3",
+            "gonemark opens here and goes on",
+            10,
+            cast(),
+            true,
+        ),
+        (
+            "item:f:4",
+            "modelmark opens here and goes on",
+            90,
+            common::model(),
+            false,
+        ),
+        (
+            "item:f:5",
+            "publicmark opens here and goes on",
+            90,
+            public,
+            false,
+        ),
+    ];
+    for (id, text, bucket, source, withdrawn) in rows {
+        let item = item_in(id, text, &group_in_bucket(bucket), source);
+        store.add(item, &entities).unwrap();
+        if withdrawn {
+            store.withdraw(id).unwrap();
+        }
+    }
+    drop(store);
+    let run = run_over(dir.path());
+    let mut m = Record(vec![]);
+    let RecallOutcome::Read(r) = run.recall(&mut m).unwrap() else {
+        panic!("answered");
+    };
+    assert_eq!(r.n, 2);
+    let shown = |mark: &str| m.0.iter().any(|t| t.contains(mark));
+    assert!(shown("trainmark") && shown("validmark"));
+    for hidden in ["gonemark", "modelmark", "publicmark"] {
+        assert!(!shown(hidden), "recall showed {hidden}");
+    }
+}