diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index e52da20..ae0b7fe 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -2952,6 +2952,8 @@ fn admit_refusals_are_counted_by_rule_and_printed_with_the_label() { struct Reciter { corpus: weksa::store::CorpusStore, answer: fn(&str) -> String, + /// An item whose text holds this gets an answer file that is not UTF-8. + unreadable: Option<&'static str>, seen: Vec, } @@ -2963,6 +2965,9 @@ impl ModelPort for Reciter { self.seen.push(prompt.text().to_string()); let key = &prompt.item_keys()[0]; let item = self.corpus.get(key).unwrap().unwrap(); + if self.unreadable.is_some_and(|t| item.text.contains(t)) { + return Err(PortError::Io(std::io::ErrorKind::InvalidData)); + } Ok((self.answer)(&item.text)) } } @@ -2977,6 +2982,7 @@ fn reciter(dao: &Dao, answer: fn(&str) -> String) -> Reciter { Reciter { corpus: weksa::store::CorpusStore::open(&dao.corpus).unwrap(), answer, + unreadable: None, seen: vec![], } } @@ -3039,3 +3045,40 @@ fn the_published_round_schema_does_not_require_a_field_older_records_lack() { assert!(!required.iter().any(|r| r == "admit_rules"), "{required:?}"); assert!(schema["properties"]["admit_rules"].is_object()); } + +#[test] +fn a_recall_mean_is_over_every_probe_an_unreadable_answer_counts_0_and_only_live_public_domain_items_are_probed() { + use weksa::corpus::Source; + use weksa::run::RecallOutcome; + let dao = dao_over(|corpus, entities| { + let entities = weksa::EntityStore::open(entities).unwrap(); + let mut store = weksa::CorpusStore::open(corpus).unwrap(); + // The same words under other sources: not public domain, not probed. + let mut written = store.get("item:daodejing:2").unwrap().unwrap(); + written.id = "item:written:1".into(); + written.text = "A line of some other source with words enough to split.".into(); + written.names = vec![]; + written.prior_turns = vec![]; + written.source = Source::OperatorWritten; + store.add(written, &entities).unwrap(); + store.withdraw("item:daodejing:3").unwrap(); + }); + let both = |dao: &Dao, unreadable: Option<&'static str>| { + let mut m = reciter(dao, second_half); + m.unreadable = unreadable; + let RecallOutcome::Read(r) = dao.run.recall(&mut m).unwrap() else { + panic!("answered"); + }; + r + }; + let full = both(&dao, None); + assert_eq!( + (full.n, full.answered), + (2, 2), + "neither the written item nor the withdrawn one is probed" + ); + // One of two unreadable: the mean falls to a half. + let part = both(&dao, Some("Heaven is long-enduring")); + assert_eq!((part.n, part.answered), (2, 1)); + assert!((part.chrf - full.chrf / 2.0).abs() < 1e-9, "{}", part.chrf); +}