Recall tests: mean over every probe, unreadable answer scores 0, written and withdrawn items are not probed

Soul lows: the mean-to-max, skipped-for-0 and withdrawn-gate mutants survived.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Weksa 2026-11-10 14:15:27 +00:00
commit 0ead162d8e

View file

@ -2952,6 +2952,8 @@ fn admit_refusals_are_counted_by_rule_and_printed_with_the_label() {
struct Reciter {
corpus: weksa::store::CorpusStore,
answer: fn(&str) -> String,
/// An item whose text holds this gets an answer file that is not UTF-8.
unreadable: Option<&'static str>,
seen: Vec<String>,
}
@ -2963,6 +2965,9 @@ impl ModelPort for Reciter {
self.seen.push(prompt.text().to_string());
let key = &prompt.item_keys()[0];
let item = self.corpus.get(key).unwrap().unwrap();
if self.unreadable.is_some_and(|t| item.text.contains(t)) {
return Err(PortError::Io(std::io::ErrorKind::InvalidData));
}
Ok((self.answer)(&item.text))
}
}
@ -2977,6 +2982,7 @@ fn reciter(dao: &Dao, answer: fn(&str) -> String) -> Reciter {
Reciter {
corpus: weksa::store::CorpusStore::open(&dao.corpus).unwrap(),
answer,
unreadable: None,
seen: vec![],
}
}
@ -3039,3 +3045,40 @@ fn the_published_round_schema_does_not_require_a_field_older_records_lack() {
assert!(!required.iter().any(|r| r == "admit_rules"), "{required:?}");
assert!(schema["properties"]["admit_rules"].is_object());
}
#[test]
fn a_recall_mean_is_over_every_probe_an_unreadable_answer_counts_0_and_only_live_public_domain_items_are_probed() {
use weksa::corpus::Source;
use weksa::run::RecallOutcome;
let dao = dao_over(|corpus, entities| {
let entities = weksa::EntityStore::open(entities).unwrap();
let mut store = weksa::CorpusStore::open(corpus).unwrap();
// The same words under other sources: not public domain, not probed.
let mut written = store.get("item:daodejing:2").unwrap().unwrap();
written.id = "item:written:1".into();
written.text = "A line of some other source with words enough to split.".into();
written.names = vec![];
written.prior_turns = vec![];
written.source = Source::OperatorWritten;
store.add(written, &entities).unwrap();
store.withdraw("item:daodejing:3").unwrap();
});
let both = |dao: &Dao, unreadable: Option<&'static str>| {
let mut m = reciter(dao, second_half);
m.unreadable = unreadable;
let RecallOutcome::Read(r) = dao.run.recall(&mut m).unwrap() else {
panic!("answered");
};
r
};
let full = both(&dao, None);
assert_eq!(
(full.n, full.answered),
(2, 2),
"neither the written item nor the withdrawn one is probed"
);
// One of two unreadable: the mean falls to a half.
let part = both(&dao, Some("Heaven is long-enduring"));
assert_eq!((part.n, part.answered), (2, 1));
assert!((part.chrf - full.chrf / 2.0).abs() < 1e-9, "{}", part.chrf);
}