Recall tests: mean over every probe, unreadable answer scores 0, written and withdrawn items are not probed
Soul lows: the mean-to-max, skipped-for-0 and withdrawn-gate mutants survived. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
d954cfb9e9
commit
0ead162d8e
1 changed files with 43 additions and 0 deletions
|
|
@ -2952,6 +2952,8 @@ fn admit_refusals_are_counted_by_rule_and_printed_with_the_label() {
|
|||
struct Reciter {
|
||||
corpus: weksa::store::CorpusStore,
|
||||
answer: fn(&str) -> String,
|
||||
/// An item whose text holds this gets an answer file that is not UTF-8.
|
||||
unreadable: Option<&'static str>,
|
||||
seen: Vec<String>,
|
||||
}
|
||||
|
||||
|
|
@ -2963,6 +2965,9 @@ impl ModelPort for Reciter {
|
|||
self.seen.push(prompt.text().to_string());
|
||||
let key = &prompt.item_keys()[0];
|
||||
let item = self.corpus.get(key).unwrap().unwrap();
|
||||
if self.unreadable.is_some_and(|t| item.text.contains(t)) {
|
||||
return Err(PortError::Io(std::io::ErrorKind::InvalidData));
|
||||
}
|
||||
Ok((self.answer)(&item.text))
|
||||
}
|
||||
}
|
||||
|
|
@ -2977,6 +2982,7 @@ fn reciter(dao: &Dao, answer: fn(&str) -> String) -> Reciter {
|
|||
Reciter {
|
||||
corpus: weksa::store::CorpusStore::open(&dao.corpus).unwrap(),
|
||||
answer,
|
||||
unreadable: None,
|
||||
seen: vec![],
|
||||
}
|
||||
}
|
||||
|
|
@ -3039,3 +3045,40 @@ fn the_published_round_schema_does_not_require_a_field_older_records_lack() {
|
|||
assert!(!required.iter().any(|r| r == "admit_rules"), "{required:?}");
|
||||
assert!(schema["properties"]["admit_rules"].is_object());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_recall_mean_is_over_every_probe_an_unreadable_answer_counts_0_and_only_live_public_domain_items_are_probed() {
|
||||
use weksa::corpus::Source;
|
||||
use weksa::run::RecallOutcome;
|
||||
let dao = dao_over(|corpus, entities| {
|
||||
let entities = weksa::EntityStore::open(entities).unwrap();
|
||||
let mut store = weksa::CorpusStore::open(corpus).unwrap();
|
||||
// The same words under other sources: not public domain, not probed.
|
||||
let mut written = store.get("item:daodejing:2").unwrap().unwrap();
|
||||
written.id = "item:written:1".into();
|
||||
written.text = "A line of some other source with words enough to split.".into();
|
||||
written.names = vec![];
|
||||
written.prior_turns = vec![];
|
||||
written.source = Source::OperatorWritten;
|
||||
store.add(written, &entities).unwrap();
|
||||
store.withdraw("item:daodejing:3").unwrap();
|
||||
});
|
||||
let both = |dao: &Dao, unreadable: Option<&'static str>| {
|
||||
let mut m = reciter(dao, second_half);
|
||||
m.unreadable = unreadable;
|
||||
let RecallOutcome::Read(r) = dao.run.recall(&mut m).unwrap() else {
|
||||
panic!("answered");
|
||||
};
|
||||
r
|
||||
};
|
||||
let full = both(&dao, None);
|
||||
assert_eq!(
|
||||
(full.n, full.answered),
|
||||
(2, 2),
|
||||
"neither the written item nor the withdrawn one is probed"
|
||||
);
|
||||
// One of two unreadable: the mean falls to a half.
|
||||
let part = both(&dao, Some("Heaven is long-enduring"));
|
||||
assert_eq!((part.n, part.answered), (2, 1));
|
||||
assert!((part.chrf - full.chrf / 2.0).abs() < 1e-9, "{}", part.chrf);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue