Loop tests follow the panel: en-US panel in every fixture run, per-language reads, recall at word boundaries

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Weksa 2026-11-12 12:29:23 +00:00
commit 3e4b035be6
6 changed files with 245 additions and 144 deletions

View file

@ -637,6 +637,17 @@ mod tests {
);
}
#[test]
fn a_mint_prompt_over_no_train_items_is_the_brief_the_feedback_and_the_listing() {
let none = TrainItems { items: vec![] };
let prompt = Prompt::mint(&none, &Inventory::default(), 0, &EditFeedback::none());
assert_eq!(
prompt.text(),
format!("{MINT_INSTRUCTIONS}\n\nlast edit: none; validation composite at head: none\n\ninventory rev:0")
);
assert!(prompt.item_keys().is_empty());
}
#[test]
fn checkpoint_offer_gate() {
// A fresh checkpoint per call: this test is the gate, not the once rule.

View file

@ -47,7 +47,7 @@ fn out(o: &Output) -> String {
String::from_utf8_lossy(&o.stdout).into_owned() + &String::from_utf8_lossy(&o.stderr)
}
const ADD: [&str; 14] = [
const ADD: [&str; 16] = [
"corpus",
"add",
"--slug",
@ -62,6 +62,8 @@ const ADD: [&str; 14] = [
"2026-10-29",
"--source",
"cast-authored",
"--language",
"en-US",
];
#[test]
@ -149,6 +151,8 @@ fn model_written_and_public_domain_sources_name_their_provenance() {
let base = [
"corpus",
"add",
"--language",
"en-US",
"--slug",
"x",
"--speaker",
@ -375,6 +379,8 @@ fn entity_put_and_name_spans() {
&[
"corpus",
"add",
"--language",
"en-US",
"--slug",
"kay",
"--text",
@ -547,6 +553,8 @@ fn add_with(dir: &std::path::Path, source: &str, extra: &[&str]) -> Output {
let mut args = vec![
"corpus",
"add",
"--language",
"en-US",
"--slug",
"nibu",
"--text",
@ -903,6 +911,8 @@ fn proof_packet_lowers_and_the_lowering_scores_against_items() {
&[
"corpus",
"add",
"--language",
"en-US",
"--slug",
"nibu",
"--text",
@ -1062,7 +1072,7 @@ fn cli_errors_never_echo_identifiers() {
let p = d.path();
cast_entities(p, &["nibu"]);
let add = |extra: &[&str]| -> Vec<String> {
["corpus", "add", "--admitted-on", "2026-10-29"]
["corpus", "add", "--language", "en-US", "--admitted-on", "2026-10-29"]
.into_iter()
.chain(extra.iter().copied())
.map(String::from)
@ -1344,6 +1354,8 @@ fn loop_open_and_unanswered_round() {
"smoke",
"--answerer",
"fake-model",
"--panel",
"en-US",
"--lowerer-reach",
"tool-less-relay",
"--answerer-family",
@ -1397,7 +1409,7 @@ fn loop_open_and_unanswered_round() {
assert!(second.status.success(), "{}", out(&second));
assert_eq!(
out(&second),
"round 1 rev 3 train n=1 scored=0 chrf=0.00 meaning=0.0000 refused=1 validation n=2 scored=0 chrf=0.00 meaning=0.0000 refused parse=2 admit=0 lower=0 parse-reasons=syntaxx3 lowering=tool-less-relay\n"
"round 1 rev 3 en-US train n=1 scored=0 chrf=0.00 meaning=0.0000 refused=1 validation n=2 scored=0 chrf=0.00 meaning=0.0000 refused parse=2 admit=0 lower=0 parse-reasons=syntaxx3 lowering=tool-less-relay\n"
);
// The global store flags are not read by loop round: a drifted store is refused.
@ -1424,6 +1436,8 @@ fn open_dao_run(d: &std::path::Path) {
"smoke",
"--answerer",
"fake-model",
"--panel",
"en-US",
"--lowerer-reach",
"tool-less-relay",
],
@ -1441,6 +1455,8 @@ fn pinned_family(d: &std::path::Path, root: &str, flag: &[&str]) -> weksa::corpu
"fam",
"--answerer",
"fake-model",
"--panel",
"en-US",
"--lowerer-reach",
"tool-less-relay",
];
@ -1526,7 +1542,7 @@ fn loop_cycle_asks_then_decides() {
assert!(done.status.success(), "{}", out(&done));
assert_eq!(
out(&done),
"cycle 1 rev 3->none base=0.0000 candidate=none floor=0.0000 threshold=0.0050 rejected edit-parse lowering=tool-less-relay\n"
"cycle 1 rev 3->none en-US base=0.0000 candidate=none floor=0.0000 threshold=0.0050 rejected edit-parse lowering=tool-less-relay\n"
);
// The next cycle asks afresh in its own directory.
@ -1565,6 +1581,8 @@ fn loop_errors_echo_no_input() {
"CANARYLABEL",
"--answerer",
"m",
"--panel",
"en-US",
"--lowerer-reach",
"tool-less-relay",
],
@ -1576,6 +1594,8 @@ fn loop_errors_echo_no_input() {
"ok",
"--answerer",
"CANARY\n",
"--panel",
"en-US",
"--lowerer-reach",
"tool-less-relay",
],
@ -1597,6 +1617,8 @@ fn loop_errors_echo_no_input() {
"ok",
"--answerer",
"m",
"--panel",
"en-US",
"--lowerer-reach",
"tool-less-relay",
])
@ -1610,7 +1632,7 @@ fn loop_errors_echo_no_input() {
fn loop_open_requires_the_lowerer_reach_and_pins_it() {
let d = tempfile::tempdir().unwrap();
dao_stores(d.path());
let base = ["loop", "open", "canaryroot", "--label", "smoke", "--answerer", "m"];
let base = ["loop", "open", "canaryroot", "--label", "smoke", "--answerer", "m", "--panel", "en-US"];
// No default: the record says how lowerings are answered, or there is no run.
let o = weksa(d.path(), &base);
assert!(!o.status.success());
@ -1698,7 +1720,7 @@ fn loop_ablate_waits_then_prints_one_line_per_split() {
let text = out(&second);
assert_eq!(
text,
"train ablation n=1 unusable=0 replay chrf=100.00 meaning=1.000 | drop n=1 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000 | permute n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none\nvalidation ablation n=2 unusable=0 replay chrf=100.00 meaning=1.000 | drop n=2 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000 | permute n=2 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000\n"
"en-US train ablation n=1 unusable=0 replay chrf=100.00 meaning=1.000 | drop n=1 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000 | permute n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none\nen-US validation ablation n=2 unusable=0 replay chrf=100.00 meaning=1.000 | drop n=2 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000 | permute n=2 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000\n"
);
for leaked in ["item:", "packet:", "highest excellence", "canaryroot"] {
assert!(!text.contains(leaked), "{text}");

View file

@ -116,12 +116,30 @@ pub fn entities_in(dir: &std::path::Path) -> EntityStore {
/// A run over the stores `world()` and `corpus()` keep in `dir`: `c.cc`,
/// `i.cc` and `entities.cc`. Read at the call, so create it after the items.
/// The run's en-US panel needs a live Train and a live Validation item: a
/// corpus short of one gets `item:panel:<n>` added, and no other item.
pub fn run_over(dir: &std::path::Path) -> weksa::run::RunStores {
for name in ["c.cc", "i.cc", "entities.cc"] {
if !dir.join(name).exists() {
std::fs::write(dir.join(name), [0x90]).unwrap();
}
}
let entities = entities_in(dir);
let mut corpus = CorpusStore::open(&dir.join("c.cc")).unwrap();
for (n, split) in [(1, weksa::Split::Train), (2, weksa::Split::Validation)] {
let held = corpus
.all()
.unwrap()
.iter()
.any(|i| SplitScheme::V1.live(i) == Some(split));
if !held {
let group = group_in_bucket(if n == 1 { 10 } else { 50 });
let key = format!("item:panel:{n}");
let text = format!("Panel fixture line {n}.");
corpus.add(item_in(&key, &text, &group, cast()), &entities).unwrap();
}
}
drop(corpus);
weksa::run::RunStores::create(
&dir.join("run"),
"t",
@ -134,6 +152,7 @@ pub fn run_over(dir: &std::path::Path) -> weksa::run::RunStores {
"bag",
weksa::run::LowererReach::ToolLessRelay,
weksa::corpus::ModelFamily::Unrecorded,
&["en-US".to_string()],
)
.unwrap()
}
@ -219,6 +238,10 @@ pub fn world() -> World {
end: 4,
}];
corpus.add(first, &entities).unwrap();
// A run's panel needs a live Validation item as well as a Train one.
let validation = group_in_bucket(50);
let second = item_in("item:panel:1", "Nibu says watch the ladder.", &validation, cast());
corpus.add(second, &entities).unwrap();
let mut inventory = InventoryStore::open(&dir.path().join("i.cc")).unwrap();
let (rev1, entries) = seed();
inventory.admit_revision(rev1, entries).unwrap();

View file

@ -110,16 +110,6 @@ fn train_items_are_in_key_order() {
["item:a:1", "item:m:1", "item:z:1"]
);
assert_eq!((t.len(), t.is_empty()), (3, false));
let dir = tempfile::tempdir().unwrap();
let empty = run_over(dir.path());
let none = empty.train().unwrap();
assert_eq!((none.len(), none.is_empty()), (0, true));
let (inv, fb) = (empty.inventory().inventory().unwrap(), EditFeedback::none());
assert_eq!(
Prompt::mint(&none, &inv, 0, &fb).text(),
format!("{MINT}\n\nlast edit: none; validation composite at head: none\n\ninventory rev:0")
);
assert!(Prompt::mint(&none, &inv, 0, &fb).item_keys().is_empty());
}
#[test]
@ -606,13 +596,28 @@ mod lowering {
}
}
/// The language line carries the tag the caller passes; no instruction
/// names a language.
#[test]
fn lower_prompt_names_its_language() {
let (view, inv, entities, item) = dao(2);
for tag in ["en-US", "ja-JP", "pt-BR"] {
let prompt = Prompt::lower(&view, &item, tag, &inv, &entities).unwrap();
let want = format!("{LOWER}\n\nlanguage: {tag}\n\npacket: ");
assert!(prompt.text().starts_with(&want), "{tag}");
}
for brief in [LOWER, weksa::context::MINT_INSTRUCTIONS] {
assert!(!brief.contains("English") && !brief.contains("en-US"));
}
}
#[test]
fn lower_prompt_text_is_pinned() {
let (view, inv, entities, item) = dao(2);
let prompt = Prompt::lower(&view, &item, &inv, &entities).unwrap();
let prompt = Prompt::lower(&view, &item, "en-US", &inv, &entities).unwrap();
let (head, listing) = prompt.text().split_once("inventory rev:3\n").unwrap();
let expected = format!(
"{LOWER}\n\npacket: {}\n\nentities: {{}}\n\n",
"{LOWER}\n\nlanguage: en-US\n\npacket: {}\n\nentities: {{}}\n\n",
concat!(
r#"{"context":{"setting":null,"medium":null,"participants":[],"audience":[]},"#,
r#""discourse":{"speech_act":"speech_act:1","addressee":null,"addressee_consent":null,"features":[]},"#,
@ -628,7 +633,7 @@ mod lowering {
assert!(listing.contains("\nfeature:18: "));
// The brief says what the spec says it must.
for needed in [
"en-US",
"the language on the language line",
"all of its meaning and nothing it does not carry",
"a grade is permille, 0 to 1000",
"felt affect and restraint",
@ -671,7 +676,7 @@ mod lowering {
let mut packet = packets().remove(2);
packet.stance = stance.clone();
let view = admitted(packet, &store, "daodejing/corpus.cc", &entities);
let prompt = Prompt::lower(&view, &item, &inv, &entities).unwrap();
let prompt = Prompt::lower(&view, &item, "en-US", &inv, &entities).unwrap();
(stance, prompt)
};
let three = [prompt_of(own), prompt_of(angry), prompt_of(Stance::default())];
@ -708,7 +713,7 @@ mod lowering {
("sa-Latn", &["Nibuwa"]),
("en-US", &["Nibu", "Nibu's"]),
])]);
let prompt = Prompt::lower(&view, &item, &cast.inventory().unwrap(), &held).unwrap();
let prompt = Prompt::lower(&view, &item, "en-US", &cast.inventory().unwrap(), &held).unwrap();
assert!(
prompt
.text()
@ -738,7 +743,7 @@ mod lowering {
),
(Held(vec![]), LowerError::UnknownEntity { entity: id }),
] {
assert_eq!(Prompt::lower(&view, &item, &inv, &held).err(), Some(want));
assert_eq!(Prompt::lower(&view, &item, "en-US", &inv, &held).err(), Some(want));
}
}
}

View file

@ -51,6 +51,7 @@ fn a_run_pins_its_stores_and_refuses_drift() {
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&["en-US".to_string()],
)
};
@ -154,6 +155,7 @@ fn a_run_works_on_its_own_copy_of_the_inventory() {
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&["en-US".to_string()],
)
.unwrap();
@ -231,6 +233,7 @@ fn create_refuses_a_root_that_holds_an_inventory_and_never_truncates_it() {
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&["en-US".to_string()],
)
};
@ -278,6 +281,7 @@ fn a_create_that_fails_leaves_no_inventory_and_the_retry_succeeds() {
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&["en-US".to_string()],
)
};
// A directory opens for reading and fails mid-copy, after the destination exists.
@ -398,6 +402,7 @@ fn a_store_changed_between_its_digest_and_its_open_is_drift() {
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&["en-US".to_string()],
|| more(&mut w)
)
.err(),
@ -415,6 +420,7 @@ fn a_store_changed_between_its_digest_and_its_open_is_drift() {
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&["en-US".to_string()],
)
.unwrap();
}
@ -447,6 +453,7 @@ fn window(
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&["en-US".to_string()],
|| change(&mut w, &root),
)
.err();
@ -866,6 +873,7 @@ fn dao_over(edit: impl FnOnce(&Path, &Path)) -> Dao {
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&["en-US".to_string()],
)
.unwrap();
Dao {
@ -923,15 +931,15 @@ fn round_over_the_dao_stores_scores_train_and_validation() {
};
assert_eq!(two_dp(chrf), 44.18);
assert!(meaning.is_some());
assert_eq!((r.train_mean().count, r.train_mean().chrf), (1, Some(chrf)));
assert_eq!((r.language("en-US").unwrap().train_mean.count, r.language("en-US").unwrap().train_mean.chrf), (1, Some(chrf)));
// Validation: means and refusal counts only.
let v = r.validation_mean();
let v = &r.language("en-US").unwrap().validation_mean;
assert_eq!(v.count, 2);
assert!((v.chrf.unwrap() - 78.56).abs() < 0.01, "{:?}", v.chrf);
assert!(v.meaning.is_some());
assert_eq!(r.validation_refused(), Default::default());
assert_eq!((r.train_scored(), r.validation_scored()), (1, 2));
assert_eq!(r.language("en-US").unwrap().validation_refused, Default::default());
assert_eq!((r.language("en-US").unwrap().train_scored, r.language("en-US").unwrap().validation_scored), (1, 2));
// The record carries the run's pins.
assert_eq!(r.prompt_digest(), pinned.prompt_digest);
@ -1041,7 +1049,7 @@ fn lowering_prompts(dao: &Dao, inventory: &InventoryStore) -> Vec<(String, Promp
let view = store
.admit(packet, inventory, &corpus, dao.run.entities())
.unwrap();
let prompt = Prompt::lower(&view, &key, &inv, dao.run.entities()).unwrap();
let prompt = Prompt::lower(&view, &key, "en-US", &inv, dao.run.entities()).unwrap();
(key.to_string(), prompt)
})
.collect()
@ -1139,7 +1147,7 @@ fn unanswered_round_writes_nothing() {
let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into());
let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap());
assert_eq!((r.n(), r.calls().len()), (1, 6));
assert_eq!(two_dp(r.train_mean().chrf.unwrap()), 44.18);
assert_eq!(two_dp(r.language("en-US").unwrap().train_mean.chrf.unwrap()), 44.18);
// Round 2 asks again, encode first.
let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into());
@ -1261,15 +1269,15 @@ fn refusal_stages_are_counted() {
assert_eq!(r.train()[0].outcome, Outcome::Refused(Stage::Parse));
assert_eq!(r.train()[0].packet, None);
// A refused item is in the mean as 0: n counts every item, scored the real ones.
let m = r.train_mean();
let m = &r.language("en-US").unwrap().train_mean;
assert_eq!((m.count, m.chrf, m.meaning), (1, Some(0.0), Some(0.0)));
assert_eq!((r.train_scored(), r.validation_scored()), (0, 1));
assert_eq!(r.validation_refused().parse, 0);
assert_eq!((r.language("en-US").unwrap().train_scored, r.language("en-US").unwrap().validation_scored), (0, 1));
assert_eq!(r.language("en-US").unwrap().validation_refused.parse, 0);
assert_eq!(
(r.validation_refused().admit, r.validation_refused().lower),
(r.language("en-US").unwrap().validation_refused.admit, r.language("en-US").unwrap().validation_refused.lower),
(1, 0)
);
let m = r.validation_mean();
let m = &r.language("en-US").unwrap().validation_mean;
assert_eq!(m.count, 2);
assert!(m.chrf.unwrap() > 0.0, "the relation item lowered and scored");
@ -1291,29 +1299,29 @@ fn refusal_stages_are_counted() {
None,
"a refused packet is not stored"
);
let v = b_only.validation_refused();
let v = b_only.language("en-US").unwrap().validation_refused;
assert_eq!((v.parse, v.admit, v.lower), (1, 0, 0));
assert_eq!(
b_only.validation_mean().count,
b_only.language("en-US").unwrap().validation_mean.count,
2,
"every Validation item counts"
);
assert_eq!(b_only.validation_scored(), 1);
let b_half = b_only.validation_mean().chrf.unwrap();
assert_eq!(b_only.language("en-US").unwrap().validation_scored, 1);
let b_half = b_only.language("en-US").unwrap().validation_mean.chrf.unwrap();
assert!(
(b_half - 74.99 / 2.0).abs() < 0.01,
"(74.99 + 0) / 2 = {b_half}"
);
let a_only = run([good_answers()[0].1.clone(), good_answers()[1].1.clone(), junk]);
assert_eq!(a_only.validation_refused().parse, 1);
let a_half = a_only.validation_mean().chrf.unwrap();
assert_eq!(a_only.language("en-US").unwrap().validation_refused.parse, 1);
let a_half = a_only.language("en-US").unwrap().validation_mean.chrf.unwrap();
assert!(
(a_half - 82.12 / 2.0).abs() < 0.01,
"(82.12 + 0) / 2 = {a_half}"
);
// The halves add to the full mean: (a + 0) / 2 + (0 + b) / 2 = (a + b) / 2.
let whole = full.validation_mean().chrf.unwrap();
let whole = full.language("en-US").unwrap().validation_mean.chrf.unwrap();
assert!(
(a_half + b_half - whole).abs() < 1e-9,
"{a_half} + {b_half} vs {whole}"
@ -1322,7 +1330,7 @@ fn refusal_stages_are_counted() {
b_half < whole && a_half < whole,
"refusing never raises the mean"
);
assert_eq!(full.validation_scored(), 2);
assert_eq!(full.language("en-US").unwrap().validation_scored, 2);
}
#[test]
@ -1356,10 +1364,10 @@ fn non_utf8_encode_answer_is_a_parse_refusal() {
let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap());
assert_eq!(r.calls().len(), 5);
// Item 3 is Validation: its refusal is a Parse refusal and the round stands.
let v = r.validation_refused();
let v = r.language("en-US").unwrap().validation_refused;
assert_eq!((v.parse, v.admit, v.lower), (1, 0, 0));
assert_eq!((r.train_scored(), r.validation_scored()), (1, 1));
assert_eq!(r.validation_mean().count, 2);
assert_eq!((r.language("en-US").unwrap().train_scored, r.language("en-US").unwrap().validation_scored), (1, 1));
assert_eq!(r.language("en-US").unwrap().validation_mean.count, 2);
assert_eq!(dao.run.rounds().unwrap(), [r]);
}
@ -1402,7 +1410,7 @@ fn a_failed_round_leaves_nothing_a_rerun_collides_with() {
"{:?}",
r.train()[0]
);
assert_eq!(r.validation_refused(), Default::default());
assert_eq!(r.language("en-US").unwrap().validation_refused, Default::default());
let stored = PacketStore::open(&dao.root.join("round-1/packets.cc"))
.unwrap()
.all()
@ -1441,7 +1449,7 @@ fn a_stale_packet_store_in_an_unrecorded_round_dir_is_discarded_at_start() {
"the revised draft is not refused as a changed packet: {:?}",
r.train()[0]
);
assert_eq!(r.validation_refused(), Default::default());
assert_eq!(r.language("en-US").unwrap().validation_refused, Default::default());
let stored = PacketStore::open(&dir.join("packets.cc"))
.unwrap()
.all()
@ -1467,11 +1475,11 @@ fn the_round_line_prints_each_split_under_its_own_name() {
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
let line = dao.run.round_line(&r);
// Train is item 1 alone, Validation items 2 and 3; the numbers differ.
let (t, v) = (r.train_mean(), r.validation_mean());
let (t, v) = (&r.language("en-US").unwrap().train_mean, &r.language("en-US").unwrap().validation_mean);
assert_eq!(
line,
format!(
"round 1 rev 3 train n=1 scored=1 chrf={:.2} meaning={:.4} refused=0 validation n=2 scored=2 chrf={:.2} meaning={:.4} refused parse=0 admit=0 lower=0 lowering=tool-less-relay",
"round 1 rev 3 en-US train n=1 scored=1 chrf={:.2} meaning={:.4} refused=0 validation n=2 scored=2 chrf={:.2} meaning={:.4} refused parse=0 admit=0 lower=0 lowering=tool-less-relay",
t.chrf.unwrap(),
t.meaning.unwrap(),
v.chrf.unwrap(),
@ -1522,6 +1530,7 @@ fn errors_echo_no_input() {
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&["en-US".to_string()],
)
.err()
.unwrap();
@ -1534,6 +1543,7 @@ fn errors_echo_no_input() {
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&["en-US".to_string()],
)
.err()
.unwrap();
@ -1546,6 +1556,7 @@ fn errors_echo_no_input() {
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&["en-US".to_string()],
)
.err()
.unwrap();
@ -1583,6 +1594,7 @@ fn errors_echo_no_input() {
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&["en-US".to_string()],
)
.unwrap();
let mut model = cycle_fake(&format!("{{\"{canary}\":1}}"));
@ -1717,13 +1729,13 @@ fn head_rev(root: &Path) -> u32 {
}
fn validation(r: &RoundRecord) -> Reading {
Reading::of(r.validation_mean()).unwrap()
Reading::of(&r.language("en-US").unwrap().validation_mean).unwrap()
}
/// The feedback line the editor is shown for a head round, built from outside.
fn feedback_line(last: &str, head: &RoundRecord) -> String {
let rounded = (validation(head).composite / LADDER_STEP).round() * LADDER_STEP;
format!("last edit: {last}; validation composite at head: {rounded:.3}")
format!("last edit: {last}; validation composite at head: en-US {rounded:.3}")
}
fn mint_texts(model: &FakeModel) -> Vec<&str> {
@ -1751,19 +1763,19 @@ fn cycle_accepts_a_gain_past_the_floor() {
);
assert_eq!((d.base_rev(), d.candidate_rev()), (3, Some(4)));
assert_eq!(d.verdict(), Verdict::Accepted);
assert_eq!(d.threshold(), LADDER_STEP);
assert_eq!(d.panel()[0].threshold, LADDER_STEP);
assert_eq!(
*d.floor(),
d.panel()[0].floor,
Reading {
chrf: 0.0,
meaning: 0.0,
composite: 0.0
}
);
assert_eq!(*d.base(), validation(&rounds[0]));
assert_eq!(d.panel()[0].base, validation(&rounds[0]));
let record = dao.run.rounds().unwrap().remove(2);
assert_eq!(d.candidate(), Some(&validation(&record)));
assert!(d.candidate().unwrap().composite - d.base().composite > LADDER_STEP);
assert_eq!(d.panel()[0].candidate, Some(validation(&record)));
assert!(d.panel()[0].candidate.unwrap().composite - d.panel()[0].base.composite > LADDER_STEP);
// The candidate round was scored under the candidate file, as cycle 1's.
let file = dao.root.join("cycle-1/inventory.cc");
@ -1787,11 +1799,11 @@ fn cycle_accepts_a_gain_past_the_floor() {
assert_eq!(d.edit().prompt, model.seen[0].0);
// The line, with the readings printed to four places.
let (b, c) = (d.base().composite, d.candidate().unwrap().composite);
let (b, c) = (d.panel()[0].base.composite, d.panel()[0].candidate.unwrap().composite);
assert_eq!(
dao.run.cycle_line(&d),
format!(
"cycle 1 rev 3->4 base={b:.4} candidate={c:.4} floor=0.0000 threshold=0.0050 accepted lowering=tool-less-relay"
"cycle 1 rev 3->4 en-US base={b:.4} candidate={c:.4} floor=0.0000 threshold=0.0050 accepted lowering=tool-less-relay"
)
);
@ -1822,7 +1834,7 @@ fn cycle_accepts_a_gain_past_the_floor() {
assert_eq!(d2.candidate_rev(), Some(5));
assert_eq!(d2.candidate_round(), Some(5));
assert_eq!(d2.verdict(), Verdict::Rejected(Why::NoGain));
assert_eq!(*d2.floor(), *d.floor(), "the floor is measured once");
assert_eq!(d2.panel()[0].floor, d.panel()[0].floor, "the floor is measured once");
assert_eq!(head_rev(&dao.root), 4, "a rejected edit moves nothing");
assert!(
mint_texts(&second)[0].contains(&feedback_line("accepted", &dao.run.rounds().unwrap()[2])),
@ -1846,12 +1858,12 @@ fn cycle_rejects_no_gain_and_keeps_the_head() {
let d = run_cycle(&mut dao, &mut model);
assert_eq!(d.verdict(), Verdict::Rejected(Why::NoGain));
assert_eq!((d.candidate_rev(), d.candidate_round()), (Some(4), Some(3)));
assert_eq!(d.candidate(), Some(d.base()));
let c = d.base().composite;
assert_eq!(d.panel()[0].candidate, Some(d.panel()[0].base));
let c = d.panel()[0].base.composite;
assert_eq!(
dao.run.cycle_line(&d),
format!(
"cycle 1 rev 3->4 base={c:.4} candidate={c:.4} floor=0.0000 threshold=0.0050 rejected no-gain lowering=tool-less-relay"
"cycle 1 rev 3->4 en-US base={c:.4} candidate={c:.4} floor=0.0000 threshold=0.0050 rejected no-gain lowering=tool-less-relay"
)
);
assert_eq!(head_rev(&dao.root), 3);
@ -1911,10 +1923,10 @@ fn distinct_replicates_set_the_floor_and_a_gain_inside_it_is_no_gain() {
// (round 1) is the floor, which is past a step and inside the noise.
let mut model = cycle_fake(ONE_CONCEPT);
let d = run_cycle(&mut dao, &mut model);
assert_eq!(*d.floor(), want);
assert_eq!(d.threshold(), want.composite);
assert_eq!(d.candidate(), Some(&b));
assert!(d.candidate().unwrap().composite - d.base().composite > LADDER_STEP);
assert_eq!(d.panel()[0].floor, want);
assert_eq!(d.panel()[0].threshold, want.composite);
assert_eq!(d.panel()[0].candidate, Some(b));
assert!(d.panel()[0].candidate.unwrap().composite - d.panel()[0].base.composite > LADDER_STEP);
assert_eq!(d.verdict(), Verdict::Rejected(Why::NoGain));
assert_eq!(head_rev(&dao.root), 3);
}
@ -2101,7 +2113,7 @@ fn unanswered_cycle_writes_nothing() {
let mint = std::fs::read_to_string(&mint_file).unwrap();
assert_eq!(format!("{:016x}", fnv1a64(mint.as_bytes())), asked[0]);
assert!(mint.starts_with(MINT_INSTRUCTIONS));
assert!(mint.contains("\n\nlast edit: none; validation composite at head: 0.435\n\n"));
assert!(mint.contains("\n\nlast edit: none; validation composite at head: en-US 0.435\n\n"));
assert!(mint.contains("\n\ninventory rev:3\n"));
assert_eq!(mint.matches("[item:").count(), 1, "one Train block");
assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 1);
@ -2349,7 +2361,7 @@ fn edit_refusals_are_decisions() {
let d = run_cycle(&mut dao, &mut model);
assert_eq!(d.verdict(), Verdict::Rejected(why), "{name}");
assert_eq!(
(d.candidate_rev(), d.candidate_round(), d.candidate()),
(d.candidate_rev(), d.candidate_round(), d.panel()[0].candidate),
(None, None, None),
"{name}"
);
@ -2377,8 +2389,8 @@ fn edit_refusals_are_decisions() {
assert_eq!(
dao.run.cycle_line(&d),
format!(
"cycle 1 rev 3->none base={:.4} candidate=none floor=0.0000 threshold=0.0050 rejected {tag} lowering=tool-less-relay",
d.base().composite
"cycle 1 rev 3->none en-US base={:.4} candidate=none floor=0.0000 threshold=0.0050 rejected {tag} lowering=tool-less-relay",
d.panel()[0].base.composite
),
"{name}"
);
@ -2419,20 +2431,8 @@ fn cycle_needs_a_floor() {
"the run has no noise floor: record two rounds first"
);
// Rounds whose Validation items are all withdrawn have no reading.
let mut dao = dao_over(|corpus, _| {
let mut store = weksa::CorpusStore::open(corpus).unwrap();
for n in [2, 3] {
store.withdraw(&format!("item:daodejing:{n}")).unwrap();
}
});
for _ in 0..2 {
let mut m = fake(&[(1, json(&draft_of(1)))]);
let r = recorded(round(&mut dao.run, &mut m, &Scorer::new(Bag)).unwrap());
assert_eq!(r.validation_mean().count, 0);
}
let mut model = cycle_fake(ONE_CONCEPT);
assert_eq!(none(&mut dao, &mut model), Some(RunError::NoValidation));
// A panel language with no live Validation item never reaches a round:
// `create` refuses it (open_refuses_a_bad_panel).
assert_eq!(
RunError::NoValidation.to_string(),
"a reading has no validation item"
@ -2515,6 +2515,7 @@ fn a_changed_head_is_drift() {
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&["en-US".to_string()],
)
.unwrap();
assert_eq!(
@ -2772,10 +2773,10 @@ fn lowering_refusals_count_at_lower() {
r.train()[0].packet.is_some(),
"{what}: the admitted packet is kept on the Train row"
);
let v = r.validation_refused();
let v = r.language("en-US").unwrap().validation_refused;
assert_eq!((v.parse, v.admit, v.lower), (0, 0, 2), "{what}");
assert_eq!((r.train_scored(), r.validation_scored()), (0, 0), "{what}");
let m = r.validation_mean();
assert_eq!((r.language("en-US").unwrap().train_scored, r.language("en-US").unwrap().validation_scored), (0, 0), "{what}");
let m = &r.language("en-US").unwrap().validation_mean;
assert_eq!((m.count, m.chrf), (2, Some(0.0)), "{what}");
}
}
@ -2851,6 +2852,7 @@ fn a_summary_line_says_how_lowerings_were_answered() {
"bag",
LowererReach::BriefedSubagent,
ModelFamily::Unrecorded,
&["en-US".to_string()],
)
.unwrap();
let run = RunStores::open(&root).unwrap();
@ -2873,12 +2875,12 @@ fn lowering_text_at_the_bound_is_scored_and_line_breaks_are_text() {
model.raw_lowering = Some(Ok(serde_json::json!({ "text": text }).to_string()));
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
assert_eq!(
(r.train_scored(), r.validation_scored()),
(r.language("en-US").unwrap().train_scored, r.language("en-US").unwrap().validation_scored),
(1, 2),
"{}",
text.len()
);
assert_eq!(r.validation_refused(), Default::default());
assert_eq!(r.language("en-US").unwrap().validation_refused, Default::default());
}
}
@ -2934,9 +2936,9 @@ fn a_lowering_prompt_that_cannot_be_built_refuses_at_lower_and_keeps_the_packet(
let kept = row.packet.as_deref().expect("the admitted packet's key is kept");
let packets = PacketStore::open(&dao.root.join("round-1/packets.cc")).unwrap();
assert!(packets.get(kept).unwrap().is_some());
let v = r.validation_refused();
let v = r.language("en-US").unwrap().validation_refused;
assert_eq!((v.parse, v.admit, v.lower), (0, 0, 1));
assert_eq!((r.train_scored(), r.validation_scored()), (1, 2));
assert_eq!((r.language("en-US").unwrap().train_scored, r.language("en-US").unwrap().validation_scored), (1, 2));
// Five items encoded, three lowering prompts built and asked.
assert_eq!(r.calls().len(), 5 + 3);
}
@ -3145,7 +3147,7 @@ fn parse_reasons_never_echo_answer() {
a[1].1 = answer;
let mut model = fake(&a);
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
assert_eq!(r.validation_refused().parse, 1);
assert_eq!(r.language("en-US").unwrap().validation_refused.parse, 1);
let line = dao.run.round_line(&r);
assert!(!line.contains(canary) && !line.contains("CANARY"), "{line}");
let bytes = std::fs::read(dao.root.join("run.cc")).unwrap();
@ -3154,7 +3156,7 @@ fn parse_reasons_never_echo_answer() {
}
}
/// A run record as written before `answerer_family` (key 13) existed.
/// A run record as written before `answerer_family` (key 13) and `panel` (key 14) existed.
#[derive(Clone, Debug, schemars::JsonSchema, cultcache_rs::DatabaseEntry)]
#[cultcache(type = "weksa.loop_run")]
struct PreFamilyRun {
@ -3184,8 +3186,8 @@ struct PreFamilyRun {
lowerer_reach: weksa::run::LowererReach,
}
/// A round record as written before `parse_reasons` and `own_family`, and
/// before `admit_rules` had a default.
/// A round record as written with the pooled means (keys 11 to 15) and before
/// `parse_reasons`, `own_family` and `languages`.
#[derive(Clone, Debug, schemars::JsonSchema, cultcache_rs::DatabaseEntry)]
#[cultcache(type = "weksa.loop_round")]
struct LegacyRound {
@ -3226,7 +3228,7 @@ struct LegacyRound {
}
#[test]
fn records_from_before_the_new_fields_decode_with_defaults() {
fn a_run_from_before_does_not_open() {
let mut dao = dao();
let mut model = fake(&good_answers());
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
@ -3243,11 +3245,11 @@ fn records_from_before_the_new_fields_decode_with_defaults() {
chrf_version: r.chrf_version().into(),
calls: r.calls().to_vec(),
train: r.train().to_vec(),
train_mean: r.train_mean().clone(),
validation_mean: r.validation_mean().clone(),
validation_refused: r.validation_refused(),
train_scored: r.train_scored(),
validation_scored: r.validation_scored(),
train_mean: r.language("en-US").unwrap().train_mean.clone(),
validation_mean: r.language("en-US").unwrap().validation_mean.clone(),
validation_refused: r.language("en-US").unwrap().validation_refused,
train_scored: r.language("en-US").unwrap().train_scored,
validation_scored: r.language("en-US").unwrap().validation_scored,
inventory_digest: r.inventory_digest().into(),
cycle: r.cycle(),
};
@ -3266,25 +3268,33 @@ fn records_from_before_the_new_fields_decode_with_defaults() {
lowerer_reach: rec.lowerer_reach,
};
drop(dao.run);
let mut cache = CultCache::new();
cache.register_entry_type::<PreFamilyRun>().unwrap();
cache.register_entry_type::<LegacyRound>().unwrap();
cache
.add_generic_backing_store(SingleFileMessagePackBackingStore::new(root.join("run.cc")))
.unwrap();
cache.pull_all_backing_stores().unwrap();
cache.put("run:dao".to_string(), &run).unwrap();
cache.put("run:dao:round:1".to_string(), &legacy).unwrap();
drop(cache);
let store = || SingleFileMessagePackBackingStore::new(root.join("run.cc"));
// A run record without a panel (no key 14) is a scratch run off git.
{
let mut cache = CultCache::new();
cache.register_entry_type::<PreFamilyRun>().unwrap();
cache.register_entry_type::<LegacyRound>().unwrap();
cache.add_generic_backing_store(store()).unwrap();
cache.pull_all_backing_stores().unwrap();
cache.put("run:dao".to_string(), &run).unwrap();
cache.put("run:dao:round:1".to_string(), &legacy).unwrap();
}
assert_eq!(RunStores::open(&root).err(), Some(RunError::Store));
// A round with the pooled means (keys 11 to 15) and no languages is
// unreadable, under a record that has its panel.
let _ = std::fs::remove_file(root.join("run.cc"));
{
let mut cache = CultCache::new();
cache.register_entry_type::<RunRecord>().unwrap();
cache.register_entry_type::<LegacyRound>().unwrap();
cache.add_generic_backing_store(store()).unwrap();
cache.pull_all_backing_stores().unwrap();
cache.put("run:dao".to_string(), &rec).unwrap();
cache.put("run:dao:round:1".to_string(), &legacy).unwrap();
}
// No round of that run can be read: it stops at its first round or cycle.
let reopened = RunStores::open(&root).unwrap();
assert_eq!(reopened.record().answerer_family, ModelFamily::Unrecorded);
let rounds = reopened.rounds().unwrap();
assert_eq!(rounds.len(), 1);
assert_eq!(rounds[0].n(), 1);
assert!(rounds[0].admit_rules().is_empty());
assert!(rounds[0].parse_reasons().is_empty());
assert_eq!(rounds[0].own_family(), 0);
assert!(reopened.rounds().is_err());
}
#[test]
@ -3310,10 +3320,24 @@ fn the_published_run_and_round_schemas_require_none_of_the_new_fields() {
}
}
/// Answers a recall probe from the corpus: the item's own second half, a
/// fixed wrong text, or junk.
/// What a recall prompt shows of its item: the text after `opening: `.
fn shown_opening(prompt: &str) -> &str {
prompt.rsplit_once("opening: ").expect("a recall prompt has an opening").1
}
/// The item's text after the opening the prompt showed.
fn held_back<'a>(text: &'a str, prompt: &str) -> &'a str {
let opening = shown_opening(prompt);
text.strip_prefix(opening)
.unwrap_or_else(|| panic!("the opening is a prefix of the item"))
.trim()
}
/// Answers a recall probe from the corpus: the item's own held-back half (its
/// text after the opening the prompt showed), a fixed wrong text, or junk.
struct Reciter {
corpus: weksa::store::CorpusStore,
/// Given the item's held-back half.
answer: fn(&str) -> String,
/// An item whose text holds this gets an answer file that is not UTF-8.
unreadable: Option<&'static str>,
@ -3331,14 +3355,12 @@ impl ModelPort for Reciter {
if self.unreadable.is_some_and(|t| item.text.contains(t)) {
return Err(PortError::Io(std::io::ErrorKind::InvalidData));
}
Ok((self.answer)(&item.text))
Ok((self.answer)(held_back(&item.text, prompt.text())))
}
}
fn second_half(text: &str) -> String {
let words: Vec<&str> = text.split_whitespace().collect();
let draft = serde_json::json!({ "text": words[words.len() / 2..].join(" ") });
draft.to_string()
fn second_half(held_back: &str) -> String {
serde_json::json!({ "text": held_back }).to_string()
}
fn reciter(dao: &Dao, answer: fn(&str) -> String) -> Reciter {
@ -3355,20 +3377,26 @@ fn a_recall_probe_asks_for_the_second_half_of_each_live_item_and_never_shows_it(
use weksa::run::RecallOutcome;
let dao = dao();
let mut perfect = reciter(&dao, second_half);
let RecallOutcome::Read(r) = dao.run.recall(&mut perfect).unwrap() else {
let RecallOutcome::Read(reading) = dao.run.recall(&mut perfect).unwrap() else {
panic!("every prompt was answered");
};
let r = reading.languages[0].clone();
// Items 1 (Train) and 2, 3 (Validation) are live; 4 and 5 are not offered.
assert_eq!((r.n, r.answered), (3, 3));
assert!((r.chrf - 100.0).abs() < 1e-9, "{}", r.chrf);
assert_eq!(perfect.seen.len(), 3);
for (text, key) in perfect.seen.iter().zip(["item:daodejing:1", "item:daodejing:2", "item:daodejing:3"]) {
let item = perfect.corpus.get(key).unwrap().unwrap();
let rest = item.text.split_whitespace().collect::<Vec<_>>();
let tail = rest[rest.len() / 2..].join(" ");
let head = rest[..rest.len() / 2].join(" ");
assert!(text.contains(&head), "{key}: the opening is shown");
assert!(!text.contains(&tail), "{key}: the held half is not shown");
// The opening is a proper prefix of the item, cut at a word boundary
// near its middle; the rest is held back.
let head = shown_opening(text);
let tail = held_back(&item.text, text);
assert!(!head.is_empty() && !tail.is_empty(), "{key}");
let squeeze = |t: &str| t.split_whitespace().collect::<String>();
assert_eq!(squeeze(&item.text), squeeze(&format!("{head}{tail}")), "{key}: head and tail are the item");
let n = item.text.len();
assert!((n / 4..=3 * n / 4).contains(&head.len()), "{key}: near the middle");
assert!(!text.contains(tail), "{key}: the held half is not shown");
}
}
@ -3377,15 +3405,17 @@ fn a_recall_of_the_wrong_words_or_no_answer_reads_low_and_a_missing_answer_waits
use weksa::run::RecallOutcome;
let dao = dao();
let mut wrong = reciter(&dao, |_| r#"{"text":"zzz qqq xxx"}"#.to_string());
let RecallOutcome::Read(r) = dao.run.recall(&mut wrong).unwrap() else {
let RecallOutcome::Read(reading) = dao.run.recall(&mut wrong).unwrap() else {
panic!("answered");
};
let r = reading.languages[0].clone();
assert_eq!((r.n, r.answered), (3, 3));
assert!(r.chrf < 10.0, "{}", r.chrf);
let mut junk = reciter(&dao, |_| "not json".to_string());
let RecallOutcome::Read(r) = dao.run.recall(&mut junk).unwrap() else {
let RecallOutcome::Read(reading) = dao.run.recall(&mut junk).unwrap() else {
panic!("answered");
};
let r = reading.languages[0].clone();
assert_eq!((r.n, r.answered, r.chrf), (3, 0, 0.0));
// With no answers on disk the files port waits on all three prompts.
let mut files = AnswerFiles::new(dao.run.recall_dir(), "m".into());
@ -3429,10 +3459,10 @@ fn a_recall_mean_is_over_every_probe_an_unreadable_answer_counts_0_and_a_withdra
let both = |dao: &Dao, unreadable: Option<&'static str>| {
let mut m = reciter(dao, second_half);
m.unreadable = unreadable;
let RecallOutcome::Read(r) = dao.run.recall(&mut m).unwrap() else {
let RecallOutcome::Read(reading) = dao.run.recall(&mut m).unwrap() else {
panic!("answered");
};
r
reading.languages[0].clone()
};
let full = both(&dao, None);
assert_eq!(
@ -3541,16 +3571,17 @@ fn mixed_family_round(family: ModelFamily) -> u32 {
"bag",
LowererReach::ToolLessRelay,
family,
&["en-US".to_string()],
)
.unwrap();
let scorer = weksa::score::Scorer::new(OneBag);
match round(&mut run, &mut NotADraft, &scorer).unwrap() {
RoundOutcome::Recorded(r) => {
// Seven items were offered: the world's own cast item and mix 1 to 6;
// Eight items were offered: the world's two cast items and mix 1 to 6;
// the Test-group and withdrawn items were not.
assert_eq!(
r.parse_reasons().iter().map(|c| c.count).sum::<u32>(),
7,
8,
"offered items"
);
r.own_family()
@ -3697,7 +3728,7 @@ fn read(
outcome: weksa::run::AblationOutcome,
) -> Vec<(weksa::Split, weksa::run::AblationReading)> {
match outcome {
weksa::run::AblationOutcome::Read(r) => r,
weksa::run::AblationOutcome::Read(r) => r.into_iter().map(|(_, split, x)| (split, x)).collect(),
weksa::run::AblationOutcome::Unanswered { prompts } => panic!("unanswered {prompts:?}"),
}
}

View file

@ -238,12 +238,14 @@ fn the_recall_probe_covers_every_live_original_item_and_names_no_work() {
"bag",
weksa::run::LowererReach::BriefedSubagent,
ModelFamily::Unrecorded,
&["en-US".to_string()],
)
.unwrap();
let mut model = Unrelated(vec![]);
let RecallOutcome::Read(r) = run.recall(&mut model).unwrap() else {
let RecallOutcome::Read(reading) = run.recall(&mut model).unwrap() else {
panic!("answered");
};
let r = reading.languages[0].clone();
assert_eq!((r.n, r.answered), (28, 28));
assert!(r.chrf < 25.0, "{}", r.chrf);
assert_eq!(model.0.len(), 28);
@ -251,9 +253,12 @@ fn the_recall_probe_covers_every_live_original_item_and_names_no_work() {
let all = CorpusStore::open(&c).unwrap().all().unwrap();
for item in &all {
let live = SplitScheme::V1.live(item);
// A recall prompt shows the first half of the words.
let words: Vec<&str> = item.text.split_whitespace().collect();
let opening = words[..words.len() / 2].join(" ");
// A recall prompt shows the first half of the item, cut at a word boundary.
let mut cut = item.text.len() / 4;
while !item.text.is_char_boundary(cut) {
cut -= 1;
}
let opening = item.text[..cut].trim_end().to_string();
let shown = model.0.iter().any(|t| t.contains(&opening));
assert_eq!(
shown,
@ -309,6 +314,7 @@ fn original_round(family: ModelFamily) -> (weksa::run::RunStores, weksa::run::Ro
"bag",
LowererReach::ToolLessRelay,
family,
&["en-US".to_string()],
)
.unwrap();
let scorer = weksa::score::Scorer::new(Bag);
@ -416,9 +422,10 @@ fn recall_probes_only_live_train_and_validation_items() {
drop(store);
let run = run_over(dir.path());
let mut m = Record(vec![]);
let RecallOutcome::Read(r) = run.recall(&mut m).unwrap() else {
let RecallOutcome::Read(reading) = run.recall(&mut m).unwrap() else {
panic!("answered");
};
let r = reading.languages[0].clone();
assert_eq!(r.n, 2);
let shown = |mark: &str| m.0.iter().any(|t| t.contains(mark));
assert!(shown("trainmark") && shown("validmark"));
@ -464,6 +471,7 @@ fn no_test_item_is_offered_to_a_round_or_recall_or_recorded() {
"bag",
LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&["en-US".to_string()],
)
.unwrap();
let items = CorpusStore::open(&p("corpus.cc")).unwrap().all().unwrap();
@ -499,9 +507,10 @@ fn no_test_item_is_offered_to_a_round_or_recall_or_recorded() {
assert!(r.train().iter().all(|row| row.item != item.id));
}
let mut recall = Record(vec![]);
let weksa::run::RecallOutcome::Read(rr) = run.recall(&mut recall).unwrap() else {
let weksa::run::RecallOutcome::Read(reading) = run.recall(&mut recall).unwrap() else {
panic!("answered");
};
let rr = reading.languages[0].clone();
assert_eq!((rr.n, rr.answered), (28, 28));
for item in &held {
assert!(!leaks(&recall.0, item), "recall prompt held {}", item.id);