From 3e4b035be6926e3c4e0a0e6d4bdd90e3284eebc6 Mon Sep 17 00:00:00 2001 From: Weksa Date: Thu, 12 Nov 2026 12:29:23 +0000 Subject: [PATCH] Loop tests follow the panel: en-US panel in every fixture run, per-language reads, recall at word boundaries Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/context.rs | 11 ++ crates/weksa/tests/cli.rs | 34 +++- crates/weksa/tests/common/mod.rs | 23 +++ crates/weksa/tests/context.rs | 37 ++-- crates/weksa/tests/golf_loop.rs | 263 ++++++++++++++------------ crates/weksa/tests/original_corpus.rs | 21 +- 6 files changed, 245 insertions(+), 144 deletions(-) diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index e225b84..7ae7cc8 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -637,6 +637,17 @@ mod tests { ); } + #[test] + fn a_mint_prompt_over_no_train_items_is_the_brief_the_feedback_and_the_listing() { + let none = TrainItems { items: vec![] }; + let prompt = Prompt::mint(&none, &Inventory::default(), 0, &EditFeedback::none()); + assert_eq!( + prompt.text(), + format!("{MINT_INSTRUCTIONS}\n\nlast edit: none; validation composite at head: none\n\ninventory rev:0") + ); + assert!(prompt.item_keys().is_empty()); + } + #[test] fn checkpoint_offer_gate() { // A fresh checkpoint per call: this test is the gate, not the once rule. diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index f5562cd..93ca5d4 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -47,7 +47,7 @@ fn out(o: &Output) -> String { String::from_utf8_lossy(&o.stdout).into_owned() + &String::from_utf8_lossy(&o.stderr) } -const ADD: [&str; 14] = [ +const ADD: [&str; 16] = [ "corpus", "add", "--slug", @@ -62,6 +62,8 @@ const ADD: [&str; 14] = [ "2026-10-29", "--source", "cast-authored", + "--language", + "en-US", ]; #[test] @@ -149,6 +151,8 @@ fn model_written_and_public_domain_sources_name_their_provenance() { let base = [ "corpus", "add", + "--language", + "en-US", "--slug", "x", "--speaker", @@ -375,6 +379,8 @@ fn entity_put_and_name_spans() { &[ "corpus", "add", + "--language", + "en-US", "--slug", "kay", "--text", @@ -547,6 +553,8 @@ fn add_with(dir: &std::path::Path, source: &str, extra: &[&str]) -> Output { let mut args = vec![ "corpus", "add", + "--language", + "en-US", "--slug", "nibu", "--text", @@ -903,6 +911,8 @@ fn proof_packet_lowers_and_the_lowering_scores_against_items() { &[ "corpus", "add", + "--language", + "en-US", "--slug", "nibu", "--text", @@ -1062,7 +1072,7 @@ fn cli_errors_never_echo_identifiers() { let p = d.path(); cast_entities(p, &["nibu"]); let add = |extra: &[&str]| -> Vec { - ["corpus", "add", "--admitted-on", "2026-10-29"] + ["corpus", "add", "--language", "en-US", "--admitted-on", "2026-10-29"] .into_iter() .chain(extra.iter().copied()) .map(String::from) @@ -1344,6 +1354,8 @@ fn loop_open_and_unanswered_round() { "smoke", "--answerer", "fake-model", + "--panel", + "en-US", "--lowerer-reach", "tool-less-relay", "--answerer-family", @@ -1397,7 +1409,7 @@ fn loop_open_and_unanswered_round() { assert!(second.status.success(), "{}", out(&second)); assert_eq!( out(&second), - "round 1 rev 3 train n=1 scored=0 chrf=0.00 meaning=0.0000 refused=1 validation n=2 scored=0 chrf=0.00 meaning=0.0000 refused parse=2 admit=0 lower=0 parse-reasons=syntaxx3 lowering=tool-less-relay\n" + "round 1 rev 3 en-US train n=1 scored=0 chrf=0.00 meaning=0.0000 refused=1 validation n=2 scored=0 chrf=0.00 meaning=0.0000 refused parse=2 admit=0 lower=0 parse-reasons=syntaxx3 lowering=tool-less-relay\n" ); // The global store flags are not read by loop round: a drifted store is refused. @@ -1424,6 +1436,8 @@ fn open_dao_run(d: &std::path::Path) { "smoke", "--answerer", "fake-model", + "--panel", + "en-US", "--lowerer-reach", "tool-less-relay", ], @@ -1441,6 +1455,8 @@ fn pinned_family(d: &std::path::Path, root: &str, flag: &[&str]) -> weksa::corpu "fam", "--answerer", "fake-model", + "--panel", + "en-US", "--lowerer-reach", "tool-less-relay", ]; @@ -1526,7 +1542,7 @@ fn loop_cycle_asks_then_decides() { assert!(done.status.success(), "{}", out(&done)); assert_eq!( out(&done), - "cycle 1 rev 3->none base=0.0000 candidate=none floor=0.0000 threshold=0.0050 rejected edit-parse lowering=tool-less-relay\n" + "cycle 1 rev 3->none en-US base=0.0000 candidate=none floor=0.0000 threshold=0.0050 rejected edit-parse lowering=tool-less-relay\n" ); // The next cycle asks afresh in its own directory. @@ -1565,6 +1581,8 @@ fn loop_errors_echo_no_input() { "CANARYLABEL", "--answerer", "m", + "--panel", + "en-US", "--lowerer-reach", "tool-less-relay", ], @@ -1576,6 +1594,8 @@ fn loop_errors_echo_no_input() { "ok", "--answerer", "CANARY\n", + "--panel", + "en-US", "--lowerer-reach", "tool-less-relay", ], @@ -1597,6 +1617,8 @@ fn loop_errors_echo_no_input() { "ok", "--answerer", "m", + "--panel", + "en-US", "--lowerer-reach", "tool-less-relay", ]) @@ -1610,7 +1632,7 @@ fn loop_errors_echo_no_input() { fn loop_open_requires_the_lowerer_reach_and_pins_it() { let d = tempfile::tempdir().unwrap(); dao_stores(d.path()); - let base = ["loop", "open", "canaryroot", "--label", "smoke", "--answerer", "m"]; + let base = ["loop", "open", "canaryroot", "--label", "smoke", "--answerer", "m", "--panel", "en-US"]; // No default: the record says how lowerings are answered, or there is no run. let o = weksa(d.path(), &base); assert!(!o.status.success()); @@ -1698,7 +1720,7 @@ fn loop_ablate_waits_then_prints_one_line_per_split() { let text = out(&second); assert_eq!( text, - "train ablation n=1 unusable=0 replay chrf=100.00 meaning=1.000 | drop n=1 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000 | permute n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none\nvalidation ablation n=2 unusable=0 replay chrf=100.00 meaning=1.000 | drop n=2 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000 | permute n=2 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000\n" + "en-US train ablation n=1 unusable=0 replay chrf=100.00 meaning=1.000 | drop n=1 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000 | permute n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none\nen-US validation ablation n=2 unusable=0 replay chrf=100.00 meaning=1.000 | drop n=2 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000 | permute n=2 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000\n" ); for leaked in ["item:", "packet:", "highest excellence", "canaryroot"] { assert!(!text.contains(leaked), "{text}"); diff --git a/crates/weksa/tests/common/mod.rs b/crates/weksa/tests/common/mod.rs index 73c8456..b7b7c7e 100644 --- a/crates/weksa/tests/common/mod.rs +++ b/crates/weksa/tests/common/mod.rs @@ -116,12 +116,30 @@ pub fn entities_in(dir: &std::path::Path) -> EntityStore { /// A run over the stores `world()` and `corpus()` keep in `dir`: `c.cc`, /// `i.cc` and `entities.cc`. Read at the call, so create it after the items. +/// The run's en-US panel needs a live Train and a live Validation item: a +/// corpus short of one gets `item:panel:` added, and no other item. pub fn run_over(dir: &std::path::Path) -> weksa::run::RunStores { for name in ["c.cc", "i.cc", "entities.cc"] { if !dir.join(name).exists() { std::fs::write(dir.join(name), [0x90]).unwrap(); } } + let entities = entities_in(dir); + let mut corpus = CorpusStore::open(&dir.join("c.cc")).unwrap(); + for (n, split) in [(1, weksa::Split::Train), (2, weksa::Split::Validation)] { + let held = corpus + .all() + .unwrap() + .iter() + .any(|i| SplitScheme::V1.live(i) == Some(split)); + if !held { + let group = group_in_bucket(if n == 1 { 10 } else { 50 }); + let key = format!("item:panel:{n}"); + let text = format!("Panel fixture line {n}."); + corpus.add(item_in(&key, &text, &group, cast()), &entities).unwrap(); + } + } + drop(corpus); weksa::run::RunStores::create( &dir.join("run"), "t", @@ -134,6 +152,7 @@ pub fn run_over(dir: &std::path::Path) -> weksa::run::RunStores { "bag", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, + &["en-US".to_string()], ) .unwrap() } @@ -219,6 +238,10 @@ pub fn world() -> World { end: 4, }]; corpus.add(first, &entities).unwrap(); + // A run's panel needs a live Validation item as well as a Train one. + let validation = group_in_bucket(50); + let second = item_in("item:panel:1", "Nibu says watch the ladder.", &validation, cast()); + corpus.add(second, &entities).unwrap(); let mut inventory = InventoryStore::open(&dir.path().join("i.cc")).unwrap(); let (rev1, entries) = seed(); inventory.admit_revision(rev1, entries).unwrap(); diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs index b908dcb..0103149 100644 --- a/crates/weksa/tests/context.rs +++ b/crates/weksa/tests/context.rs @@ -110,16 +110,6 @@ fn train_items_are_in_key_order() { ["item:a:1", "item:m:1", "item:z:1"] ); assert_eq!((t.len(), t.is_empty()), (3, false)); - let dir = tempfile::tempdir().unwrap(); - let empty = run_over(dir.path()); - let none = empty.train().unwrap(); - assert_eq!((none.len(), none.is_empty()), (0, true)); - let (inv, fb) = (empty.inventory().inventory().unwrap(), EditFeedback::none()); - assert_eq!( - Prompt::mint(&none, &inv, 0, &fb).text(), - format!("{MINT}\n\nlast edit: none; validation composite at head: none\n\ninventory rev:0") - ); - assert!(Prompt::mint(&none, &inv, 0, &fb).item_keys().is_empty()); } #[test] @@ -606,13 +596,28 @@ mod lowering { } } + /// The language line carries the tag the caller passes; no instruction + /// names a language. + #[test] + fn lower_prompt_names_its_language() { + let (view, inv, entities, item) = dao(2); + for tag in ["en-US", "ja-JP", "pt-BR"] { + let prompt = Prompt::lower(&view, &item, tag, &inv, &entities).unwrap(); + let want = format!("{LOWER}\n\nlanguage: {tag}\n\npacket: "); + assert!(prompt.text().starts_with(&want), "{tag}"); + } + for brief in [LOWER, weksa::context::MINT_INSTRUCTIONS] { + assert!(!brief.contains("English") && !brief.contains("en-US")); + } + } + #[test] fn lower_prompt_text_is_pinned() { let (view, inv, entities, item) = dao(2); - let prompt = Prompt::lower(&view, &item, &inv, &entities).unwrap(); + let prompt = Prompt::lower(&view, &item, "en-US", &inv, &entities).unwrap(); let (head, listing) = prompt.text().split_once("inventory rev:3\n").unwrap(); let expected = format!( - "{LOWER}\n\npacket: {}\n\nentities: {{}}\n\n", + "{LOWER}\n\nlanguage: en-US\n\npacket: {}\n\nentities: {{}}\n\n", concat!( r#"{"context":{"setting":null,"medium":null,"participants":[],"audience":[]},"#, r#""discourse":{"speech_act":"speech_act:1","addressee":null,"addressee_consent":null,"features":[]},"#, @@ -628,7 +633,7 @@ mod lowering { assert!(listing.contains("\nfeature:18: ")); // The brief says what the spec says it must. for needed in [ - "en-US", + "the language on the language line", "all of its meaning and nothing it does not carry", "a grade is permille, 0 to 1000", "felt affect and restraint", @@ -671,7 +676,7 @@ mod lowering { let mut packet = packets().remove(2); packet.stance = stance.clone(); let view = admitted(packet, &store, "daodejing/corpus.cc", &entities); - let prompt = Prompt::lower(&view, &item, &inv, &entities).unwrap(); + let prompt = Prompt::lower(&view, &item, "en-US", &inv, &entities).unwrap(); (stance, prompt) }; let three = [prompt_of(own), prompt_of(angry), prompt_of(Stance::default())]; @@ -708,7 +713,7 @@ mod lowering { ("sa-Latn", &["Nibuwa"]), ("en-US", &["Nibu", "Nibu's"]), ])]); - let prompt = Prompt::lower(&view, &item, &cast.inventory().unwrap(), &held).unwrap(); + let prompt = Prompt::lower(&view, &item, "en-US", &cast.inventory().unwrap(), &held).unwrap(); assert!( prompt .text() @@ -738,7 +743,7 @@ mod lowering { ), (Held(vec![]), LowerError::UnknownEntity { entity: id }), ] { - assert_eq!(Prompt::lower(&view, &item, &inv, &held).err(), Some(want)); + assert_eq!(Prompt::lower(&view, &item, "en-US", &inv, &held).err(), Some(want)); } } } diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 4a1c7ee..d476dac 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -51,6 +51,7 @@ fn a_run_pins_its_stores_and_refuses_drift() { "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded, + &["en-US".to_string()], ) }; @@ -154,6 +155,7 @@ fn a_run_works_on_its_own_copy_of_the_inventory() { "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded, + &["en-US".to_string()], ) .unwrap(); @@ -231,6 +233,7 @@ fn create_refuses_a_root_that_holds_an_inventory_and_never_truncates_it() { "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded, + &["en-US".to_string()], ) }; @@ -278,6 +281,7 @@ fn a_create_that_fails_leaves_no_inventory_and_the_retry_succeeds() { "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded, + &["en-US".to_string()], ) }; // A directory opens for reading and fails mid-copy, after the destination exists. @@ -398,6 +402,7 @@ fn a_store_changed_between_its_digest_and_its_open_is_drift() { "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded, + &["en-US".to_string()], || more(&mut w) ) .err(), @@ -415,6 +420,7 @@ fn a_store_changed_between_its_digest_and_its_open_is_drift() { "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded, + &["en-US".to_string()], ) .unwrap(); } @@ -447,6 +453,7 @@ fn window( "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded, + &["en-US".to_string()], || change(&mut w, &root), ) .err(); @@ -866,6 +873,7 @@ fn dao_over(edit: impl FnOnce(&Path, &Path)) -> Dao { "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded, + &["en-US".to_string()], ) .unwrap(); Dao { @@ -923,15 +931,15 @@ fn round_over_the_dao_stores_scores_train_and_validation() { }; assert_eq!(two_dp(chrf), 44.18); assert!(meaning.is_some()); - assert_eq!((r.train_mean().count, r.train_mean().chrf), (1, Some(chrf))); + assert_eq!((r.language("en-US").unwrap().train_mean.count, r.language("en-US").unwrap().train_mean.chrf), (1, Some(chrf))); // Validation: means and refusal counts only. - let v = r.validation_mean(); + let v = &r.language("en-US").unwrap().validation_mean; assert_eq!(v.count, 2); assert!((v.chrf.unwrap() - 78.56).abs() < 0.01, "{:?}", v.chrf); assert!(v.meaning.is_some()); - assert_eq!(r.validation_refused(), Default::default()); - assert_eq!((r.train_scored(), r.validation_scored()), (1, 2)); + assert_eq!(r.language("en-US").unwrap().validation_refused, Default::default()); + assert_eq!((r.language("en-US").unwrap().train_scored, r.language("en-US").unwrap().validation_scored), (1, 2)); // The record carries the run's pins. assert_eq!(r.prompt_digest(), pinned.prompt_digest); @@ -1041,7 +1049,7 @@ fn lowering_prompts(dao: &Dao, inventory: &InventoryStore) -> Vec<(String, Promp let view = store .admit(packet, inventory, &corpus, dao.run.entities()) .unwrap(); - let prompt = Prompt::lower(&view, &key, &inv, dao.run.entities()).unwrap(); + let prompt = Prompt::lower(&view, &key, "en-US", &inv, dao.run.entities()).unwrap(); (key.to_string(), prompt) }) .collect() @@ -1139,7 +1147,7 @@ fn unanswered_round_writes_nothing() { let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()); assert_eq!((r.n(), r.calls().len()), (1, 6)); - assert_eq!(two_dp(r.train_mean().chrf.unwrap()), 44.18); + assert_eq!(two_dp(r.language("en-US").unwrap().train_mean.chrf.unwrap()), 44.18); // Round 2 asks again, encode first. let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); @@ -1261,15 +1269,15 @@ fn refusal_stages_are_counted() { assert_eq!(r.train()[0].outcome, Outcome::Refused(Stage::Parse)); assert_eq!(r.train()[0].packet, None); // A refused item is in the mean as 0: n counts every item, scored the real ones. - let m = r.train_mean(); + let m = &r.language("en-US").unwrap().train_mean; assert_eq!((m.count, m.chrf, m.meaning), (1, Some(0.0), Some(0.0))); - assert_eq!((r.train_scored(), r.validation_scored()), (0, 1)); - assert_eq!(r.validation_refused().parse, 0); + assert_eq!((r.language("en-US").unwrap().train_scored, r.language("en-US").unwrap().validation_scored), (0, 1)); + assert_eq!(r.language("en-US").unwrap().validation_refused.parse, 0); assert_eq!( - (r.validation_refused().admit, r.validation_refused().lower), + (r.language("en-US").unwrap().validation_refused.admit, r.language("en-US").unwrap().validation_refused.lower), (1, 0) ); - let m = r.validation_mean(); + let m = &r.language("en-US").unwrap().validation_mean; assert_eq!(m.count, 2); assert!(m.chrf.unwrap() > 0.0, "the relation item lowered and scored"); @@ -1291,29 +1299,29 @@ fn refusal_stages_are_counted() { None, "a refused packet is not stored" ); - let v = b_only.validation_refused(); + let v = b_only.language("en-US").unwrap().validation_refused; assert_eq!((v.parse, v.admit, v.lower), (1, 0, 0)); assert_eq!( - b_only.validation_mean().count, + b_only.language("en-US").unwrap().validation_mean.count, 2, "every Validation item counts" ); - assert_eq!(b_only.validation_scored(), 1); - let b_half = b_only.validation_mean().chrf.unwrap(); + assert_eq!(b_only.language("en-US").unwrap().validation_scored, 1); + let b_half = b_only.language("en-US").unwrap().validation_mean.chrf.unwrap(); assert!( (b_half - 74.99 / 2.0).abs() < 0.01, "(74.99 + 0) / 2 = {b_half}" ); let a_only = run([good_answers()[0].1.clone(), good_answers()[1].1.clone(), junk]); - assert_eq!(a_only.validation_refused().parse, 1); - let a_half = a_only.validation_mean().chrf.unwrap(); + assert_eq!(a_only.language("en-US").unwrap().validation_refused.parse, 1); + let a_half = a_only.language("en-US").unwrap().validation_mean.chrf.unwrap(); assert!( (a_half - 82.12 / 2.0).abs() < 0.01, "(82.12 + 0) / 2 = {a_half}" ); // The halves add to the full mean: (a + 0) / 2 + (0 + b) / 2 = (a + b) / 2. - let whole = full.validation_mean().chrf.unwrap(); + let whole = full.language("en-US").unwrap().validation_mean.chrf.unwrap(); assert!( (a_half + b_half - whole).abs() < 1e-9, "{a_half} + {b_half} vs {whole}" @@ -1322,7 +1330,7 @@ fn refusal_stages_are_counted() { b_half < whole && a_half < whole, "refusing never raises the mean" ); - assert_eq!(full.validation_scored(), 2); + assert_eq!(full.language("en-US").unwrap().validation_scored, 2); } #[test] @@ -1356,10 +1364,10 @@ fn non_utf8_encode_answer_is_a_parse_refusal() { let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()); assert_eq!(r.calls().len(), 5); // Item 3 is Validation: its refusal is a Parse refusal and the round stands. - let v = r.validation_refused(); + let v = r.language("en-US").unwrap().validation_refused; assert_eq!((v.parse, v.admit, v.lower), (1, 0, 0)); - assert_eq!((r.train_scored(), r.validation_scored()), (1, 1)); - assert_eq!(r.validation_mean().count, 2); + assert_eq!((r.language("en-US").unwrap().train_scored, r.language("en-US").unwrap().validation_scored), (1, 1)); + assert_eq!(r.language("en-US").unwrap().validation_mean.count, 2); assert_eq!(dao.run.rounds().unwrap(), [r]); } @@ -1402,7 +1410,7 @@ fn a_failed_round_leaves_nothing_a_rerun_collides_with() { "{:?}", r.train()[0] ); - assert_eq!(r.validation_refused(), Default::default()); + assert_eq!(r.language("en-US").unwrap().validation_refused, Default::default()); let stored = PacketStore::open(&dao.root.join("round-1/packets.cc")) .unwrap() .all() @@ -1441,7 +1449,7 @@ fn a_stale_packet_store_in_an_unrecorded_round_dir_is_discarded_at_start() { "the revised draft is not refused as a changed packet: {:?}", r.train()[0] ); - assert_eq!(r.validation_refused(), Default::default()); + assert_eq!(r.language("en-US").unwrap().validation_refused, Default::default()); let stored = PacketStore::open(&dir.join("packets.cc")) .unwrap() .all() @@ -1467,11 +1475,11 @@ fn the_round_line_prints_each_split_under_its_own_name() { let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); let line = dao.run.round_line(&r); // Train is item 1 alone, Validation items 2 and 3; the numbers differ. - let (t, v) = (r.train_mean(), r.validation_mean()); + let (t, v) = (&r.language("en-US").unwrap().train_mean, &r.language("en-US").unwrap().validation_mean); assert_eq!( line, format!( - "round 1 rev 3 train n=1 scored=1 chrf={:.2} meaning={:.4} refused=0 validation n=2 scored=2 chrf={:.2} meaning={:.4} refused parse=0 admit=0 lower=0 lowering=tool-less-relay", + "round 1 rev 3 en-US train n=1 scored=1 chrf={:.2} meaning={:.4} refused=0 validation n=2 scored=2 chrf={:.2} meaning={:.4} refused parse=0 admit=0 lower=0 lowering=tool-less-relay", t.chrf.unwrap(), t.meaning.unwrap(), v.chrf.unwrap(), @@ -1522,6 +1530,7 @@ fn errors_echo_no_input() { "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded, + &["en-US".to_string()], ) .err() .unwrap(); @@ -1534,6 +1543,7 @@ fn errors_echo_no_input() { "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded, + &["en-US".to_string()], ) .err() .unwrap(); @@ -1546,6 +1556,7 @@ fn errors_echo_no_input() { "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded, + &["en-US".to_string()], ) .err() .unwrap(); @@ -1583,6 +1594,7 @@ fn errors_echo_no_input() { "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded, + &["en-US".to_string()], ) .unwrap(); let mut model = cycle_fake(&format!("{{\"{canary}\":1}}")); @@ -1717,13 +1729,13 @@ fn head_rev(root: &Path) -> u32 { } fn validation(r: &RoundRecord) -> Reading { - Reading::of(r.validation_mean()).unwrap() + Reading::of(&r.language("en-US").unwrap().validation_mean).unwrap() } /// The feedback line the editor is shown for a head round, built from outside. fn feedback_line(last: &str, head: &RoundRecord) -> String { let rounded = (validation(head).composite / LADDER_STEP).round() * LADDER_STEP; - format!("last edit: {last}; validation composite at head: {rounded:.3}") + format!("last edit: {last}; validation composite at head: en-US {rounded:.3}") } fn mint_texts(model: &FakeModel) -> Vec<&str> { @@ -1751,19 +1763,19 @@ fn cycle_accepts_a_gain_past_the_floor() { ); assert_eq!((d.base_rev(), d.candidate_rev()), (3, Some(4))); assert_eq!(d.verdict(), Verdict::Accepted); - assert_eq!(d.threshold(), LADDER_STEP); + assert_eq!(d.panel()[0].threshold, LADDER_STEP); assert_eq!( - *d.floor(), + d.panel()[0].floor, Reading { chrf: 0.0, meaning: 0.0, composite: 0.0 } ); - assert_eq!(*d.base(), validation(&rounds[0])); + assert_eq!(d.panel()[0].base, validation(&rounds[0])); let record = dao.run.rounds().unwrap().remove(2); - assert_eq!(d.candidate(), Some(&validation(&record))); - assert!(d.candidate().unwrap().composite - d.base().composite > LADDER_STEP); + assert_eq!(d.panel()[0].candidate, Some(validation(&record))); + assert!(d.panel()[0].candidate.unwrap().composite - d.panel()[0].base.composite > LADDER_STEP); // The candidate round was scored under the candidate file, as cycle 1's. let file = dao.root.join("cycle-1/inventory.cc"); @@ -1787,11 +1799,11 @@ fn cycle_accepts_a_gain_past_the_floor() { assert_eq!(d.edit().prompt, model.seen[0].0); // The line, with the readings printed to four places. - let (b, c) = (d.base().composite, d.candidate().unwrap().composite); + let (b, c) = (d.panel()[0].base.composite, d.panel()[0].candidate.unwrap().composite); assert_eq!( dao.run.cycle_line(&d), format!( - "cycle 1 rev 3->4 base={b:.4} candidate={c:.4} floor=0.0000 threshold=0.0050 accepted lowering=tool-less-relay" + "cycle 1 rev 3->4 en-US base={b:.4} candidate={c:.4} floor=0.0000 threshold=0.0050 accepted lowering=tool-less-relay" ) ); @@ -1822,7 +1834,7 @@ fn cycle_accepts_a_gain_past_the_floor() { assert_eq!(d2.candidate_rev(), Some(5)); assert_eq!(d2.candidate_round(), Some(5)); assert_eq!(d2.verdict(), Verdict::Rejected(Why::NoGain)); - assert_eq!(*d2.floor(), *d.floor(), "the floor is measured once"); + assert_eq!(d2.panel()[0].floor, d.panel()[0].floor, "the floor is measured once"); assert_eq!(head_rev(&dao.root), 4, "a rejected edit moves nothing"); assert!( mint_texts(&second)[0].contains(&feedback_line("accepted", &dao.run.rounds().unwrap()[2])), @@ -1846,12 +1858,12 @@ fn cycle_rejects_no_gain_and_keeps_the_head() { let d = run_cycle(&mut dao, &mut model); assert_eq!(d.verdict(), Verdict::Rejected(Why::NoGain)); assert_eq!((d.candidate_rev(), d.candidate_round()), (Some(4), Some(3))); - assert_eq!(d.candidate(), Some(d.base())); - let c = d.base().composite; + assert_eq!(d.panel()[0].candidate, Some(d.panel()[0].base)); + let c = d.panel()[0].base.composite; assert_eq!( dao.run.cycle_line(&d), format!( - "cycle 1 rev 3->4 base={c:.4} candidate={c:.4} floor=0.0000 threshold=0.0050 rejected no-gain lowering=tool-less-relay" + "cycle 1 rev 3->4 en-US base={c:.4} candidate={c:.4} floor=0.0000 threshold=0.0050 rejected no-gain lowering=tool-less-relay" ) ); assert_eq!(head_rev(&dao.root), 3); @@ -1911,10 +1923,10 @@ fn distinct_replicates_set_the_floor_and_a_gain_inside_it_is_no_gain() { // (round 1) is the floor, which is past a step and inside the noise. let mut model = cycle_fake(ONE_CONCEPT); let d = run_cycle(&mut dao, &mut model); - assert_eq!(*d.floor(), want); - assert_eq!(d.threshold(), want.composite); - assert_eq!(d.candidate(), Some(&b)); - assert!(d.candidate().unwrap().composite - d.base().composite > LADDER_STEP); + assert_eq!(d.panel()[0].floor, want); + assert_eq!(d.panel()[0].threshold, want.composite); + assert_eq!(d.panel()[0].candidate, Some(b)); + assert!(d.panel()[0].candidate.unwrap().composite - d.panel()[0].base.composite > LADDER_STEP); assert_eq!(d.verdict(), Verdict::Rejected(Why::NoGain)); assert_eq!(head_rev(&dao.root), 3); } @@ -2101,7 +2113,7 @@ fn unanswered_cycle_writes_nothing() { let mint = std::fs::read_to_string(&mint_file).unwrap(); assert_eq!(format!("{:016x}", fnv1a64(mint.as_bytes())), asked[0]); assert!(mint.starts_with(MINT_INSTRUCTIONS)); - assert!(mint.contains("\n\nlast edit: none; validation composite at head: 0.435\n\n")); + assert!(mint.contains("\n\nlast edit: none; validation composite at head: en-US 0.435\n\n")); assert!(mint.contains("\n\ninventory rev:3\n")); assert_eq!(mint.matches("[item:").count(), 1, "one Train block"); assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 1); @@ -2349,7 +2361,7 @@ fn edit_refusals_are_decisions() { let d = run_cycle(&mut dao, &mut model); assert_eq!(d.verdict(), Verdict::Rejected(why), "{name}"); assert_eq!( - (d.candidate_rev(), d.candidate_round(), d.candidate()), + (d.candidate_rev(), d.candidate_round(), d.panel()[0].candidate), (None, None, None), "{name}" ); @@ -2377,8 +2389,8 @@ fn edit_refusals_are_decisions() { assert_eq!( dao.run.cycle_line(&d), format!( - "cycle 1 rev 3->none base={:.4} candidate=none floor=0.0000 threshold=0.0050 rejected {tag} lowering=tool-less-relay", - d.base().composite + "cycle 1 rev 3->none en-US base={:.4} candidate=none floor=0.0000 threshold=0.0050 rejected {tag} lowering=tool-less-relay", + d.panel()[0].base.composite ), "{name}" ); @@ -2419,20 +2431,8 @@ fn cycle_needs_a_floor() { "the run has no noise floor: record two rounds first" ); - // Rounds whose Validation items are all withdrawn have no reading. - let mut dao = dao_over(|corpus, _| { - let mut store = weksa::CorpusStore::open(corpus).unwrap(); - for n in [2, 3] { - store.withdraw(&format!("item:daodejing:{n}")).unwrap(); - } - }); - for _ in 0..2 { - let mut m = fake(&[(1, json(&draft_of(1)))]); - let r = recorded(round(&mut dao.run, &mut m, &Scorer::new(Bag)).unwrap()); - assert_eq!(r.validation_mean().count, 0); - } - let mut model = cycle_fake(ONE_CONCEPT); - assert_eq!(none(&mut dao, &mut model), Some(RunError::NoValidation)); + // A panel language with no live Validation item never reaches a round: + // `create` refuses it (open_refuses_a_bad_panel). assert_eq!( RunError::NoValidation.to_string(), "a reading has no validation item" @@ -2515,6 +2515,7 @@ fn a_changed_head_is_drift() { "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded, + &["en-US".to_string()], ) .unwrap(); assert_eq!( @@ -2772,10 +2773,10 @@ fn lowering_refusals_count_at_lower() { r.train()[0].packet.is_some(), "{what}: the admitted packet is kept on the Train row" ); - let v = r.validation_refused(); + let v = r.language("en-US").unwrap().validation_refused; assert_eq!((v.parse, v.admit, v.lower), (0, 0, 2), "{what}"); - assert_eq!((r.train_scored(), r.validation_scored()), (0, 0), "{what}"); - let m = r.validation_mean(); + assert_eq!((r.language("en-US").unwrap().train_scored, r.language("en-US").unwrap().validation_scored), (0, 0), "{what}"); + let m = &r.language("en-US").unwrap().validation_mean; assert_eq!((m.count, m.chrf), (2, Some(0.0)), "{what}"); } } @@ -2851,6 +2852,7 @@ fn a_summary_line_says_how_lowerings_were_answered() { "bag", LowererReach::BriefedSubagent, ModelFamily::Unrecorded, + &["en-US".to_string()], ) .unwrap(); let run = RunStores::open(&root).unwrap(); @@ -2873,12 +2875,12 @@ fn lowering_text_at_the_bound_is_scored_and_line_breaks_are_text() { model.raw_lowering = Some(Ok(serde_json::json!({ "text": text }).to_string())); let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); assert_eq!( - (r.train_scored(), r.validation_scored()), + (r.language("en-US").unwrap().train_scored, r.language("en-US").unwrap().validation_scored), (1, 2), "{}", text.len() ); - assert_eq!(r.validation_refused(), Default::default()); + assert_eq!(r.language("en-US").unwrap().validation_refused, Default::default()); } } @@ -2934,9 +2936,9 @@ fn a_lowering_prompt_that_cannot_be_built_refuses_at_lower_and_keeps_the_packet( let kept = row.packet.as_deref().expect("the admitted packet's key is kept"); let packets = PacketStore::open(&dao.root.join("round-1/packets.cc")).unwrap(); assert!(packets.get(kept).unwrap().is_some()); - let v = r.validation_refused(); + let v = r.language("en-US").unwrap().validation_refused; assert_eq!((v.parse, v.admit, v.lower), (0, 0, 1)); - assert_eq!((r.train_scored(), r.validation_scored()), (1, 2)); + assert_eq!((r.language("en-US").unwrap().train_scored, r.language("en-US").unwrap().validation_scored), (1, 2)); // Five items encoded, three lowering prompts built and asked. assert_eq!(r.calls().len(), 5 + 3); } @@ -3145,7 +3147,7 @@ fn parse_reasons_never_echo_answer() { a[1].1 = answer; let mut model = fake(&a); let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); - assert_eq!(r.validation_refused().parse, 1); + assert_eq!(r.language("en-US").unwrap().validation_refused.parse, 1); let line = dao.run.round_line(&r); assert!(!line.contains(canary) && !line.contains("CANARY"), "{line}"); let bytes = std::fs::read(dao.root.join("run.cc")).unwrap(); @@ -3154,7 +3156,7 @@ fn parse_reasons_never_echo_answer() { } } -/// A run record as written before `answerer_family` (key 13) existed. +/// A run record as written before `answerer_family` (key 13) and `panel` (key 14) existed. #[derive(Clone, Debug, schemars::JsonSchema, cultcache_rs::DatabaseEntry)] #[cultcache(type = "weksa.loop_run")] struct PreFamilyRun { @@ -3184,8 +3186,8 @@ struct PreFamilyRun { lowerer_reach: weksa::run::LowererReach, } -/// A round record as written before `parse_reasons` and `own_family`, and -/// before `admit_rules` had a default. +/// A round record as written with the pooled means (keys 11 to 15) and before +/// `parse_reasons`, `own_family` and `languages`. #[derive(Clone, Debug, schemars::JsonSchema, cultcache_rs::DatabaseEntry)] #[cultcache(type = "weksa.loop_round")] struct LegacyRound { @@ -3226,7 +3228,7 @@ struct LegacyRound { } #[test] -fn records_from_before_the_new_fields_decode_with_defaults() { +fn a_run_from_before_does_not_open() { let mut dao = dao(); let mut model = fake(&good_answers()); let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); @@ -3243,11 +3245,11 @@ fn records_from_before_the_new_fields_decode_with_defaults() { chrf_version: r.chrf_version().into(), calls: r.calls().to_vec(), train: r.train().to_vec(), - train_mean: r.train_mean().clone(), - validation_mean: r.validation_mean().clone(), - validation_refused: r.validation_refused(), - train_scored: r.train_scored(), - validation_scored: r.validation_scored(), + train_mean: r.language("en-US").unwrap().train_mean.clone(), + validation_mean: r.language("en-US").unwrap().validation_mean.clone(), + validation_refused: r.language("en-US").unwrap().validation_refused, + train_scored: r.language("en-US").unwrap().train_scored, + validation_scored: r.language("en-US").unwrap().validation_scored, inventory_digest: r.inventory_digest().into(), cycle: r.cycle(), }; @@ -3266,25 +3268,33 @@ fn records_from_before_the_new_fields_decode_with_defaults() { lowerer_reach: rec.lowerer_reach, }; drop(dao.run); - let mut cache = CultCache::new(); - cache.register_entry_type::().unwrap(); - cache.register_entry_type::().unwrap(); - cache - .add_generic_backing_store(SingleFileMessagePackBackingStore::new(root.join("run.cc"))) - .unwrap(); - cache.pull_all_backing_stores().unwrap(); - cache.put("run:dao".to_string(), &run).unwrap(); - cache.put("run:dao:round:1".to_string(), &legacy).unwrap(); - drop(cache); - + let store = || SingleFileMessagePackBackingStore::new(root.join("run.cc")); + // A run record without a panel (no key 14) is a scratch run off git. + { + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache.add_generic_backing_store(store()).unwrap(); + cache.pull_all_backing_stores().unwrap(); + cache.put("run:dao".to_string(), &run).unwrap(); + cache.put("run:dao:round:1".to_string(), &legacy).unwrap(); + } + assert_eq!(RunStores::open(&root).err(), Some(RunError::Store)); + // A round with the pooled means (keys 11 to 15) and no languages is + // unreadable, under a record that has its panel. + let _ = std::fs::remove_file(root.join("run.cc")); + { + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache.add_generic_backing_store(store()).unwrap(); + cache.pull_all_backing_stores().unwrap(); + cache.put("run:dao".to_string(), &rec).unwrap(); + cache.put("run:dao:round:1".to_string(), &legacy).unwrap(); + } + // No round of that run can be read: it stops at its first round or cycle. let reopened = RunStores::open(&root).unwrap(); - assert_eq!(reopened.record().answerer_family, ModelFamily::Unrecorded); - let rounds = reopened.rounds().unwrap(); - assert_eq!(rounds.len(), 1); - assert_eq!(rounds[0].n(), 1); - assert!(rounds[0].admit_rules().is_empty()); - assert!(rounds[0].parse_reasons().is_empty()); - assert_eq!(rounds[0].own_family(), 0); + assert!(reopened.rounds().is_err()); } #[test] @@ -3310,10 +3320,24 @@ fn the_published_run_and_round_schemas_require_none_of_the_new_fields() { } } -/// Answers a recall probe from the corpus: the item's own second half, a -/// fixed wrong text, or junk. +/// What a recall prompt shows of its item: the text after `opening: `. +fn shown_opening(prompt: &str) -> &str { + prompt.rsplit_once("opening: ").expect("a recall prompt has an opening").1 +} + +/// The item's text after the opening the prompt showed. +fn held_back<'a>(text: &'a str, prompt: &str) -> &'a str { + let opening = shown_opening(prompt); + text.strip_prefix(opening) + .unwrap_or_else(|| panic!("the opening is a prefix of the item")) + .trim() +} + +/// Answers a recall probe from the corpus: the item's own held-back half (its +/// text after the opening the prompt showed), a fixed wrong text, or junk. struct Reciter { corpus: weksa::store::CorpusStore, + /// Given the item's held-back half. answer: fn(&str) -> String, /// An item whose text holds this gets an answer file that is not UTF-8. unreadable: Option<&'static str>, @@ -3331,14 +3355,12 @@ impl ModelPort for Reciter { if self.unreadable.is_some_and(|t| item.text.contains(t)) { return Err(PortError::Io(std::io::ErrorKind::InvalidData)); } - Ok((self.answer)(&item.text)) + Ok((self.answer)(held_back(&item.text, prompt.text()))) } } -fn second_half(text: &str) -> String { - let words: Vec<&str> = text.split_whitespace().collect(); - let draft = serde_json::json!({ "text": words[words.len() / 2..].join(" ") }); - draft.to_string() +fn second_half(held_back: &str) -> String { + serde_json::json!({ "text": held_back }).to_string() } fn reciter(dao: &Dao, answer: fn(&str) -> String) -> Reciter { @@ -3355,20 +3377,26 @@ fn a_recall_probe_asks_for_the_second_half_of_each_live_item_and_never_shows_it( use weksa::run::RecallOutcome; let dao = dao(); let mut perfect = reciter(&dao, second_half); - let RecallOutcome::Read(r) = dao.run.recall(&mut perfect).unwrap() else { + let RecallOutcome::Read(reading) = dao.run.recall(&mut perfect).unwrap() else { panic!("every prompt was answered"); }; + let r = reading.languages[0].clone(); // Items 1 (Train) and 2, 3 (Validation) are live; 4 and 5 are not offered. assert_eq!((r.n, r.answered), (3, 3)); assert!((r.chrf - 100.0).abs() < 1e-9, "{}", r.chrf); assert_eq!(perfect.seen.len(), 3); for (text, key) in perfect.seen.iter().zip(["item:daodejing:1", "item:daodejing:2", "item:daodejing:3"]) { let item = perfect.corpus.get(key).unwrap().unwrap(); - let rest = item.text.split_whitespace().collect::>(); - let tail = rest[rest.len() / 2..].join(" "); - let head = rest[..rest.len() / 2].join(" "); - assert!(text.contains(&head), "{key}: the opening is shown"); - assert!(!text.contains(&tail), "{key}: the held half is not shown"); + // The opening is a proper prefix of the item, cut at a word boundary + // near its middle; the rest is held back. + let head = shown_opening(text); + let tail = held_back(&item.text, text); + assert!(!head.is_empty() && !tail.is_empty(), "{key}"); + let squeeze = |t: &str| t.split_whitespace().collect::(); + assert_eq!(squeeze(&item.text), squeeze(&format!("{head}{tail}")), "{key}: head and tail are the item"); + let n = item.text.len(); + assert!((n / 4..=3 * n / 4).contains(&head.len()), "{key}: near the middle"); + assert!(!text.contains(tail), "{key}: the held half is not shown"); } } @@ -3377,15 +3405,17 @@ fn a_recall_of_the_wrong_words_or_no_answer_reads_low_and_a_missing_answer_waits use weksa::run::RecallOutcome; let dao = dao(); let mut wrong = reciter(&dao, |_| r#"{"text":"zzz qqq xxx"}"#.to_string()); - let RecallOutcome::Read(r) = dao.run.recall(&mut wrong).unwrap() else { + let RecallOutcome::Read(reading) = dao.run.recall(&mut wrong).unwrap() else { panic!("answered"); }; + let r = reading.languages[0].clone(); assert_eq!((r.n, r.answered), (3, 3)); assert!(r.chrf < 10.0, "{}", r.chrf); let mut junk = reciter(&dao, |_| "not json".to_string()); - let RecallOutcome::Read(r) = dao.run.recall(&mut junk).unwrap() else { + let RecallOutcome::Read(reading) = dao.run.recall(&mut junk).unwrap() else { panic!("answered"); }; + let r = reading.languages[0].clone(); assert_eq!((r.n, r.answered, r.chrf), (3, 0, 0.0)); // With no answers on disk the files port waits on all three prompts. let mut files = AnswerFiles::new(dao.run.recall_dir(), "m".into()); @@ -3429,10 +3459,10 @@ fn a_recall_mean_is_over_every_probe_an_unreadable_answer_counts_0_and_a_withdra let both = |dao: &Dao, unreadable: Option<&'static str>| { let mut m = reciter(dao, second_half); m.unreadable = unreadable; - let RecallOutcome::Read(r) = dao.run.recall(&mut m).unwrap() else { + let RecallOutcome::Read(reading) = dao.run.recall(&mut m).unwrap() else { panic!("answered"); }; - r + reading.languages[0].clone() }; let full = both(&dao, None); assert_eq!( @@ -3541,16 +3571,17 @@ fn mixed_family_round(family: ModelFamily) -> u32 { "bag", LowererReach::ToolLessRelay, family, + &["en-US".to_string()], ) .unwrap(); let scorer = weksa::score::Scorer::new(OneBag); match round(&mut run, &mut NotADraft, &scorer).unwrap() { RoundOutcome::Recorded(r) => { - // Seven items were offered: the world's own cast item and mix 1 to 6; + // Eight items were offered: the world's two cast items and mix 1 to 6; // the Test-group and withdrawn items were not. assert_eq!( r.parse_reasons().iter().map(|c| c.count).sum::(), - 7, + 8, "offered items" ); r.own_family() @@ -3697,7 +3728,7 @@ fn read( outcome: weksa::run::AblationOutcome, ) -> Vec<(weksa::Split, weksa::run::AblationReading)> { match outcome { - weksa::run::AblationOutcome::Read(r) => r, + weksa::run::AblationOutcome::Read(r) => r.into_iter().map(|(_, split, x)| (split, x)).collect(), weksa::run::AblationOutcome::Unanswered { prompts } => panic!("unanswered {prompts:?}"), } } diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 186cb2a..21f7a1d 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -238,12 +238,14 @@ fn the_recall_probe_covers_every_live_original_item_and_names_no_work() { "bag", weksa::run::LowererReach::BriefedSubagent, ModelFamily::Unrecorded, + &["en-US".to_string()], ) .unwrap(); let mut model = Unrelated(vec![]); - let RecallOutcome::Read(r) = run.recall(&mut model).unwrap() else { + let RecallOutcome::Read(reading) = run.recall(&mut model).unwrap() else { panic!("answered"); }; + let r = reading.languages[0].clone(); assert_eq!((r.n, r.answered), (28, 28)); assert!(r.chrf < 25.0, "{}", r.chrf); assert_eq!(model.0.len(), 28); @@ -251,9 +253,12 @@ fn the_recall_probe_covers_every_live_original_item_and_names_no_work() { let all = CorpusStore::open(&c).unwrap().all().unwrap(); for item in &all { let live = SplitScheme::V1.live(item); - // A recall prompt shows the first half of the words. - let words: Vec<&str> = item.text.split_whitespace().collect(); - let opening = words[..words.len() / 2].join(" "); + // A recall prompt shows the first half of the item, cut at a word boundary. + let mut cut = item.text.len() / 4; + while !item.text.is_char_boundary(cut) { + cut -= 1; + } + let opening = item.text[..cut].trim_end().to_string(); let shown = model.0.iter().any(|t| t.contains(&opening)); assert_eq!( shown, @@ -309,6 +314,7 @@ fn original_round(family: ModelFamily) -> (weksa::run::RunStores, weksa::run::Ro "bag", LowererReach::ToolLessRelay, family, + &["en-US".to_string()], ) .unwrap(); let scorer = weksa::score::Scorer::new(Bag); @@ -416,9 +422,10 @@ fn recall_probes_only_live_train_and_validation_items() { drop(store); let run = run_over(dir.path()); let mut m = Record(vec![]); - let RecallOutcome::Read(r) = run.recall(&mut m).unwrap() else { + let RecallOutcome::Read(reading) = run.recall(&mut m).unwrap() else { panic!("answered"); }; + let r = reading.languages[0].clone(); assert_eq!(r.n, 2); let shown = |mark: &str| m.0.iter().any(|t| t.contains(mark)); assert!(shown("trainmark") && shown("validmark")); @@ -464,6 +471,7 @@ fn no_test_item_is_offered_to_a_round_or_recall_or_recorded() { "bag", LowererReach::ToolLessRelay, ModelFamily::Unrecorded, + &["en-US".to_string()], ) .unwrap(); let items = CorpusStore::open(&p("corpus.cc")).unwrap().all().unwrap(); @@ -499,9 +507,10 @@ fn no_test_item_is_offered_to_a_round_or_recall_or_recorded() { assert!(r.train().iter().all(|row| row.item != item.id)); } let mut recall = Record(vec![]); - let weksa::run::RecallOutcome::Read(rr) = run.recall(&mut recall).unwrap() else { + let weksa::run::RecallOutcome::Read(reading) = run.recall(&mut recall).unwrap() else { panic!("answered"); }; + let rr = reading.languages[0].clone(); assert_eq!((rr.n, rr.answered), (28, 28)); for item in &held { assert!(!leaks(&recall.0, item), "recall prompt held {}", item.id);