diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 7b11969..7f1b7b4 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -1383,7 +1383,8 @@ impl RunStores { /// The lowering prompt of `packet` for `item`: admitted into `store`, /// assembled and lowered. The one builder for the round's prompt, which - /// ablation and relower must reproduce byte for byte. + /// ablation and relower must reproduce byte for byte. `None` when the + /// lowering prompt itself is refused, as it was for the round. fn lowering_prompt( &self, store: &mut PacketStore, @@ -1391,7 +1392,7 @@ impl RunStores { item: &CorpusItem, inventory: &crate::Inventory, ontology: &crate::Ontology, - ) -> Result { + ) -> Result, RunError> { let packet_item = packet.provenance.corpus_item.clone(); let view = store .admit(packet, &self.inventory, &self.corpus, &self.entities) @@ -1399,8 +1400,7 @@ impl RunStores { let view = self .assembled(view, &self.inventory) .map_err(|_| RunError::Store)?; - Prompt::lower(&view, &packet_item, &item.language, inventory, &self.entities, ontology) - .map_err(|_| RunError::Store) + Ok(Prompt::lower(&view, &packet_item, &item.language, inventory, &self.entities, ontology).ok()) } /// `/ablation-`, where a stance ablation's prompts, answers and @@ -1455,7 +1455,8 @@ impl RunStores { let mut rows = Vec::new(); for (i, (item, packet)) in held.iter().enumerate() { let mut prompt_of = |slot: usize, packet: Packet| -> Result { - self.lowering_prompt(&mut stores[slot], packet, item, &inventory, &ontology) + self.lowering_prompt(&mut stores[slot], packet, item, &inventory, &ontology)? + .ok_or(RunError::Store) }; let base = prompt_of(0, packet.clone())?; let mut variants: [Option; 2] = [None, None]; @@ -1574,10 +1575,20 @@ impl RunStores { #[derive(Clone, Debug, PartialEq)] pub struct RelowerReading { pub language: String, - /// Live Validation items of the language in the round. + /// Live Validation items of the language read: those with a round packet + /// whose lowering prompt the round could build. Every reading below is over + /// these items, so recorded, arm and floor stay comparable. pub n: u32, + /// Live Validation items of the language not read: refused by the round + /// before admission (no packet) or at the lowering prompt. They are in no + /// reading, so `recorded` can differ from the round's own Validation + /// reading, which scores them 0. + pub skipped: u32, /// Arm answers that were not a usable lowering; each scored 0. pub unusable: u32, + /// Floor-arm answers that were not a usable lowering; each scored 0, which + /// widens `floor`. Zero without a floor arm. + pub floor_unusable: u32, /// The round's own lowering, read against the items. pub recorded: Reading, pub arm: Reading, @@ -1593,11 +1604,14 @@ impl RelowerReading { let floor = self .floor .as_ref() - .map_or(String::new(), |f| format!(" | floor {}", show(f))); + .map_or(String::new(), |f| { + format!(" | floor {} unusable={}", show(f), self.floor_unusable) + }); format!( - "relower {} n={} unusable={} recorded {} | arm {}{floor}", + "relower {} n={} skipped={} unusable={} recorded {} | arm {}{floor}", self.language, self.n, + self.skipped, self.unusable, show(&self.recorded), show(&self.arm), @@ -1661,8 +1675,13 @@ impl RunStores { for (_, _, items) in &held { let mut language_rows = Vec::new(); for (item, packet) in items { - let prompt = - self.lowering_prompt(&mut scratch, packet.clone(), item, &inventory, &ontology)?; + // An item the round could not lower has no prompt to ask; it is skipped + // and counted, not scored. + let Some(prompt) = + self.lowering_prompt(&mut scratch, packet.clone(), item, &inventory, &ontology)? + else { + continue; + }; let text = |a: Asked| match a { Asked::Text(t) => Some(t), _ => None, @@ -1730,9 +1749,10 @@ impl RunStores { .ok_or(RunError::NoValidation) }; let mut languages = Vec::new(); + let live = self.corpus.all().map_err(store)?; for ((language, _, _), language_rows) in held.iter().zip(&rows) { let (mut rec, mut arm_scores, mut floor_scores) = (Vec::new(), Vec::new(), Vec::new()); - let mut unusable = 0u32; + let (mut unusable, mut floor_unusable) = (0u32, 0u32); for (_, r, a, f) in language_rows { rec.push(next(r)?); arm_scores.push(next(a)?); @@ -1741,13 +1761,24 @@ impl RunStores { } if floor.is_some() { floor_scores.push(next(f)?); + if f.is_none() { + floor_unusable += 1; + } } } let recorded_reading = reading_of(&rec)?; languages.push(RelowerReading { language: language.clone(), n: language_rows.len() as u32, + skipped: live + .iter() + .filter(|i| { + &i.language == language && SplitScheme::V1.live(i) == Some(Split::Validation) + }) + .count() as u32 + - language_rows.len() as u32, unusable, + floor_unusable, recorded: recorded_reading, arm: reading_of(&arm_scores)?, floor: if floor.is_some() { diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 455f751..95a147a 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -1820,7 +1820,7 @@ fn cli_loop_relower_exits_3_until_answered_and_refuses_bad_labels() { let second = relower(&["--arm", "tuned"]); assert!(second.status.success(), "{}", out(&second)); let text = out(&second); - assert!(text.starts_with("relower en-US n=2 unusable=0 recorded "), "{text}"); + assert!(text.starts_with("relower en-US n=2 skipped=0 unusable=0 recorded "), "{text}"); assert!(text.contains("arm=tuned lowerer=tuned-x"), "{text}"); assert!(text.contains(" lowering="), "{text}"); assert!(!text.contains("verdict") && !text.contains("floor"), "{text}"); diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 771da27..179d0ea 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -5503,6 +5503,96 @@ fn relower_floor_is_the_spread_of_recorded_and_the_floor_arm() { assert!((f.composite - (l.recorded.composite - a.composite).abs()).abs() < 1e-9); assert!(l.line().contains(&format!("| floor chrf={:.2}", f.chrf)), "{}", l.line()); assert_eq!(l.unusable, 0, "unusable counts the arm, not the floor"); + assert_eq!(l.floor_unusable, 1, "the floor arm's refusal is counted apart"); + assert!(l.line().contains(" unusable=0 recorded "), "{}", l.line()); + assert!(l.line().ends_with(" unusable=1"), "the floor's refusal is on the line: {}", l.line()); + // Without a floor arm there is none to count. + assert_eq!(as_arm[0].floor_unusable, 0); +} + +#[test] +fn relower_skips_and_counts_an_item_the_round_refused_before_admission() { + // Validation item 3 is not an encodable draft in the round: it has no packet. + let (dao, _) = recorded_dao(|answers| { + answers.insert("item:daodejing:3".into(), "not a draft".into()); + }); + let (mut rec, mut arm) = (Lowerer::new(blind), Lowerer::new(blind)); + let (langs, _) = relowered(relower(&dao, &mut rec, &mut arm, None)); + let l = &langs[0]; + assert_eq!((l.n, l.skipped, l.unusable), (2, 1, 0), "{}", l.line()); + assert!(l.line().starts_with("relower en-US n=2 skipped=1 unusable=0 "), "{}", l.line()); + assert_eq!(l.recorded, l.arm, "both readings are over the same two items"); + assert!(arm.seen.iter().all(|(_, key, _)| key != "item:daodejing:3"), "nothing is asked for it"); +} + +#[test] +fn relower_skips_and_counts_an_item_whose_lowering_prompt_the_round_refused() { + // ja-JP item 12 cites an entity with no ja form: the round admitted it and refused it + // at Lower. Item 13 cites none and lowers. + let mut dao = bilingual(None); + let mut model = bilingual_fake(); + model.answers.insert("item:daodejing:12".into(), citing_laozi(2)); + model.answers.insert("item:daodejing:13".into(), json(&draft_of(3))); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + assert_eq!(r.language("ja-JP").unwrap().validation_refused.lower, 1); + let (mut rec, mut arm) = (Lowerer::new(bi_blind), Lowerer::new(bi_blind)); + let (langs, _) = relowered(relower(&dao, &mut rec, &mut arm, None)); + let ja = langs.iter().find(|l| l.language == "ja-JP").unwrap(); + assert_eq!((ja.n, ja.skipped), (1, 1), "{}", ja.line()); + assert_eq!(ja.recorded, ja.arm); + let en = langs.iter().find(|l| l.language == "en-US").unwrap(); + assert_eq!((en.n, en.skipped), (2, 0), "{}", en.line()); + assert!(arm.seen.iter().all(|(_, key, _)| key != "item:daodejing:12")); +} + +/// The ja-JP item's own text, else the English one: `f` for en-US. +fn bi(key: &str, text: &str, f: fn(&str, &str) -> String) -> String { + match JA_TEXTS.iter().find(|(k, ..)| *k == key) { + Some((_, ja, _)) => ja.to_string(), + None => f(key, text), + } +} + +fn bi_blind(key: &str, text: &str) -> String { + bi(key, text, blind) +} + +fn bi_reader(key: &str, text: &str) -> String { + bi(key, text, reader) +} + +/// Fine in English, nonsense in ja-JP. +fn bi_ja_lost(key: &str, text: &str) -> String { + match JA_TEXTS.iter().any(|(k, ..)| *k == key) { + true => "zzzz".into(), + false => blind(key, text), + } +} + +#[test] +fn relower_decides_over_the_whole_panel_not_its_first_language() { + use weksa::run::decide_panel; + let mut dao = bilingual(LAOZI_JA); + recorded(round(&mut dao.run, &mut bilingual_fake(), &Scorer::new(Bag)).unwrap()); + // en-US gains (the arm drops the stance the recorded lowering repeats); ja-JP is lost. + let (langs, verdict) = relowered(relower( + &dao, + &mut Lowerer::new(bi_reader), + &mut Lowerer::new(bi_ja_lost), + Some(&mut Lowerer::new(bi_reader)), + )); + assert_eq!( + langs.iter().map(|l| (l.language.as_str(), l.n)).collect::>(), + [("en-US", 2), ("ja-JP", 2)], + "panel order, Validation only" + ); + assert!(langs[0].arm.composite > langs[0].recorded.composite, "en-US gains"); + assert!(langs[1].arm.composite < langs[1].recorded.composite, "ja-JP is lost"); + let panel: Vec<_> = langs.iter().map(|l| (l.recorded, l.arm, l.floor.unwrap())).collect(); + assert_eq!(verdict, Some(decide_panel(&panel))); + assert!(matches!(verdict, Some(Verdict::Rejected(Why::Regressed(_)))), "{verdict:?}"); + // The first language alone would have accepted. + assert_eq!(decide_panel(&panel[..1]), Verdict::Accepted); } #[test]