diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 9ba2135..fc57236 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -154,8 +154,9 @@ enum LoopCommand { ollama_url: String, }, /// Re-lower round ROUND's Validation packets through arm ARM, score them - /// against the items, and, with --floor ARM, decide against that second arm - /// by the panel accept rule. Prompts and answers live in + /// against the items, and, with --floor ARM given two or more times (prompted + /// repeats of the lowering), decide against the paired noise floor of those + /// repeats by the panel accept rule. Prompts and answers live in /// `/relower--`; the command exits 3 until the arm is /// answered. --lowerer names the arm's model in the printed line. A /// diagnostic, never a gate; records nothing; Test and Train are never @@ -169,7 +170,7 @@ enum LoopCommand { #[arg(long)] lowerer: String, #[arg(long)] - floor: Option, + floor: Vec, #[arg(long, default_value = "http://127.0.0.1:11434")] ollama_url: String, }, @@ -1147,7 +1148,10 @@ fn main() -> Result<()> { (1..=32).contains(&l.len()) && l.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') }; - if !label(&arm) || floor.as_deref().is_some_and(|f| !label(f) || f == arm) { + if !label(&arm) + || floor.iter().any(|f| !label(f) || *f == arm) + || floor.iter().enumerate().any(|(i, f)| floor[..i].contains(f)) + { anyhow::bail!("arm and floor labels must match [a-z0-9-]{{1,32}} and differ"); } let run = RunStores::open(&root)?; @@ -1156,16 +1160,19 @@ fn main() -> Result<()> { run.record().model.clone(), ); let mut files = run.answer_files(run.relower_dir(round, &arm))?; - let mut floor_port = floor.as_deref().map(|f| { - RecordedAnswers::new(run.relower_dir(round, f), run.record().model.clone()) - }); + let mut floor_ports: Vec = floor + .iter() + .map(|f| RecordedAnswers::new(run.relower_dir(round, f), run.record().model.clone())) + .collect(); + let mut floor_dyn: Vec<&mut dyn ModelPort> = + floor_ports.iter_mut().map(|p| p as &mut dyn ModelPort).collect(); let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder)); match run.relower( round, &arm, &mut recorded, &mut files, - floor_port.as_mut().map(|p| p as &mut dyn ModelPort), + &mut floor_dyn, &scorer, )? { RelowerOutcome::Unanswered { prompts } => { diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 1dc4780..6212b75 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -498,6 +498,8 @@ pub enum RunError { /// The floor arm of a relower holds no answer to a prompt; it is a /// reference, never written to, and must be answered whole first. FloorUnanswered, + /// A relower was given fewer floor arms than [`FLOOR_ARMS_MIN`]. + FloorArms, } impl fmt::Display for RunError { @@ -526,6 +528,7 @@ impl fmt::Display for RunError { RunError::SpeakerState => { f.write_str("a scene does not read against the run's inventory") } + RunError::FloorArms => f.write_str("a floor needs three or more floor arms; fewer measure no noise"), RunError::FloorUnanswered => { f.write_str("the floor arm holds no answer to a lowering prompt; answer it whole first") } @@ -1586,13 +1589,14 @@ pub struct RelowerReading { pub skipped: u32, /// Arm answers that were not a usable lowering; each scored 0. pub unusable: u32, - /// Floor-arm answers that were not a usable lowering; each scored 0, which - /// widens `floor`. Zero without a floor arm. + /// Floor-arm answers that were not a usable lowering, over every floor arm; + /// each scored 0, which widens `floor`. Zero without floor arms. pub floor_unusable: u32, /// The round's own lowering, read against the items. pub recorded: Reading, pub arm: Reading, - /// `spread(recorded, floor arm)`, when a floor arm was given. + /// [`paired_floor`] of the recorded lowering against the floor arms, when + /// floor arms were given. pub floor: Option, } @@ -1650,7 +1654,7 @@ impl RunStores { arm_label: &str, recorded: &mut dyn ModelPort, arm: &mut dyn ModelPort, - mut floor: Option<&mut dyn ModelPort>, + floors: &mut [&mut dyn ModelPort], scorer: &Scorer, ) -> Result { let store = |_| RunError::Store; @@ -1666,9 +1670,12 @@ impl RunStores { } let mut scratch = PacketStore::open(&dir.join("base.cc")).map_err(store)?; - // Per language, per item: the item text, then the recorded, arm and floor - // lowerings (`None` when unusable). - type Row = (String, Option, Option, Option); + if (1..FLOOR_ARMS_MIN).contains(&floors.len()) { + return Err(RunError::FloorArms); + } + // Per language, per item: the item text, then the recorded and arm + // lowerings and one per floor arm (`None` when unusable). + type Row = (String, Option, Option, Vec>); let mut waiting: Vec = Vec::new(); let mut floor_waiting = false; let mut rows: Vec> = Vec::new(); @@ -1690,16 +1697,16 @@ impl RunStores { Asked::Waiting => return Err(RunError::Unrecorded), other => text(other), }; - let floor_text = match floor.as_deref_mut() { - Some(f) => match asked(f, &prompt)? { + let mut floor_texts = Vec::new(); + for f in floors.iter_mut() { + floor_texts.push(match asked(&mut **f, &prompt)? { Asked::Waiting => { floor_waiting = true; None } other => text(other), - }, - None => None, - }; + }); + } let arm_text = match asked(arm, &prompt)? { Asked::Waiting => { let d = prompt.digest(); @@ -1710,7 +1717,7 @@ impl RunStores { } other => text(other), }; - language_rows.push((item.text.clone(), recorded_text, arm_text, floor_text)); + language_rows.push((item.text.clone(), recorded_text, arm_text, floor_texts)); } rows.push(language_rows); } @@ -1723,8 +1730,8 @@ impl RunStores { // One score call: each usable lowering against its item, in row order. let mut inputs: Vec = Vec::new(); - for (reference, recorded, arm, floor_text) in rows.iter().flatten() { - for output in [recorded, arm, floor_text].into_iter().flatten() { + for (reference, recorded, arm, floor_texts) in rows.iter().flatten() { + for output in [recorded, arm].into_iter().chain(floor_texts).flatten() { inputs.push(ScoreInput { reference, output }); } } @@ -1751,7 +1758,8 @@ impl RunStores { let mut languages = Vec::new(); let live = self.corpus.all().map_err(store)?; for ((language, _, _), language_rows) in held.iter().zip(&rows) { - let (mut rec, mut arm_scores, mut floor_scores) = (Vec::new(), Vec::new(), Vec::new()); + let (mut rec, mut arm_scores) = (Vec::new(), Vec::new()); + let mut floor_scores: Vec> = vec![Vec::new(); floors.len()]; let (mut unusable, mut floor_unusable) = (0u32, 0u32); for (_, r, a, f) in language_rows { rec.push(next(r)?); @@ -1759,9 +1767,9 @@ impl RunStores { if a.is_none() { unusable += 1; } - if floor.is_some() { - floor_scores.push(next(f)?); - if f.is_none() { + for (scores, t) in floor_scores.iter_mut().zip(f) { + scores.push(next(t)?); + if t.is_none() { floor_unusable += 1; } } @@ -1781,14 +1789,10 @@ impl RunStores { floor_unusable, recorded: recorded_reading, arm: reading_of(&arm_scores)?, - floor: if floor.is_some() { - Some(spread(&recorded_reading, &reading_of(&floor_scores)?)) - } else { - None - }, + floor: if floors.is_empty() { None } else { Some(paired_floor(&rec, &floor_scores)) }, }); } - let verdict = if floor.is_some() { + let verdict = if !floors.is_empty() { let panel: Vec<(Reading, Reading, Reading)> = languages .iter() .filter_map(|l| Some((l.recorded, l.arm, l.floor?))) @@ -2261,7 +2265,11 @@ pub fn decide(base: &Reading, candidate: &Reading, floor: &Reading) -> Verdict { /// decides whether a language regressed (any regression rejects; the first in /// panel order names the metric), and the gain test is [`decide`]'s for one /// language and, for two or more, the panel's: the sum over languages of the -/// composite deltas (billionths) must exceed one [`LADDER_STEP`]. A language +/// composite deltas (billionths) must exceed the larger of one [`LADDER_STEP`] +/// and the noise of that sum, the root of the languages' composite floors +/// squared (independent languages; a one-language panel's noise is its floor, +/// as in [`decide`]). Without that noise term a prompted repeat of the base +/// lowering read Accepted in 30% of the calibration trials on `distil-base`. A language /// may lose up to its own noise floor if the panel nets more than a step; the /// rule gives up strict Pareto non-regression, which on noisy per-language /// readings ratchets every base to a favourable draw and strangles the loop @@ -2289,7 +2297,10 @@ pub fn decide_panel(rows: &[(Reading, Reading, Reading)]) -> Verdict { .iter() .map(|(base, candidate, _)| nano(candidate.composite - base.composite)) .sum(); - if net > nano(LADDER_STEP) { + // The noise of a sum of independent languages' differences: the root of the + // sum of their floors squared. + let noise = rows.iter().map(|(_, _, floor)| floor.composite.powi(2)).sum::().sqrt(); + if net > nano(noise.max(LADDER_STEP)) { Verdict::Accepted } else { Verdict::Rejected(Why::NoGain) @@ -2488,6 +2499,49 @@ fn floor_of( .collect() } +/// How many standard errors of a prompted-vs-prompted mean difference the +/// relower floor is wide, and the fewest floor arms (prompted repeats of the +/// recorded lowering) it is measured over. Calibrated on `runs/distil-base` +/// (seven prompted draws of 14 en-US and 14 ja-JP Validation items): with +/// three floor arms a prompted repeat reads NoGain in 92% of the trials +/// (bootstrap 95% over draws 80 to 100%), a "Yes." arm and a mild +/// degradation (one detail blurred per utterance) read Regressed in 100% and +/// 99%, and a second, subtler one in 54% (15 to 95%): at 14 items a +/// degradation under about three standard errors cannot be told from noise. +pub const FLOOR_Z: f64 = 3.0; +pub const FLOOR_ARMS_MIN: usize = 3; + +/// The lowering-only noise floor of a relower: `FLOOR_Z` standard errors of the +/// mean per-item difference between the recorded lowering and a repeat of it, +/// metric by metric. Each floor arm gives one paired sample of per-item +/// differences; their variances are pooled over the arms, so the floor rests on +/// `arms * (n - 1)` degrees of freedom instead of the one coincidental +/// difference of means a two-run spread is (which reads 0 whenever the two +/// means happen to meet, and then a one-step dip regresses). Paired per item, +/// after Koehn (2004). NaN (fail closed in [`decide`]) when there is no pair +/// of items or no arm. +pub fn paired_floor(recorded: &[ItemScore], arms: &[Vec]) -> Reading { + let n = recorded.len(); + if n < 2 || arms.is_empty() || arms.iter().any(|a| a.len() != n) { + return Reading { chrf: f64::NAN, meaning: f64::NAN, composite: f64::NAN }; + } + let pooled = |of: &dyn Fn(&ItemScore) -> f64| { + let variance = |arm: &Vec| { + let d: Vec = recorded.iter().zip(arm).map(|(r, a)| of(r) - of(a)).collect(); + let mean = d.iter().sum::() / n as f64; + d.iter().map(|x| (x - mean).powi(2)).sum::() / (n - 1) as f64 + }; + let v = arms.iter().map(variance).sum::() / arms.len() as f64; + FLOOR_Z * (v / n as f64).sqrt() + }; + let meaning = |s: &ItemScore| s.meaning.unwrap_or(0.0); + Reading { + chrf: pooled(&|s| s.chrf), + meaning: pooled(&meaning), + composite: pooled(&|s| (s.chrf / 100.0 + meaning(s)) / 2.0), + } +} + /// The absolute difference of two readings, metric by metric. fn spread(a: &Reading, b: &Reading) -> Reading { Reading { @@ -2869,6 +2923,42 @@ mod tests { assert_eq!(decide(&base, &gain(0.506), &floor), rejected(Why::NoGain)); } + #[test] + fn paired_floor_is_z_standard_errors_of_the_paired_difference() { + let item = |chrf: f64, meaning: f64| ItemScore { chrf, meaning: Some(meaning) }; + let recorded = [item(60.0, 0.8), item(40.0, 0.6), item(50.0, 0.7), item(30.0, 0.5)]; + // An arm that differs by a constant has no spread of differences: a + // shifted mean is the thing the floor is read against, not noise. + let shifted: Vec = recorded.iter().map(|s| item(s.chrf - 5.0, s.meaning.unwrap() - 0.1)).collect(); + let f = paired_floor(&recorded, &[shifted.clone(), shifted.clone(), shifted]); + assert!(f.chrf.abs() < 1e-9 && f.meaning.abs() < 1e-9 && f.composite.abs() < 1e-9, "{f:?}"); + // Differences 2, -2, 2, -2 on chrF: sample variance 16/3, standard error + // sqrt(16/3/4), FLOOR_Z of it; arms pool, so a second equal arm changes nothing. + let wobble: Vec = recorded + .iter() + .enumerate() + .map(|(i, s)| item(s.chrf - if i % 2 == 0 { 2.0 } else { -2.0 }, s.meaning.unwrap())) + .collect(); + let want = FLOOR_Z * (16.0_f64 / 3.0 / 4.0).sqrt(); + let one = paired_floor(&recorded, std::slice::from_ref(&wobble)); + let two = paired_floor(&recorded, &[wobble.clone(), wobble]); + assert!((one.chrf - want).abs() < 1e-9 && (two.chrf - want).abs() < 1e-9, "{one:?} {two:?}"); + assert!(one.meaning.abs() < 1e-9); + // Under two items, no arms or a ragged arm there is no floor: NaN fails + // closed in `decide`. + for f in [ + paired_floor(&recorded[..1], &[vec![item(1.0, 1.0)]]), + paired_floor(&recorded, &[]), + paired_floor(&recorded, &[vec![item(1.0, 1.0)]]), + ] { + assert!(f.composite.is_nan()); + assert_eq!( + decide(&reading(50.0, 0.5, 0.5), &reading(50.0, 0.5, 0.9), &f), + Verdict::Rejected(Why::Regressed(Metric::Composite)) + ); + } + } + #[test] fn decide_panel_table() { let zero = reading(0.0, 0.0, 0.0); @@ -2922,9 +3012,8 @@ mod tests { assert_eq!(net(LADDER_STEP + 6e-10), Verdict::Accepted); assert_eq!(net(LADDER_STEP + 2e-9), Verdict::Accepted); // Liveness: no language need pass its own threshold. Two languages each - // gaining 0.6 of a step at zero floor accept; so does one gaining two - // steps under a 0.02 floor (its own threshold) with the others flat; so - // does a gain with another language falling inside its own floor. + // gaining 0.6 of a step at zero floor accept; so does a gain past the + // noise of the sum with another language falling inside its own floor. let each = 0.6 * LADDER_STEP; assert_eq!( decide_panel(&[ @@ -2933,27 +3022,32 @@ mod tests { ]), Verdict::Accepted ); + // The sum must also clear its own noise: the root of the floors squared + // (0.02 and 0.02 give 0.0283), not just a step. A repeat of the base + // lowering gains that much by chance. let big_floor = reading(0.0, 0.0, 0.02); - assert_eq!( - decide_panel(&[(base, reading(50.0, 0.5, 0.5 + 2.0 * LADDER_STEP), big_floor), (base, flat, big_floor)]), - Verdict::Accepted - ); + let sum_of = |gain: f64| { + decide_panel(&[(base, reading(50.0, 0.5, 0.5 + gain), big_floor), (base, flat, big_floor)]) + }; + assert_eq!(sum_of(0.02), rejected(Why::NoGain), "past a step, under the noise of the sum"); + assert_eq!(sum_of(0.028), rejected(Why::NoGain)); + assert_eq!(sum_of(0.029), Verdict::Accepted); let own_floor = reading(0.0, 0.0, 0.004); + assert_eq!( + decide_panel(&[ + (base, reading(50.0, 0.5, 0.55), reading(0.0, 0.0, 0.03)), + (base, reading(50.0, 0.5, 0.497), own_floor) + ]), + Verdict::Accepted, + "a fall inside the language's floor, net 0.047 over a noise of 0.0303" + ); assert_eq!( decide_panel(&[ (base, reading(50.0, 0.5, 0.51), reading(0.0, 0.0, 0.03)), (base, reading(50.0, 0.5, 0.497), own_floor) ]), - Verdict::Accepted, - "a fall inside the language's floor, net 0.007" - ); - // The same fall with the net exactly a step rejects. - assert_eq!( - decide_panel(&[ - (base, reading(50.0, 0.5, 0.508), reading(0.0, 0.0, 0.03)), - (base, reading(50.0, 0.5, 0.497), own_floor) - ]), - rejected(Why::NoGain) + rejected(Why::NoGain), + "net 0.007 is under the noise" ); // The sum is over the deltas, with each language's fall counted. assert_eq!( diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 95a147a..e3abfc7 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -1833,12 +1833,19 @@ fn cli_loop_relower_exits_3_until_answered_and_refuses_bad_labels() { assert_eq!(o.status.code(), Some(1), "{}", out(&o)); assert!(!root.join("relower-1-second").exists()); - // Answered floor: a verdict line. - std::fs::create_dir(root.join("relower-1-second")).unwrap(); - for digest in &digests { - std::fs::write(root.join(format!("relower-1-second/{digest}.answer")), said).unwrap(); - } + // Fewer than three floor arms is refused: one pair measures no noise. let o = relower(&["--arm", "tuned", "--floor", "second"]); + assert_eq!(o.status.code(), Some(1), "{}", out(&o)); + assert!(out(&o).contains("three or more floor arms"), "{}", out(&o)); + + // Answered floors: a verdict line. + for label in ["second", "third", "fourth"] { + std::fs::create_dir_all(root.join(format!("relower-1-{label}"))).unwrap(); + for digest in &digests { + std::fs::write(root.join(format!("relower-1-{label}/{digest}.answer")), said).unwrap(); + } + } + let o = relower(&["--arm", "tuned", "--floor", "second", "--floor", "third", "--floor", "fourth"]); assert!(o.status.success(), "{}", out(&o)); let text = out(&o); assert!(text.contains("| floor chrf=0.00") && text.trim_end().ends_with("verdict Rejected(NoGain)"), "{text}"); @@ -1852,6 +1859,7 @@ fn cli_loop_relower_exits_3_until_answered_and_refuses_bad_labels() { vec!["--arm", &long], vec!["--arm", "../x"], vec!["--arm", "tuned", "--floor", "Bad"], + vec!["--arm", "tuned", "--floor", "b", "--floor", "b", "--floor", "c"], ] { let o = relower(&args); assert_eq!(o.status.code(), Some(1), "{args:?}: {}", out(&o)); diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 179d0ea..2d0320a 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -5335,14 +5335,14 @@ fn the_cycles_candidate_round_stops_on_a_withheld_lowering_prompt() { use weksa::run::{RelowerOutcome, RelowerReading}; -/// `relower` of round 1 with the Bag scorer; `floor` is an optional second arm. +/// `relower` of round 1 with the Bag scorer; `floors` are the floor arms (none, or three or more). fn relower( dao: &Dao, recorded: &mut dyn ModelPort, arm: &mut dyn ModelPort, - floor: Option<&mut dyn ModelPort>, + floors: &mut [&mut dyn ModelPort], ) -> Result { - dao.run.relower(1, "arm", recorded, arm, floor, &Scorer::new(Bag)) + dao.run.relower(1, "arm", recorded, arm, floors, &Scorer::new(Bag)) } fn relowered(outcome: Result) -> (Vec, Option) { @@ -5360,7 +5360,7 @@ fn relower_asks_only_validation_lowering_prompts_of_the_round() { // which digest is which item's. let (mut every, mut rec, mut arm) = (Lowerer::new(reader), Lowerer::new(reader), Lowerer::new(reader)); read(ablate(&dao, &mut every, &mut Lowerer::new(reader)).unwrap()); - relowered(relower(&dao, &mut rec, &mut arm, None)); + relowered(relower(&dao, &mut rec, &mut arm, &mut [])); let of = |keys: &[&str]| -> Vec { every .seen @@ -5400,7 +5400,7 @@ fn relower_reads_recorded_and_arm_against_the_item() { &dao, &mut Lowerer::new(reader), &mut Lowerer::new(blind), - None, + &mut [], )); let l = &langs[0]; assert_eq!((l.language.as_str(), l.n, l.unusable), ("en-US", 3, 0)); @@ -5408,7 +5408,7 @@ fn relower_reads_recorded_and_arm_against_the_item() { assert!(l.arm.composite > l.recorded.composite); assert!(l.floor.is_none() && verdict.is_none(), "no floor arm, no verdict"); // An arm equal to the recorded answers reads equal. - let (langs, _) = relowered(relower(&dao, &mut Lowerer::new(reader), &mut Lowerer::new(reader), None)); + let (langs, _) = relowered(relower(&dao, &mut Lowerer::new(reader), &mut Lowerer::new(reader), &mut [])); assert_eq!(langs[0].recorded, langs[0].arm); } @@ -5432,11 +5432,11 @@ impl ModelPort for Breaks { fn relower_counts_an_unusable_arm_answer_as_zero() { let (dao, _) = recorded_dao(|_| {}); let keys = ["item:daodejing:2", "item:daodejing:3", WRITTEN]; - let (whole, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), None)); + let (whole, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), &mut [])); let whole = whole[0].arm; let (mut chrf, mut meaning) = (0.0, 0.0); for key in keys { - let (langs, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Breaks(key), None)); + let (langs, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Breaks(key), &mut [])); let l = &langs[0]; assert_eq!((l.n, l.unusable), (3, 1), "{}", l.line()); assert!(l.arm.chrf < whole.chrf && l.arm.meaning < whole.meaning, "{}", l.line()); @@ -5452,8 +5452,8 @@ fn relower_counts_an_unusable_arm_answer_as_zero() { #[test] fn relower_scores_an_unusable_recorded_answer_as_zero_without_counting_it_against_the_arm() { let (dao, _) = recorded_dao(|_| {}); - let (whole, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), None)); - let (langs, _) = relowered(relower(&dao, &mut Breaks("item:daodejing:3"), &mut Lowerer::new(blind), None)); + let (whole, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), &mut [])); + let (langs, _) = relowered(relower(&dao, &mut Breaks("item:daodejing:3"), &mut Lowerer::new(blind), &mut [])); let l = &langs[0]; assert_eq!(l.unusable, 0, "{}", l.line()); assert_eq!(l.arm, whole[0].arm); @@ -5465,16 +5465,16 @@ fn relower_decides_by_decide_panel_against_the_floor_arm() { use weksa::run::decide_panel; let (dao, _) = recorded_dao(|_| {}); let run = |recorded: fn(&str, &str) -> String, arm: fn(&str, &str) -> String| { - let mut floor = Lowerer::new(recorded); + let (mut f1, mut f2, mut f3) = (Lowerer::new(recorded), Lowerer::new(recorded), Lowerer::new(recorded)); let (langs, verdict) = relowered(relower( &dao, &mut Lowerer::new(recorded), &mut Lowerer::new(arm), - Some(&mut floor), + &mut [&mut f1, &mut f2, &mut f3], )); let l = &langs[0]; let floor_reading = l.floor.expect("a floor arm gives a floor"); - // A floor arm that equals the recorded answers has no spread. + // Floor arms that equal the recorded answers have no spread. assert_eq!((floor_reading.chrf, floor_reading.meaning, floor_reading.composite), (0.0, 0.0, 0.0)); assert_eq!(verdict, Some(decide_panel(&[(l.recorded, l.arm, floor_reading)]))); verdict.unwrap() @@ -5485,29 +5485,34 @@ fn relower_decides_by_decide_panel_against_the_floor_arm() { } #[test] -fn relower_floor_is_the_spread_of_recorded_and_the_floor_arm() { +fn relower_floor_is_the_paired_noise_of_recorded_against_the_floor_arms() { let (dao, _) = recorded_dao(|_| {}); let (langs, _) = relowered(relower( &dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), - Some(&mut Breaks("item:daodejing:3")), + &mut [&mut Lowerer::new(blind), &mut Lowerer::new(blind), &mut Breaks("item:daodejing:3")], )); let l = &langs[0]; let f = l.floor.unwrap(); - // The floor arm read as an arm against the same recorded answers. - let (as_arm, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Breaks("item:daodejing:3"), None)); - let a = as_arm[0].arm; - assert!(f.chrf > 0.0 && f.meaning > 0.0 && f.composite > 0.0); - assert!((f.chrf - (l.recorded.chrf - a.chrf).abs()).abs() < 1e-9, "{}", l.line()); - assert!((f.composite - (l.recorded.composite - a.composite).abs()).abs() < 1e-9); + assert!(f.chrf > 0.0 && f.meaning > 0.0 && f.composite > 0.0, "{}", l.line()); assert!(l.line().contains(&format!("| floor chrf={:.2}", f.chrf)), "{}", l.line()); assert_eq!(l.unusable, 0, "unusable counts the arm, not the floor"); - assert_eq!(l.floor_unusable, 1, "the floor arm's refusal is counted apart"); + assert_eq!(l.floor_unusable, 1, "a floor arm's refusal is counted apart"); assert!(l.line().contains(" unusable=0 recorded "), "{}", l.line()); assert!(l.line().ends_with(" unusable=1"), "the floor's refusal is on the line: {}", l.line()); - // Without a floor arm there is none to count. - assert_eq!(as_arm[0].floor_unusable, 0); + // A floor needs three arms: fewer measure no noise, and nothing is asked of the arm. + for arms in [1usize, 2] { + let mut fs: Vec = (0..arms).map(|_| Lowerer::new(blind)).collect(); + let mut dyns: Vec<&mut dyn ModelPort> = fs.iter_mut().map(|f| f as &mut dyn ModelPort).collect(); + let mut arm = Lowerer::new(blind); + let r = relower(&dao, &mut Lowerer::new(blind), &mut arm, &mut dyns); + assert!(matches!(r, Err(RunError::FloorArms))); + assert!(arm.seen.is_empty()); + } + // Without floor arms there is none to count. + let (none, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), &mut [])); + assert_eq!(none[0].floor_unusable, 0); } #[test] @@ -5517,7 +5522,7 @@ fn relower_skips_and_counts_an_item_the_round_refused_before_admission() { answers.insert("item:daodejing:3".into(), "not a draft".into()); }); let (mut rec, mut arm) = (Lowerer::new(blind), Lowerer::new(blind)); - let (langs, _) = relowered(relower(&dao, &mut rec, &mut arm, None)); + let (langs, _) = relowered(relower(&dao, &mut rec, &mut arm, &mut [])); let l = &langs[0]; assert_eq!((l.n, l.skipped, l.unusable), (2, 1, 0), "{}", l.line()); assert!(l.line().starts_with("relower en-US n=2 skipped=1 unusable=0 "), "{}", l.line()); @@ -5536,7 +5541,7 @@ fn relower_skips_and_counts_an_item_whose_lowering_prompt_the_round_refused() { let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); assert_eq!(r.language("ja-JP").unwrap().validation_refused.lower, 1); let (mut rec, mut arm) = (Lowerer::new(bi_blind), Lowerer::new(bi_blind)); - let (langs, _) = relowered(relower(&dao, &mut rec, &mut arm, None)); + let (langs, _) = relowered(relower(&dao, &mut rec, &mut arm, &mut [])); let ja = langs.iter().find(|l| l.language == "ja-JP").unwrap(); assert_eq!((ja.n, ja.skipped), (1, 1), "{}", ja.line()); assert_eq!(ja.recorded, ja.arm); @@ -5579,7 +5584,7 @@ fn relower_decides_over_the_whole_panel_not_its_first_language() { &dao, &mut Lowerer::new(bi_reader), &mut Lowerer::new(bi_ja_lost), - Some(&mut Lowerer::new(bi_reader)), + &mut [&mut Lowerer::new(bi_reader), &mut Lowerer::new(bi_reader), &mut Lowerer::new(bi_reader)], )); assert_eq!( langs.iter().map(|l| (l.language.as_str(), l.n)).collect::>(), @@ -5603,16 +5608,16 @@ fn relower_refuses_a_cycle_round_and_a_missing_recorded_answer() { run_cycle(&mut dao, &mut cycle_fake(ONE_CONCEPT)); let candidate = dao.run.rounds().unwrap().into_iter().find(|r| r.cycle().is_some()).unwrap().n(); let mut arm = Lowerer::new(reader); - let r = dao.run.relower(candidate, "arm", &mut Lowerer::new(reader), &mut arm, None, &Scorer::new(Bag)); + let r = dao.run.relower(candidate, "arm", &mut Lowerer::new(reader), &mut arm, &mut [], &Scorer::new(Bag)); assert!(matches!(r, Err(RunError::Malformed("round")))); - let r = dao.run.relower(99, "arm", &mut Lowerer::new(reader), &mut arm, None, &Scorer::new(Bag)); + let r = dao.run.relower(99, "arm", &mut Lowerer::new(reader), &mut arm, &mut [], &Scorer::new(Bag)); assert!(matches!(r, Err(RunError::Malformed("round")))); assert!(arm.seen.is_empty()); let (dao, _) = recorded_dao(|_| {}); let mut empty = RecordedAnswers::new(dao.root.join("round-1/nowhere"), "fake-model".into()); let mut arm = Lowerer::new(reader); - let r = relower(&dao, &mut empty, &mut arm, None); + let r = relower(&dao, &mut empty, &mut arm, &mut []); assert!(matches!(r, Err(RunError::Unrecorded))); assert!(arm.seen.is_empty(), "nothing is asked of the arm before the round's answer exists"); } @@ -5622,7 +5627,12 @@ fn relower_floor_arm_with_an_unanswered_prompt_is_refused() { use weksa::model::RecordedAnswers; let (dao, _) = recorded_dao(|_| {}); let mut floor = RecordedAnswers::new(dao.root.join("relower-1-floor"), "fake-model".into()); - let r = relower(&dao, &mut Lowerer::new(reader), &mut Lowerer::new(reader), Some(&mut floor)); + let r = relower( + &dao, + &mut Lowerer::new(reader), + &mut Lowerer::new(reader), + &mut [&mut floor, &mut Lowerer::new(reader), &mut Lowerer::new(reader)], + ); assert!(matches!(r, Err(RunError::FloorUnanswered))); assert!(!dao.root.join("relower-1-floor").exists(), "a floor arm is never written to"); } @@ -5640,9 +5650,9 @@ fn relower_checks_the_embedder_not_the_model() { } let (dao, _) = recorded_dao(|_| {}); let mut other = Named(Lowerer::new(blind)); - relowered(relower(&dao, &mut Lowerer::new(reader), &mut other, None)); + relowered(relower(&dao, &mut Lowerer::new(reader), &mut other, &mut [])); let mut arm = Lowerer::new(reader); - let r = dao.run.relower(1, "arm", &mut Lowerer::new(reader), &mut arm, None, &Scorer::new(OtherEmbedder)); + let r = dao.run.relower(1, "arm", &mut Lowerer::new(reader), &mut arm, &mut [], &Scorer::new(OtherEmbedder)); assert!(matches!(r, Err(RunError::Drift(DriftOf::Embedder)))); assert!(arm.seen.is_empty()); } @@ -5676,7 +5686,7 @@ fn relower_records_nothing() { // An unanswered arm writes its prompts in its own directory only. let mut files = dao.run.answer_files(dao.run.relower_dir(1, "arm")).unwrap(); // The recorded answers are files here, as the relay leaves them. - let r = relower(&dao, &mut recorded, &mut files, None); + let r = relower(&dao, &mut recorded, &mut files, &mut []); assert!(matches!(r, Err(RunError::Unrecorded)), "no round answer files yet"); let after = tree(&dao.root); for (path, bytes) in &before { @@ -5699,7 +5709,7 @@ fn relower_records_nothing() { let before = tree(&dao.root); let mut recorded = RecordedAnswers::new(dao.root.join("round-1"), "fake-model".into()); let mut files = dao.run.answer_files(dao.run.relower_dir(1, "arm")).unwrap(); - let RelowerOutcome::Unanswered { prompts } = relower(&dao, &mut recorded, &mut files, None).unwrap() else { + let RelowerOutcome::Unanswered { prompts } = relower(&dao, &mut recorded, &mut files, &mut []).unwrap() else { panic!("the arm is unanswered") }; assert_eq!(prompts.len(), 3); @@ -5727,7 +5737,7 @@ fn relower_arm_prompt_holding_test_text_is_withheld() { |_| {}, ); let mut files = dao.run.answer_files(dao.run.relower_dir(1, "arm")).unwrap(); - let r = relower(&dao, &mut Lowerer::new(reader), &mut files, None); + let r = relower(&dao, &mut Lowerer::new(reader), &mut files, &mut []); assert!(matches!(r, Err(RunError::Withheld)), "{:?}", r.map(|_| ())); let dir = dao.run.relower_dir(1, "arm"); let prompts = std::fs::read_dir(&dir) diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 4a78abe..3f07521 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -359,7 +359,7 @@ the training set is the whole corpus re-encoded under the frozen revision. A lowerer is compared on round n's Validation packets with the prompts byte-identical to the round's, by `weksa loop relower ROOT --round N --arm ARM ---lowerer LABEL [--floor ARM2]`. Every arm is answered through the one prompt +--lowerer LABEL [--floor ARM2 --floor ARM3 --floor ARM4]`. Every arm is answered through the one prompt door (`RunStores::answer_files`, the Test-text guard); a tuned model is a script that reads `.prompt` and writes `.answer` in `/relower--`, not a new adapter. The recorded reading is the @@ -369,15 +369,26 @@ ask: it is skipped, so the readings exclude it, and it is counted as `skipped=` on the line; `n` is the items read, so `recorded` can differ from the round's own Validation reading, which scores those items 0. An arm answer that is not a usable lowering scores 0 and counts in `unusable=`; the same for the floor arm, -counted after `floor ... unusable=`. The floor is the spread of the recorded -lowering and a second prompted arm (lowering-only noise, no encoder noise). The -verdict is `decide_panel` over (recorded, arm, floor): Accepted is the rule's -name for a gain past the floor, Rejected(Regressed) for a loss past it, -Rejected(NoGain) for neither. None is yet a finding about the arm. The rule is calibration-pending: a one-pair floor is noisy -(a simulated prompted repeat reads NoGain only about 7-15% of the time, Regressed -44-60%, Accepted 33-41%), so NoGain is not yet evidence of non-inferiority and -is not yet the bar for cut 13; the calibration arms (a prompted repeat, a -control) decide what the floor must be before the verdict is read as a ruling. +counted (over all floor arms) after `floor ... unusable=`. The floor needs three +or more floor arms, each a prompted repeat of the recorded lowering on the same +prompts: it is `FLOOR_Z` (3) standard errors of the mean per-item difference +between the recorded lowering and a repeat, per metric, with the variances +pooled over the arms (`paired_floor`; lowering-only noise, no encoder noise). The +verdict is `decide_panel` over (recorded, arm, floor): Accepted is a composite +gain past the noise of the panel's sum (the root of the languages' composite +floors squared, at least one step), Rejected(Regressed) a loss past a language's +floor, Rejected(NoGain) neither. The cycle's own floor is still the spread of two +round replicates (`floor_of`); it has no per-item scores to pair. +Calibration (`runs/distil-base`, 7 prompted draws, 14 en-US and 14 ja-JP +Validation items; every base, candidate and 3-arm floor combination, 420 trials; +intervals bootstrap over draws): a prompted repeat reads NoGain 92% (80 to 100), +Regressed 8% (0 to 20), Accepted 0%; an all-"Yes." arm reads Regressed 100%; a +mild degradation (one detail blurred per utterance) 99% (95 to 100); a subtler +one 54% (15 to 95). At 14 items a degradation under about three standard errors +(a composite near 0.06) cannot be told from noise, and no arm known to be better +exists, so the rule's power to accept a real gain is unmeasured. `FLOOR_Z` was +chosen on this data (2.5 gives NoGain 85% and 76% on the subtler arm), so the +rates are in-sample. Training pairs are Train only, at a frozen revision; the export is 12b. ## Lowerer first cut (`golf-loop-lowerer-cut`)