From 36297f8d971aa4e4686eb2c0f4adaffca47217f4 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 14:53:42 +0000 Subject: [PATCH] calibrate: per-arm candidate counts and mild-arm offsets (#82) --- crates/weksa/src/main.rs | 16 ++-- crates/weksa/src/run.rs | 95 +++++++++++++++++++++++- crates/weksa/tests/cli.rs | 18 ++++- docs/calibration.md | 15 +++- docs/targets/interlingua-lowering-map.md | 8 +- 5 files changed, 131 insertions(+), 21 deletions(-) diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index ff444d8..e6a6cbb 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -1299,14 +1299,8 @@ fn main() -> Result<()> { run.labelled(&format!("calibrate class={} prompted-{side} {}", c.name, k.line())) ); } - for (label, d) in degraded.iter().zip(&t.degraded) { - println!( - "{}", - run.labelled(&format!( - "calibrate class={} degraded={label} trials={} regressed={}", - c.name, d.trials, d.regressed - )) - ); + for line in c.degraded_lines(°raded) { + println!("{}", run.labelled(&format!("calibrate class={} {line}", c.name))); } println!( "{}", @@ -1315,6 +1309,12 @@ fn main() -> Result<()> { c.name, t.candidate.trials, t.candidate.accepted, t.candidate.nogain, t.candidate.regressed )) ); + for (label, k) in candidate.iter().zip(&t.candidate_arms) { + println!( + "{}", + run.labelled(&format!("calibrate class={} candidate={label} {}", c.name, k.line())) + ); + } } } } diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 32b29bd..0620864 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -1918,8 +1918,15 @@ pub struct Trials { pub r_free: Counts, /// Per degraded arm in the order given: the arm as the candidate. pub degraded: Vec, - /// Every candidate arm as the candidate, pooled. + /// Every candidate arm as the candidate, pooled: the sum of `candidate_arms`. pub candidate: Counts, + /// Per candidate arm in the order given: the arm as the candidate. Arms of + /// one kind can pool unlike draws, so the pooled count is not a power. + pub candidate_arms: Vec, + /// Per degraded arm in the order given, per language: the arm's composite + /// minus the mean composite of the prompted draws (the recorded lowering + /// included), over the items read. NaN where a language has no items. + pub degraded_offset: Vec>, /// The mean of [`panel_noise`] over every (base, floor) choice. pub noise: f64, } @@ -1975,7 +1982,7 @@ pub fn calibration_trials( let pool: Vec = (0..prompted).collect(); let without = |out: &[usize]| -> Vec { pool.iter().copied().filter(|p| !out.contains(p)).collect() }; let (mut r_candidate, mut r_base, mut r_free) = (Counts::default(), Counts::default(), Counts::default()); - let mut candidate_counts = Counts::default(); + let mut candidate_arms = vec![Counts::default(); candidate]; let mut degraded_counts = vec![Counts::default(); degraded]; let (mut noise, mut noises) = (0.0, 0u32); for &b in &pool { @@ -1984,7 +1991,7 @@ pub fn calibration_trials( counts.tally(decide_panel(&rows(b, prompted + d, &floors))); } for c in prompted + degraded..prompted + degraded + candidate { - candidate_counts.tally(decide_panel(&rows(b, c, &floors))); + candidate_arms[c - prompted - degraded].tally(decide_panel(&rows(b, c, &floors))); } noise += panel_noise(&rows(b, b, &floors)); noises += 1; @@ -2002,7 +2009,20 @@ pub fn calibration_trials( r_base, r_free, degraded: degraded_counts, - candidate: candidate_counts, + candidate: candidate_arms.iter().fold(Counts::default(), |a, &k| a.plus(k)), + candidate_arms, + degraded_offset: (0..degraded) + .map(|d| { + (0..languages) + .map(|l| { + let composite = |p: usize| { + reading_of(&draws[p][l]).map_or(f64::NAN, |r| r.composite) + }; + composite(prompted + d) - pool.iter().map(|&p| composite(p)).sum::() / prompted as f64 + }) + .collect() + }) + .collect(), noise: noise / f64::from(noises), } } @@ -2024,6 +2044,25 @@ pub struct ClassTrials { pub trials: Trials, } +impl ClassTrials { + /// The text after `calibrate class=NAME ` of each degraded arm's line, one + /// per `labels` entry: its Regressed count and its composite offset from the + /// prompted mean in each language of the class. + pub fn degraded_lines(&self, labels: &[String]) -> Vec { + let t = &self.trials; + labels + .iter() + .zip(&t.degraded) + .zip(&t.degraded_offset) + .map(|((label, d), offsets)| { + let offsets: String = + self.languages.iter().zip(offsets).map(|(l, o)| format!(" offset {}={o:+.4}", l.language)).collect(); + format!("degraded={label} trials={} regressed={}{offsets}", d.trials, d.regressed) + }) + .collect() + } +} + #[derive(Debug)] pub enum CalibrateOutcome { /// Draw labels with the number of prompts each has no answer to. @@ -3095,6 +3134,47 @@ mod tests { .collect() } + /// A wobble-free draw whose items all score `bases[language]` (chrF as a percentage, meaning as a fraction). + fn flat(bases: [f64; 2], items: usize) -> Vec> { + bases.iter().map(|&b| (0..items).map(|_| scored(100.0 * b, b)).collect()).collect() + } + + #[test] + fn arm_reports_attribute_each_arm_and_language() { + // R (port 0) is above the five other prompted draws; the two languages differ everywhere. + let mut draws = vec![flat([0.8, 0.7], 8)]; + draws.extend((0..5).map(|_| flat([0.6, 0.6], 8))); + // Degraded arms: composites differ per arm and per language. + draws.push(flat([0.5, 0.2], 8)); + draws.push(flat([0.1, 0.4], 8)); + // Candidate arms: far above, level with the prompted, far below. + draws.push(flat([0.95, 0.95], 8)); + draws.push(flat([0.6, 0.6], 8)); + draws.push(flat([0.05, 0.05], 8)); + let t = calibration_trials(&draws, 6, 2, 3); + // Offsets: arm minus the mean over all six prompted ports (R included): en 0.6333, ja 0.6167. + let want = [[0.5 - 3.8 / 6.0, 0.2 - 3.7 / 6.0], [0.1 - 3.8 / 6.0, 0.4 - 3.7 / 6.0]]; + for (got, want) in t.degraded_offset.iter().flatten().zip(want.iter().flatten()) { + assert!((got - want).abs() < 1e-9, "{:?} against {want:?}", t.degraded_offset); + } + // Each arm keeps its own counts: the high arm gains, the level one gains nothing, the low one regresses. + let [high, level, low] = [t.candidate_arms[0], t.candidate_arms[1], t.candidate_arms[2]]; + assert_eq!((high.trials, level.trials, low.trials), (60, 60, 60)); + assert_eq!(high.accepted, 60); + assert_eq!(level.accepted, 0); + assert_eq!(low.regressed, 60); + assert_eq!(t.candidate, high.plus(level).plus(low)); + // The printed lines keep each arm with its label and each offset with its language. + let class = ClassTrials { + name: "all".into(), + languages: ["en-US", "ja-JP"].map(|l| ClassLanguage { language: l.into(), n: 8, skipped: 0 }).into(), + trials: t, + }; + let lines = class.degraded_lines(&["mild-1".to_string(), "mild-2".to_string()]); + assert_eq!(lines[0], "degraded=mild-1 trials=60 regressed=60 offset en-US=-0.1333 offset ja-JP=-0.4167"); + assert_eq!(lines[1], "degraded=mild-2 trials=60 regressed=60 offset en-US=-0.5333 offset ja-JP=-0.2167"); + } + #[test] fn calibration_trials_count_every_choice_and_read_the_fixed_cases() { // Six prompted draws, two degraded, three candidates, two languages of 8 items. @@ -3114,6 +3194,13 @@ mod tests { // A far better draw is accepted in every trial. assert_eq!(t.candidate.accepted, t.candidate.trials); + // Per-arm candidate counts sum to the pooled ones; a degraded draw of + // all zeros sits below the prompted mean (0.6 chrF and meaning) in both languages. + assert_eq!(t.candidate_arms.len(), 3); + assert_eq!(t.candidate_arms.iter().map(|k| k.trials).sum::(), t.candidate.trials); + assert_eq!(t.candidate_arms.iter().fold(Counts::default(), |a, &k| a.plus(k)), t.candidate); + assert_eq!((t.degraded_offset.len(), t.degraded_offset[0].len()), (2, 2)); + assert!(t.degraded_offset.iter().flatten().all(|o| (*o + 0.6).abs() < 0.05), "{:?}", t.degraded_offset); // R (port 0) scores above the other prompted draws: it is Accepted as a // candidate and Regressed as a base, and the trials without it read NoGain. let mut r_high: Vec<_> = (0..6).map(|s| draw(2, 8, if s == 0 { 0.8 } else { 0.6 }, 0.02, s)).collect(); diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 33cd6a3..7f624b8 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -1914,7 +1914,7 @@ fn cli_loop_calibrate_reads_answered_draws_and_refuses_unanswered_and_bad_labels .collect(); assert_eq!(digests.len(), 2); // `P1` and `a_b` are answered too, so only the label check can refuse them. - for label in ["p1", "p2", "p3", "p4", "bad", "P1", "a_b"] { + for label in ["p1", "p2", "p3", "p4", "bad", "cand1", "cand2", "P1", "a_b"] { std::fs::create_dir_all(root.join(format!("relower-1-{label}"))).unwrap(); for digest in &digests { std::fs::write(root.join(format!("relower-1-{label}/{digest}.answer")), said).unwrap(); @@ -1947,16 +1947,26 @@ fn cli_loop_calibrate_reads_answered_draws_and_refuses_unanswered_and_bad_labels assert!(!out(&o).contains("unanswered"), "{args:?} must be refused for itself: {}", out(&o)); } - let o = calibrate(&[&four[..], &["--degraded", "bad", "--class", "old=item:daodejing"]].concat()); + let o = calibrate( + &[&four[..], &["--degraded", "bad", "--candidate", "cand1", "--candidate", "cand2", "--class", "old=item:daodejing"]].concat(), + ); assert!(o.status.success(), "{}", out(&o)); let text = out(&o); - assert!(text.starts_with("calibrate draws prompted=4 degraded=1 candidate=0 floor-arms=3 unusable=0"), "{text}"); + assert!(text.starts_with("calibrate draws prompted=4 degraded=1 candidate=2 floor-arms=3 unusable=0"), "{text}"); assert!(text.contains("calibrate class=all items en-US n=2 skipped=0"), "{text}"); assert!(text.contains("calibrate class=all prompted trials=20 nogain=20 accepted=0 regressed=0"), "{text}"); for side in ["r-candidate trials=4", "r-base trials=4", "r-free trials=12"] { assert!(text.contains(&format!("calibrate class=all prompted-{side} nogain=")), "{text}"); } - assert!(text.contains("calibrate class=old degraded=bad trials=20 regressed=0"), "{text}"); + assert!(text.contains("calibrate class=old degraded=bad trials=20 regressed=0 offset en-US=+0.0000"), "{text}"); + // The class=all lines in print order: prompted, its three R roles, degraded, pooled candidate, each candidate arm. + let all: Vec<&str> = text.lines().filter(|l| l.contains("calibrate class=all ")).collect(); + let kinds = ["items", "prompted trials", "prompted-r-candidate", "prompted-r-base", "prompted-r-free", "degraded=bad", "candidate trials", "candidate=cand1", "candidate=cand2"]; + assert_eq!(all.len(), kinds.len(), "{text}"); + for (line, kind) in all.iter().zip(kinds) { + assert!(line.contains(&format!("class=all {kind}")), "{kind}: {line}"); + } + assert!(all[7].contains("candidate=cand1 trials=20 nogain=20 accepted=0 regressed=0"), "{}", all[7]); for leaked in ["item:daodejing", "packet:", "highest excellence", "canaryroot"] { assert!(!text.contains(leaked), "{text}"); } diff --git a/docs/calibration.md b/docs/calibration.md index 128b669..e1888e5 100644 --- a/docs/calibration.md +++ b/docs/calibration.md @@ -11,10 +11,12 @@ Regressed counts, and the mean panel noise. That is all it produces: the Accepted/NoGain/Regressed counts and the noise mean in the map's "Recalibration on the grown corpus" block are reproduced by it, and the prompted trials are also printed split by the recorded lowering `R`'s role (`prompted-r-candidate`, -`prompted-r-base`, `prompted-r-free`, see below); the other figures there (the +`prompted-r-base`, `prompted-r-free`, see below), the candidate counts per arm +and the mild arms' composite offsets from the prompted mean (see below); the +other figures there (the 35% with R removed, the noise range, the 0.0185 gain and its 0.41 ratio, the bootstrap intervals, the per-item SDs, R's +0.024, the mild-arm duplicate -counts and mean composite offsets, the Train-vocabulary overlap, the old/new +counts, the Train-vocabulary overlap, the old/new class composites and the ten parse-refused item ids) were computed by scratch scripts that are not in the repo. It is a diagnostic: it records nothing, writes no prompts, and never touches Test or Train. @@ -44,7 +46,14 @@ draws; the floor size is `FLOOR_ARMS_MIN` (3), the live rule's. 3-subset of the rest of `P`, the degraded draw as the candidate. These should read Regressed. - Candidate trials, pooled over the `--candidate` arms: the same, tallied - Accepted, NoGain, Regressed. This is the rule's power against that arm. + Accepted, NoGain, Regressed. This is the rule's power against that arm. The + pooled line is followed by one `candidate=LABEL` line per arm: arms of one + kind can differ completely (Train-guide h1 and h2 Regressed in every trial + while h3 and h4 gained), and then the pooled figure is not a power. +- Mild offset: each `degraded=LABEL` line ends with `offset LANGUAGE=+x.xxxx`, the + arm's composite minus the mean composite of the prompted draws (`R` included), + per language over the items read. A degraded arm that is barely below the + prompted mean is a weak probe of the rule's sensitivity. - Noise: the mean `panel_noise` over every (base, floor) choice. With six prompted draws including `R` that is 120 prompted trials, 60 per diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 6b1ecb1..aa92332 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -548,7 +548,11 @@ except the R split in the first bullet, which `calibrate` prints as `prompted-r-candidate`, `prompted-r-base` and `prompted-r-free` since the `calibrate-report-r-involvement` cut; Soul re-ran growth-scene with it and the printed `all` lines read r-candidate 11/9/0, r-base 0/11/9, r-free 0/80/0, as -below, in Accepted/NoGain/Regressed): +below, in Accepted/NoGain/Regressed; likewise the per-arm Train-guide counts +(`candidate=h1..h4`) and the mild arms' composite offsets from the prompted +mean, R included (`offset`), which `calibrate` prints since +`calibrate-arm-counts-and-mild-offsets` and which reproduce the figures below, +Soul re-run on growth-scene): - Every one of the 11 false accepts has R as the candidate (bases prompted-2, p4, p6); every one of the 9 false rejects has R as the base. The @@ -568,7 +572,7 @@ below, in Accepted/NoGain/Regressed): alone can produce the gap. - Mild arms: they overlap 8 of 35 answers (en-US) and 9 (ja-JP) over the limit of 7. They do not degrade the new items (composite -0.004 en-US - [-0.021, +0.012], +0.013 ja-JP); the Regressed counts come from the old + [-0.021, +0.012], +0.013 ja-JP for mild-1, +0.005 for mild-2); the Regressed counts come from the old en-US items (-0.072). The rule's sensitivity on the new items is unmeasured. - Train-guide pools unlike arms. h1 and h2 Regress in 120 of 120 trials; h3 and h4 gain (Accepted 45 and 40 of 60); on the new items h3 is Accepted 50 of 60