From 064483dba7015485f42c738cae485b30c8606f53 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 13:54:22 +0000 Subject: [PATCH 1/3] calibrate: per-arm candidate counts, mild-arm composite offsets, line order pinned Trials gains candidate_arms (pooled candidate is their sum) and degraded_offset (arm composite minus the prompted mean, per language). The CLI test pins the order of the class lines and the per-arm candidate lines. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/main.rs | 16 ++++++++++++++-- crates/weksa/src/run.rs | 35 +++++++++++++++++++++++++++++++---- crates/weksa/tests/cli.rs | 18 ++++++++++++++---- docs/calibration.md | 15 ++++++++++++--- 4 files changed, 71 insertions(+), 13 deletions(-) diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index ff444d8..2d85215 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -1299,11 +1299,17 @@ fn main() -> Result<()> { run.labelled(&format!("calibrate class={} prompted-{side} {}", c.name, k.line())) ); } - for (label, d) in degraded.iter().zip(&t.degraded) { + for ((label, d), offsets) in degraded.iter().zip(&t.degraded).zip(&t.degraded_offset) { + let offsets: String = c + .languages + .iter() + .zip(offsets) + .map(|(l, o)| format!(" offset {}={o:+.4}", l.language)) + .collect(); println!( "{}", run.labelled(&format!( - "calibrate class={} degraded={label} trials={} regressed={}", + "calibrate class={} degraded={label} trials={} regressed={}{offsets}", c.name, d.trials, d.regressed )) ); @@ -1315,6 +1321,12 @@ fn main() -> Result<()> { c.name, t.candidate.trials, t.candidate.accepted, t.candidate.nogain, t.candidate.regressed )) ); + for (label, k) in candidate.iter().zip(&t.candidate_arms) { + println!( + "{}", + run.labelled(&format!("calibrate class={} candidate={label} {}", c.name, k.line())) + ); + } } } } diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 32b29bd..732930f 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -1918,8 +1918,15 @@ pub struct Trials { pub r_free: Counts, /// Per degraded arm in the order given: the arm as the candidate. pub degraded: Vec, - /// Every candidate arm as the candidate, pooled. + /// Every candidate arm as the candidate, pooled: the sum of `candidate_arms`. pub candidate: Counts, + /// Per candidate arm in the order given: the arm as the candidate. Arms of + /// one kind can pool unlike draws, so the pooled count is not a power. + pub candidate_arms: Vec, + /// Per degraded arm in the order given, per language: the arm's composite + /// minus the mean composite of the prompted draws (the recorded lowering + /// included), over the items read. NaN where a language has no items. + pub degraded_offset: Vec>, /// The mean of [`panel_noise`] over every (base, floor) choice. pub noise: f64, } @@ -1975,7 +1982,7 @@ pub fn calibration_trials( let pool: Vec = (0..prompted).collect(); let without = |out: &[usize]| -> Vec { pool.iter().copied().filter(|p| !out.contains(p)).collect() }; let (mut r_candidate, mut r_base, mut r_free) = (Counts::default(), Counts::default(), Counts::default()); - let mut candidate_counts = Counts::default(); + let mut candidate_arms = vec![Counts::default(); candidate]; let mut degraded_counts = vec![Counts::default(); degraded]; let (mut noise, mut noises) = (0.0, 0u32); for &b in &pool { @@ -1984,7 +1991,7 @@ pub fn calibration_trials( counts.tally(decide_panel(&rows(b, prompted + d, &floors))); } for c in prompted + degraded..prompted + degraded + candidate { - candidate_counts.tally(decide_panel(&rows(b, c, &floors))); + candidate_arms[c - prompted - degraded].tally(decide_panel(&rows(b, c, &floors))); } noise += panel_noise(&rows(b, b, &floors)); noises += 1; @@ -2002,7 +2009,20 @@ pub fn calibration_trials( r_base, r_free, degraded: degraded_counts, - candidate: candidate_counts, + candidate: candidate_arms.iter().fold(Counts::default(), |a, &k| a.plus(k)), + candidate_arms, + degraded_offset: (0..degraded) + .map(|d| { + (0..languages) + .map(|l| { + let composite = |p: usize| { + reading_of(&draws[p][l]).map_or(f64::NAN, |r| r.composite) + }; + composite(prompted + d) - pool.iter().map(|&p| composite(p)).sum::() / prompted as f64 + }) + .collect() + }) + .collect(), noise: noise / f64::from(noises), } } @@ -3114,6 +3134,13 @@ mod tests { // A far better draw is accepted in every trial. assert_eq!(t.candidate.accepted, t.candidate.trials); + // Per-arm candidate counts sum to the pooled ones; a degraded draw of + // all zeros sits below the prompted mean (0.6 chrF and meaning) in both languages. + assert_eq!(t.candidate_arms.len(), 3); + assert_eq!(t.candidate_arms.iter().map(|k| k.trials).sum::(), t.candidate.trials); + assert_eq!(t.candidate_arms.iter().fold(Counts::default(), |a, &k| a.plus(k)), t.candidate); + assert_eq!((t.degraded_offset.len(), t.degraded_offset[0].len()), (2, 2)); + assert!(t.degraded_offset.iter().flatten().all(|o| (*o + 0.6).abs() < 0.05), "{:?}", t.degraded_offset); // R (port 0) scores above the other prompted draws: it is Accepted as a // candidate and Regressed as a base, and the trials without it read NoGain. let mut r_high: Vec<_> = (0..6).map(|s| draw(2, 8, if s == 0 { 0.8 } else { 0.6 }, 0.02, s)).collect(); diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 33cd6a3..7f624b8 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -1914,7 +1914,7 @@ fn cli_loop_calibrate_reads_answered_draws_and_refuses_unanswered_and_bad_labels .collect(); assert_eq!(digests.len(), 2); // `P1` and `a_b` are answered too, so only the label check can refuse them. - for label in ["p1", "p2", "p3", "p4", "bad", "P1", "a_b"] { + for label in ["p1", "p2", "p3", "p4", "bad", "cand1", "cand2", "P1", "a_b"] { std::fs::create_dir_all(root.join(format!("relower-1-{label}"))).unwrap(); for digest in &digests { std::fs::write(root.join(format!("relower-1-{label}/{digest}.answer")), said).unwrap(); @@ -1947,16 +1947,26 @@ fn cli_loop_calibrate_reads_answered_draws_and_refuses_unanswered_and_bad_labels assert!(!out(&o).contains("unanswered"), "{args:?} must be refused for itself: {}", out(&o)); } - let o = calibrate(&[&four[..], &["--degraded", "bad", "--class", "old=item:daodejing"]].concat()); + let o = calibrate( + &[&four[..], &["--degraded", "bad", "--candidate", "cand1", "--candidate", "cand2", "--class", "old=item:daodejing"]].concat(), + ); assert!(o.status.success(), "{}", out(&o)); let text = out(&o); - assert!(text.starts_with("calibrate draws prompted=4 degraded=1 candidate=0 floor-arms=3 unusable=0"), "{text}"); + assert!(text.starts_with("calibrate draws prompted=4 degraded=1 candidate=2 floor-arms=3 unusable=0"), "{text}"); assert!(text.contains("calibrate class=all items en-US n=2 skipped=0"), "{text}"); assert!(text.contains("calibrate class=all prompted trials=20 nogain=20 accepted=0 regressed=0"), "{text}"); for side in ["r-candidate trials=4", "r-base trials=4", "r-free trials=12"] { assert!(text.contains(&format!("calibrate class=all prompted-{side} nogain=")), "{text}"); } - assert!(text.contains("calibrate class=old degraded=bad trials=20 regressed=0"), "{text}"); + assert!(text.contains("calibrate class=old degraded=bad trials=20 regressed=0 offset en-US=+0.0000"), "{text}"); + // The class=all lines in print order: prompted, its three R roles, degraded, pooled candidate, each candidate arm. + let all: Vec<&str> = text.lines().filter(|l| l.contains("calibrate class=all ")).collect(); + let kinds = ["items", "prompted trials", "prompted-r-candidate", "prompted-r-base", "prompted-r-free", "degraded=bad", "candidate trials", "candidate=cand1", "candidate=cand2"]; + assert_eq!(all.len(), kinds.len(), "{text}"); + for (line, kind) in all.iter().zip(kinds) { + assert!(line.contains(&format!("class=all {kind}")), "{kind}: {line}"); + } + assert!(all[7].contains("candidate=cand1 trials=20 nogain=20 accepted=0 regressed=0"), "{}", all[7]); for leaked in ["item:daodejing", "packet:", "highest excellence", "canaryroot"] { assert!(!text.contains(leaked), "{text}"); } diff --git a/docs/calibration.md b/docs/calibration.md index 128b669..e1888e5 100644 --- a/docs/calibration.md +++ b/docs/calibration.md @@ -11,10 +11,12 @@ Regressed counts, and the mean panel noise. That is all it produces: the Accepted/NoGain/Regressed counts and the noise mean in the map's "Recalibration on the grown corpus" block are reproduced by it, and the prompted trials are also printed split by the recorded lowering `R`'s role (`prompted-r-candidate`, -`prompted-r-base`, `prompted-r-free`, see below); the other figures there (the +`prompted-r-base`, `prompted-r-free`, see below), the candidate counts per arm +and the mild arms' composite offsets from the prompted mean (see below); the +other figures there (the 35% with R removed, the noise range, the 0.0185 gain and its 0.41 ratio, the bootstrap intervals, the per-item SDs, R's +0.024, the mild-arm duplicate -counts and mean composite offsets, the Train-vocabulary overlap, the old/new +counts, the Train-vocabulary overlap, the old/new class composites and the ten parse-refused item ids) were computed by scratch scripts that are not in the repo. It is a diagnostic: it records nothing, writes no prompts, and never touches Test or Train. @@ -44,7 +46,14 @@ draws; the floor size is `FLOOR_ARMS_MIN` (3), the live rule's. 3-subset of the rest of `P`, the degraded draw as the candidate. These should read Regressed. - Candidate trials, pooled over the `--candidate` arms: the same, tallied - Accepted, NoGain, Regressed. This is the rule's power against that arm. + Accepted, NoGain, Regressed. This is the rule's power against that arm. The + pooled line is followed by one `candidate=LABEL` line per arm: arms of one + kind can differ completely (Train-guide h1 and h2 Regressed in every trial + while h3 and h4 gained), and then the pooled figure is not a power. +- Mild offset: each `degraded=LABEL` line ends with `offset LANGUAGE=+x.xxxx`, the + arm's composite minus the mean composite of the prompted draws (`R` included), + per language over the items read. A degraded arm that is barely below the + prompted mean is a weak probe of the rule's sensitivity. - Noise: the mean `panel_noise` over every (base, floor) choice. With six prompted draws including `R` that is 120 prompted trials, 60 per -- 2.47.3 From a85d410e7792ff5df5ba65200170b4b8656c49fe Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 14:49:31 +0000 Subject: [PATCH 2/3] calibrate: tests that tell arms and languages apart; map notes the harness now prints them Soul arms s1 finding: unit test could not distinguish arms, languages or the R-included mean. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/run.rs | 32 ++++++++++++++++++++++++ docs/targets/interlingua-lowering-map.md | 6 ++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 732930f..3e51faa 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -3115,6 +3115,38 @@ mod tests { .collect() } + /// A wobble-free draw whose items all score `bases[language]` (chrF as a percentage, meaning as a fraction). + fn flat(bases: [f64; 2], items: usize) -> Vec> { + bases.iter().map(|&b| (0..items).map(|_| scored(100.0 * b, b)).collect()).collect() + } + + #[test] + fn arm_reports_attribute_each_arm_and_language() { + // R (port 0) is above the five other prompted draws; the two languages differ everywhere. + let mut draws = vec![flat([0.8, 0.7], 8)]; + draws.extend((0..5).map(|_| flat([0.6, 0.6], 8))); + // Degraded arms: composites differ per arm and per language. + draws.push(flat([0.5, 0.2], 8)); + draws.push(flat([0.1, 0.4], 8)); + // Candidate arms: far above, level with the prompted, far below. + draws.push(flat([0.95, 0.95], 8)); + draws.push(flat([0.6, 0.6], 8)); + draws.push(flat([0.05, 0.05], 8)); + let t = calibration_trials(&draws, 6, 2, 3); + // Offsets: arm minus the mean over all six prompted ports (R included): en 0.6333, ja 0.6167. + let want = [[0.5 - 3.8 / 6.0, 0.2 - 3.7 / 6.0], [0.1 - 3.8 / 6.0, 0.4 - 3.7 / 6.0]]; + for (got, want) in t.degraded_offset.iter().flatten().zip(want.iter().flatten()) { + assert!((got - want).abs() < 1e-9, "{:?} against {want:?}", t.degraded_offset); + } + // Each arm keeps its own counts: the high arm gains, the level one gains nothing, the low one regresses. + let [high, level, low] = [t.candidate_arms[0], t.candidate_arms[1], t.candidate_arms[2]]; + assert_eq!((high.trials, level.trials, low.trials), (60, 60, 60)); + assert_eq!(high.accepted, 60); + assert_eq!(level.accepted, 0); + assert_eq!(low.regressed, 60); + assert_eq!(t.candidate, high.plus(level).plus(low)); + } + #[test] fn calibration_trials_count_every_choice_and_read_the_fixed_cases() { // Six prompted draws, two degraded, three candidates, two languages of 8 items. diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 6b1ecb1..d02bb49 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -548,7 +548,11 @@ except the R split in the first bullet, which `calibrate` prints as `prompted-r-candidate`, `prompted-r-base` and `prompted-r-free` since the `calibrate-report-r-involvement` cut; Soul re-ran growth-scene with it and the printed `all` lines read r-candidate 11/9/0, r-base 0/11/9, r-free 0/80/0, as -below, in Accepted/NoGain/Regressed): +below, in Accepted/NoGain/Regressed; likewise the per-arm Train-guide counts +(`candidate=h1..h4`) and the mild arms' composite offsets from the prompted +mean, R included (`offset`), which `calibrate` prints since +`calibrate-arm-counts-and-mild-offsets` and which reproduce the figures below, +Soul re-run on growth-scene): - Every one of the 11 false accepts has R as the candidate (bases prompted-2, p4, p6); every one of the 9 false rejects has R as the base. The -- 2.47.3 From e94563b0204033d35392cca7924ef0585d0df5f8 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 14:53:32 +0000 Subject: [PATCH 3/3] calibrate: degraded lines built in the lib and pinned by the arm-and-language test Soul arms s1 re-check: CLI-only mutants M7 and M10 (offsets zipped against the wrong language or label) survived. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/main.rs | 16 ++-------------- crates/weksa/src/run.rs | 28 ++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 2d85215..e6a6cbb 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -1299,20 +1299,8 @@ fn main() -> Result<()> { run.labelled(&format!("calibrate class={} prompted-{side} {}", c.name, k.line())) ); } - for ((label, d), offsets) in degraded.iter().zip(&t.degraded).zip(&t.degraded_offset) { - let offsets: String = c - .languages - .iter() - .zip(offsets) - .map(|(l, o)| format!(" offset {}={o:+.4}", l.language)) - .collect(); - println!( - "{}", - run.labelled(&format!( - "calibrate class={} degraded={label} trials={} regressed={}{offsets}", - c.name, d.trials, d.regressed - )) - ); + for line in c.degraded_lines(°raded) { + println!("{}", run.labelled(&format!("calibrate class={} {line}", c.name))); } println!( "{}", diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 3e51faa..0620864 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -2044,6 +2044,25 @@ pub struct ClassTrials { pub trials: Trials, } +impl ClassTrials { + /// The text after `calibrate class=NAME ` of each degraded arm's line, one + /// per `labels` entry: its Regressed count and its composite offset from the + /// prompted mean in each language of the class. + pub fn degraded_lines(&self, labels: &[String]) -> Vec { + let t = &self.trials; + labels + .iter() + .zip(&t.degraded) + .zip(&t.degraded_offset) + .map(|((label, d), offsets)| { + let offsets: String = + self.languages.iter().zip(offsets).map(|(l, o)| format!(" offset {}={o:+.4}", l.language)).collect(); + format!("degraded={label} trials={} regressed={}{offsets}", d.trials, d.regressed) + }) + .collect() + } +} + #[derive(Debug)] pub enum CalibrateOutcome { /// Draw labels with the number of prompts each has no answer to. @@ -3145,6 +3164,15 @@ mod tests { assert_eq!(level.accepted, 0); assert_eq!(low.regressed, 60); assert_eq!(t.candidate, high.plus(level).plus(low)); + // The printed lines keep each arm with its label and each offset with its language. + let class = ClassTrials { + name: "all".into(), + languages: ["en-US", "ja-JP"].map(|l| ClassLanguage { language: l.into(), n: 8, skipped: 0 }).into(), + trials: t, + }; + let lines = class.degraded_lines(&["mild-1".to_string(), "mild-2".to_string()]); + assert_eq!(lines[0], "degraded=mild-1 trials=60 regressed=60 offset en-US=-0.1333 offset ja-JP=-0.4167"); + assert_eq!(lines[1], "degraded=mild-2 trials=60 regressed=60 offset en-US=-0.5333 offset ja-JP=-0.2167"); } #[test] -- 2.47.3