diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index ff444d8..2d85215 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -1299,11 +1299,17 @@ fn main() -> Result<()> { run.labelled(&format!("calibrate class={} prompted-{side} {}", c.name, k.line())) ); } - for (label, d) in degraded.iter().zip(&t.degraded) { + for ((label, d), offsets) in degraded.iter().zip(&t.degraded).zip(&t.degraded_offset) { + let offsets: String = c + .languages + .iter() + .zip(offsets) + .map(|(l, o)| format!(" offset {}={o:+.4}", l.language)) + .collect(); println!( "{}", run.labelled(&format!( - "calibrate class={} degraded={label} trials={} regressed={}", + "calibrate class={} degraded={label} trials={} regressed={}{offsets}", c.name, d.trials, d.regressed )) ); @@ -1315,6 +1321,12 @@ fn main() -> Result<()> { c.name, t.candidate.trials, t.candidate.accepted, t.candidate.nogain, t.candidate.regressed )) ); + for (label, k) in candidate.iter().zip(&t.candidate_arms) { + println!( + "{}", + run.labelled(&format!("calibrate class={} candidate={label} {}", c.name, k.line())) + ); + } } } } diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 32b29bd..732930f 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -1918,8 +1918,15 @@ pub struct Trials { pub r_free: Counts, /// Per degraded arm in the order given: the arm as the candidate. pub degraded: Vec, - /// Every candidate arm as the candidate, pooled. + /// Every candidate arm as the candidate, pooled: the sum of `candidate_arms`. pub candidate: Counts, + /// Per candidate arm in the order given: the arm as the candidate. Arms of + /// one kind can pool unlike draws, so the pooled count is not a power. + pub candidate_arms: Vec, + /// Per degraded arm in the order given, per language: the arm's composite + /// minus the mean composite of the prompted draws (the recorded lowering + /// included), over the items read. NaN where a language has no items. + pub degraded_offset: Vec>, /// The mean of [`panel_noise`] over every (base, floor) choice. pub noise: f64, } @@ -1975,7 +1982,7 @@ pub fn calibration_trials( let pool: Vec = (0..prompted).collect(); let without = |out: &[usize]| -> Vec { pool.iter().copied().filter(|p| !out.contains(p)).collect() }; let (mut r_candidate, mut r_base, mut r_free) = (Counts::default(), Counts::default(), Counts::default()); - let mut candidate_counts = Counts::default(); + let mut candidate_arms = vec![Counts::default(); candidate]; let mut degraded_counts = vec![Counts::default(); degraded]; let (mut noise, mut noises) = (0.0, 0u32); for &b in &pool { @@ -1984,7 +1991,7 @@ pub fn calibration_trials( counts.tally(decide_panel(&rows(b, prompted + d, &floors))); } for c in prompted + degraded..prompted + degraded + candidate { - candidate_counts.tally(decide_panel(&rows(b, c, &floors))); + candidate_arms[c - prompted - degraded].tally(decide_panel(&rows(b, c, &floors))); } noise += panel_noise(&rows(b, b, &floors)); noises += 1; @@ -2002,7 +2009,20 @@ pub fn calibration_trials( r_base, r_free, degraded: degraded_counts, - candidate: candidate_counts, + candidate: candidate_arms.iter().fold(Counts::default(), |a, &k| a.plus(k)), + candidate_arms, + degraded_offset: (0..degraded) + .map(|d| { + (0..languages) + .map(|l| { + let composite = |p: usize| { + reading_of(&draws[p][l]).map_or(f64::NAN, |r| r.composite) + }; + composite(prompted + d) - pool.iter().map(|&p| composite(p)).sum::() / prompted as f64 + }) + .collect() + }) + .collect(), noise: noise / f64::from(noises), } } @@ -3114,6 +3134,13 @@ mod tests { // A far better draw is accepted in every trial. assert_eq!(t.candidate.accepted, t.candidate.trials); + // Per-arm candidate counts sum to the pooled ones; a degraded draw of + // all zeros sits below the prompted mean (0.6 chrF and meaning) in both languages. + assert_eq!(t.candidate_arms.len(), 3); + assert_eq!(t.candidate_arms.iter().map(|k| k.trials).sum::(), t.candidate.trials); + assert_eq!(t.candidate_arms.iter().fold(Counts::default(), |a, &k| a.plus(k)), t.candidate); + assert_eq!((t.degraded_offset.len(), t.degraded_offset[0].len()), (2, 2)); + assert!(t.degraded_offset.iter().flatten().all(|o| (*o + 0.6).abs() < 0.05), "{:?}", t.degraded_offset); // R (port 0) scores above the other prompted draws: it is Accepted as a // candidate and Regressed as a base, and the trials without it read NoGain. let mut r_high: Vec<_> = (0..6).map(|s| draw(2, 8, if s == 0 { 0.8 } else { 0.6 }, 0.02, s)).collect(); diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 33cd6a3..7f624b8 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -1914,7 +1914,7 @@ fn cli_loop_calibrate_reads_answered_draws_and_refuses_unanswered_and_bad_labels .collect(); assert_eq!(digests.len(), 2); // `P1` and `a_b` are answered too, so only the label check can refuse them. - for label in ["p1", "p2", "p3", "p4", "bad", "P1", "a_b"] { + for label in ["p1", "p2", "p3", "p4", "bad", "cand1", "cand2", "P1", "a_b"] { std::fs::create_dir_all(root.join(format!("relower-1-{label}"))).unwrap(); for digest in &digests { std::fs::write(root.join(format!("relower-1-{label}/{digest}.answer")), said).unwrap(); @@ -1947,16 +1947,26 @@ fn cli_loop_calibrate_reads_answered_draws_and_refuses_unanswered_and_bad_labels assert!(!out(&o).contains("unanswered"), "{args:?} must be refused for itself: {}", out(&o)); } - let o = calibrate(&[&four[..], &["--degraded", "bad", "--class", "old=item:daodejing"]].concat()); + let o = calibrate( + &[&four[..], &["--degraded", "bad", "--candidate", "cand1", "--candidate", "cand2", "--class", "old=item:daodejing"]].concat(), + ); assert!(o.status.success(), "{}", out(&o)); let text = out(&o); - assert!(text.starts_with("calibrate draws prompted=4 degraded=1 candidate=0 floor-arms=3 unusable=0"), "{text}"); + assert!(text.starts_with("calibrate draws prompted=4 degraded=1 candidate=2 floor-arms=3 unusable=0"), "{text}"); assert!(text.contains("calibrate class=all items en-US n=2 skipped=0"), "{text}"); assert!(text.contains("calibrate class=all prompted trials=20 nogain=20 accepted=0 regressed=0"), "{text}"); for side in ["r-candidate trials=4", "r-base trials=4", "r-free trials=12"] { assert!(text.contains(&format!("calibrate class=all prompted-{side} nogain=")), "{text}"); } - assert!(text.contains("calibrate class=old degraded=bad trials=20 regressed=0"), "{text}"); + assert!(text.contains("calibrate class=old degraded=bad trials=20 regressed=0 offset en-US=+0.0000"), "{text}"); + // The class=all lines in print order: prompted, its three R roles, degraded, pooled candidate, each candidate arm. + let all: Vec<&str> = text.lines().filter(|l| l.contains("calibrate class=all ")).collect(); + let kinds = ["items", "prompted trials", "prompted-r-candidate", "prompted-r-base", "prompted-r-free", "degraded=bad", "candidate trials", "candidate=cand1", "candidate=cand2"]; + assert_eq!(all.len(), kinds.len(), "{text}"); + for (line, kind) in all.iter().zip(kinds) { + assert!(line.contains(&format!("class=all {kind}")), "{kind}: {line}"); + } + assert!(all[7].contains("candidate=cand1 trials=20 nogain=20 accepted=0 regressed=0"), "{}", all[7]); for leaked in ["item:daodejing", "packet:", "highest excellence", "canaryroot"] { assert!(!text.contains(leaked), "{text}"); } diff --git a/docs/calibration.md b/docs/calibration.md index 128b669..e1888e5 100644 --- a/docs/calibration.md +++ b/docs/calibration.md @@ -11,10 +11,12 @@ Regressed counts, and the mean panel noise. That is all it produces: the Accepted/NoGain/Regressed counts and the noise mean in the map's "Recalibration on the grown corpus" block are reproduced by it, and the prompted trials are also printed split by the recorded lowering `R`'s role (`prompted-r-candidate`, -`prompted-r-base`, `prompted-r-free`, see below); the other figures there (the +`prompted-r-base`, `prompted-r-free`, see below), the candidate counts per arm +and the mild arms' composite offsets from the prompted mean (see below); the +other figures there (the 35% with R removed, the noise range, the 0.0185 gain and its 0.41 ratio, the bootstrap intervals, the per-item SDs, R's +0.024, the mild-arm duplicate -counts and mean composite offsets, the Train-vocabulary overlap, the old/new +counts, the Train-vocabulary overlap, the old/new class composites and the ten parse-refused item ids) were computed by scratch scripts that are not in the repo. It is a diagnostic: it records nothing, writes no prompts, and never touches Test or Train. @@ -44,7 +46,14 @@ draws; the floor size is `FLOOR_ARMS_MIN` (3), the live rule's. 3-subset of the rest of `P`, the degraded draw as the candidate. These should read Regressed. - Candidate trials, pooled over the `--candidate` arms: the same, tallied - Accepted, NoGain, Regressed. This is the rule's power against that arm. + Accepted, NoGain, Regressed. This is the rule's power against that arm. The + pooled line is followed by one `candidate=LABEL` line per arm: arms of one + kind can differ completely (Train-guide h1 and h2 Regressed in every trial + while h3 and h4 gained), and then the pooled figure is not a power. +- Mild offset: each `degraded=LABEL` line ends with `offset LANGUAGE=+x.xxxx`, the + arm's composite minus the mean composite of the prompted draws (`R` included), + per language over the items read. A degraded arm that is barely below the + prompted mean is a weak probe of the rule's sensitivity. - Noise: the mean `panel_noise` over every (base, floor) choice. With six prompted draws including `R` that is 120 prompted trials, 60 per