diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 3cd2235..ff444d8 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -1293,6 +1293,12 @@ fn main() -> Result<()> { c.name, t.prompted.trials, t.prompted.nogain, t.prompted.accepted, t.prompted.regressed, t.noise )) ); + for (side, k) in t.r_roles() { + println!( + "{}", + run.labelled(&format!("calibrate class={} prompted-{side} {}", c.name, k.line())) + ); + } for (label, d) in degraded.iter().zip(&t.degraded) { println!( "{}", diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index b15fd07..32b29bd 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -1885,13 +1885,37 @@ impl Counts { Verdict::Rejected(_) => self.nogain += 1, } } + + /// The `trials=.. nogain=.. accepted=.. regressed=..` text of a calibrate line. + pub fn line(&self) -> String { + format!("trials={} nogain={} accepted={} regressed={}", self.trials, self.nogain, self.accepted, self.regressed) + } + + fn plus(self, o: Counts) -> Counts { + Counts { + trials: self.trials + o.trials, + accepted: self.accepted + o.accepted, + nogain: self.nogain + o.nogain, + regressed: self.regressed + o.regressed, + } + } } /// What [`calibration_trials`] counted over one set of items. #[derive(Clone, Debug, PartialEq)] pub struct Trials { - /// Ordered (base, candidate) pairs of prompted draws, each over every floor. + /// Ordered (base, candidate) pairs of prompted draws, each over every floor: + /// the sum of the three counts below. pub prompted: Counts, + /// The prompted trials whose candidate is the recorded lowering `R` (port 0). + /// `R` comes from the round, not from `relower`, so it is not exchangeable + /// with a repeat: every false accept of a grown-corpus calibration lived here. + pub r_candidate: Counts, + /// The prompted trials whose base is `R`. + pub r_base: Counts, + /// The prompted trials with `R` neither base nor candidate (it may sit in + /// the floor): the exchangeable repeats. + pub r_free: Counts, /// Per degraded arm in the order given: the arm as the candidate. pub degraded: Vec, /// Every candidate arm as the candidate, pooled. @@ -1900,6 +1924,14 @@ pub struct Trials { pub noise: f64, } +impl Trials { + /// The prompted trials by the recorded lowering `R`'s role, labelled as the + /// calibrate lines print them. + pub fn r_roles(&self) -> [(&'static str, Counts); 3] { + [("r-candidate", self.r_candidate), ("r-base", self.r_base), ("r-free", self.r_free)] + } +} + /// Every `k`-subset of `pool`, in order. fn subsets(pool: &[usize], k: usize) -> Vec> { if k == 0 { @@ -1942,7 +1974,8 @@ pub fn calibration_trials( }; let pool: Vec = (0..prompted).collect(); let without = |out: &[usize]| -> Vec { pool.iter().copied().filter(|p| !out.contains(p)).collect() }; - let (mut prompted_counts, mut candidate_counts) = (Counts::default(), Counts::default()); + let (mut r_candidate, mut r_base, mut r_free) = (Counts::default(), Counts::default(), Counts::default()); + let mut candidate_counts = Counts::default(); let mut degraded_counts = vec![Counts::default(); degraded]; let (mut noise, mut noises) = (0.0, 0u32); for &b in &pool { @@ -1958,12 +1991,16 @@ pub fn calibration_trials( } for &c in pool.iter().filter(|&&c| c != b) { for floors in subsets(&without(&[b, c]), FLOOR_ARMS_MIN) { - prompted_counts.tally(decide_panel(&rows(b, c, &floors))); + let side = if c == 0 { &mut r_candidate } else if b == 0 { &mut r_base } else { &mut r_free }; + side.tally(decide_panel(&rows(b, c, &floors))); } } } Trials { - prompted: prompted_counts, + prompted: r_candidate.plus(r_base).plus(r_free), + r_candidate, + r_base, + r_free, degraded: degraded_counts, candidate: candidate_counts, noise: noise / f64::from(noises), @@ -3066,6 +3103,9 @@ mod tests { draws.extend((0..3).map(|s| draw(2, 8, 0.8, 0.02, s))); let t = calibration_trials(&draws, 6, 2, 3); assert_eq!(t.prompted.trials, 120); + // R is port 0: 20 pairs with R as candidate, 20 as base, 20 between the other five, 4 floors each. + assert_eq!((t.r_candidate.trials, t.r_base.trials, t.r_free.trials), (20, 20, 80)); + assert_eq!(t.prompted.trials, t.r_candidate.trials + t.r_base.trials + t.r_free.trials); assert_eq!(t.degraded.iter().map(|d| d.trials).collect::>(), [60, 60]); assert_eq!(t.candidate.trials, 6 * 10 * 3); assert!(t.noise > 0.0); @@ -3074,6 +3114,20 @@ mod tests { // A far better draw is accepted in every trial. assert_eq!(t.candidate.accepted, t.candidate.trials); + // R (port 0) scores above the other prompted draws: it is Accepted as a + // candidate and Regressed as a base, and the trials without it read NoGain. + let mut r_high: Vec<_> = (0..6).map(|s| draw(2, 8, if s == 0 { 0.8 } else { 0.6 }, 0.02, s)).collect(); + let t = calibration_trials(&r_high, 6, 0, 0); + let [(cl, c), (bl, b), (fl, f)] = t.r_roles(); + assert_eq!((cl, bl, fl), ("r-candidate", "r-base", "r-free")); + assert_eq!((c.accepted, c.regressed), (c.trials, 0)); + assert_eq!((b.regressed, b.accepted), (b.trials, 0)); + assert_eq!((f.nogain, f.trials), (80, 80)); + assert_eq!(b.line(), "trials=20 nogain=0 accepted=0 regressed=20"); + assert_eq!(c.line(), "trials=20 nogain=0 accepted=20 regressed=0"); + assert_eq!(f.line(), "trials=80 nogain=80 accepted=0 regressed=0"); + r_high.clear(); + // Identical prompted draws: nothing to gain, nothing regressed, no noise. let same: Vec<_> = (0..6).map(|_| draw(2, 8, 0.6, 0.02, 1)).collect(); let t = calibration_trials(&same, 6, 0, 0); diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 99ddc0e..33cd6a3 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -1953,6 +1953,9 @@ fn cli_loop_calibrate_reads_answered_draws_and_refuses_unanswered_and_bad_labels assert!(text.starts_with("calibrate draws prompted=4 degraded=1 candidate=0 floor-arms=3 unusable=0"), "{text}"); assert!(text.contains("calibrate class=all items en-US n=2 skipped=0"), "{text}"); assert!(text.contains("calibrate class=all prompted trials=20 nogain=20 accepted=0 regressed=0"), "{text}"); + for side in ["r-candidate trials=4", "r-base trials=4", "r-free trials=12"] { + assert!(text.contains(&format!("calibrate class=all prompted-{side} nogain=")), "{text}"); + } assert!(text.contains("calibrate class=old degraded=bad trials=20 regressed=0"), "{text}"); for leaked in ["item:daodejing", "packet:", "highest excellence", "canaryroot"] { assert!(!text.contains(leaked), "{text}"); diff --git a/docs/calibration.md b/docs/calibration.md index 5c00ae8..128b669 100644 --- a/docs/calibration.md +++ b/docs/calibration.md @@ -9,7 +9,9 @@ reads the answered draws through `RunStores::read_draws` (the scoring read and skipped per language, the trial totals with their Accepted, NoGain and Regressed counts, and the mean panel noise. That is all it produces: the Accepted/NoGain/Regressed counts and the noise mean in the map's "Recalibration -on the grown corpus" block are reproduced by it; the other figures there (the +on the grown corpus" block are reproduced by it, and the prompted trials are +also printed split by the recorded lowering `R`'s role (`prompted-r-candidate`, +`prompted-r-base`, `prompted-r-free`, see below); the other figures there (the 35% with R removed, the noise range, the 0.0185 gain and its 0.41 ratio, the bootstrap intervals, the per-item SDs, R's +0.024, the mild-arm duplicate counts and mean composite offsets, the Train-vocabulary overlap, the old/new @@ -27,7 +29,17 @@ draws; the floor size is `FLOOR_ARMS_MIN` (3), the live rule's. 3-subset of the rest of `P` as the floor. A repeat of the same lowering should read NoGain; Accepted is a false accept, Regressed a false reject. (`R` comes from the round, not from `relower`, so it may not be - exchangeable with a repeat; read the pairs that involve it with that in mind.) + exchangeable with a repeat.) The prompted line is therefore printed three + more times, by `R`'s role in the trial: `prompted-r-candidate` (`R` is the + candidate), `prompted-r-base` (`R` is the base) and `prompted-r-free` (`R` is + neither; it may sit in the floor). The three sum to the prompted line. Read the Accepted + and Regressed columns of each line: a false accept sits mostly under the + first and a false reject under the second, but `R`-as-candidate trials can + Regress too (class `old` on `growth-scene`: 5 of its 15). The third + is the exchangeable rate: on `growth-scene` it was 0 Accepted and 0 Regressed + of 80 while all 11 false accepts and 9 false rejects involved `R`. Which of + the two rates the live rule should be held to is undecided: the live loop + meets `R` as the head's establishing lowering. - Degraded trials, per `--degraded` arm: every base in `P` against every 3-subset of the rest of `P`, the degraded draw as the candidate. These should read Regressed. diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 1a0b836..6b1ecb1 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -543,14 +543,22 @@ mild 60, Train-guide 240; Accepted / NoGain / Regressed): | old (14) | 0/105/15 | 0.0844 | 60 | 60 | 40/80/120 | | new (21) | 10/97/13 | 0.0340 | 3 | 4 | 50/129/61 | -Measured by Soul scratch scripts outside git (not reproduced by the harness; -`calibrate` prints class totals only): +Measured by Soul scratch scripts outside git (not reproduced by the harness, +except the R split in the first bullet, which `calibrate` prints as +`prompted-r-candidate`, `prompted-r-base` and `prompted-r-free` since the +`calibrate-report-r-involvement` cut; Soul re-ran growth-scene with it and the +printed `all` lines read r-candidate 11/9/0, r-base 0/11/9, r-free 0/80/0, as +below, in Accepted/NoGain/Regressed): - Every one of the 11 false accepts has R as the candidate (bases prompted-2, p4, p6); every one of the 9 false rejects has R as the base. The 80 trials without R read 0 Accepted and 0 Regressed in every class (95% upper bound 4.5%). R scores above the other prompted draws (+0.012 en-US and +0.023 ja-JP on all items), so R is not exchangeable with a repeat. + Which of the two rates the live rule should be held to is undecided: the + live loop meets R as the head's establishing lowering, so the R-candidate + rate may be the real one for cycles, while the 80 R-free trials are the + exchangeable repeat rate; neither number stands for the other. - The old/new split of the false-accept lives only in the 20 trials where R is the candidate, from 5 bases: new 10/20 [0.27, 0.73], old 0/20 [0.00, 0.17] (Clopper-Pearson); by base 3 of 5 against 0 of 5 (Fisher p 0.17). The