From 3b72f9b2be9e13da96163b7dc8719cf604f078d3 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 13:24:09 +0000 Subject: [PATCH 1/5] calibrate: split prompted trials by the recorded lowering R's role Prints prompted-r-candidate, prompted-r-base and prompted-r-free beside the prompted total, so the false accepts and rejects that involve R and the R-free trials are visible from the harness, not from scratch scripts. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/main.rs | 9 +++++++++ crates/weksa/src/run.rs | 35 +++++++++++++++++++++++++++++++---- crates/weksa/tests/cli.rs | 3 +++ docs/calibration.md | 14 ++++++++++++-- 4 files changed, 55 insertions(+), 6 deletions(-) diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 3cd2235..873a2ec 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -1293,6 +1293,15 @@ fn main() -> Result<()> { c.name, t.prompted.trials, t.prompted.nogain, t.prompted.accepted, t.prompted.regressed, t.noise )) ); + for (side, k) in [("r-candidate", t.r_candidate), ("r-base", t.r_base), ("r-free", t.r_free)] { + println!( + "{}", + run.labelled(&format!( + "calibrate class={} prompted-{side} trials={} nogain={} accepted={} regressed={}", + c.name, k.trials, k.nogain, k.accepted, k.regressed + )) + ); + } for (label, d) in degraded.iter().zip(&t.degraded) { println!( "{}", diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index b15fd07..a6bc735 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -1885,13 +1885,32 @@ impl Counts { Verdict::Rejected(_) => self.nogain += 1, } } + + fn plus(self, o: Counts) -> Counts { + Counts { + trials: self.trials + o.trials, + accepted: self.accepted + o.accepted, + nogain: self.nogain + o.nogain, + regressed: self.regressed + o.regressed, + } + } } /// What [`calibration_trials`] counted over one set of items. #[derive(Clone, Debug, PartialEq)] pub struct Trials { - /// Ordered (base, candidate) pairs of prompted draws, each over every floor. + /// Ordered (base, candidate) pairs of prompted draws, each over every floor: + /// the sum of the three counts below. pub prompted: Counts, + /// The prompted trials whose candidate is the recorded lowering `R` (port 0). + /// `R` comes from the round, not from `relower`, so it is not exchangeable + /// with a repeat: every false accept of a grown-corpus calibration lived here. + pub r_candidate: Counts, + /// The prompted trials whose base is `R`. + pub r_base: Counts, + /// The prompted trials with `R` neither base nor candidate (it may sit in + /// the floor): the exchangeable repeats. + pub r_free: Counts, /// Per degraded arm in the order given: the arm as the candidate. pub degraded: Vec, /// Every candidate arm as the candidate, pooled. @@ -1942,7 +1961,8 @@ pub fn calibration_trials( }; let pool: Vec = (0..prompted).collect(); let without = |out: &[usize]| -> Vec { pool.iter().copied().filter(|p| !out.contains(p)).collect() }; - let (mut prompted_counts, mut candidate_counts) = (Counts::default(), Counts::default()); + let (mut r_candidate, mut r_base, mut r_free) = (Counts::default(), Counts::default(), Counts::default()); + let mut candidate_counts = Counts::default(); let mut degraded_counts = vec![Counts::default(); degraded]; let (mut noise, mut noises) = (0.0, 0u32); for &b in &pool { @@ -1958,12 +1978,16 @@ pub fn calibration_trials( } for &c in pool.iter().filter(|&&c| c != b) { for floors in subsets(&without(&[b, c]), FLOOR_ARMS_MIN) { - prompted_counts.tally(decide_panel(&rows(b, c, &floors))); + let side = if c == 0 { &mut r_candidate } else if b == 0 { &mut r_base } else { &mut r_free }; + side.tally(decide_panel(&rows(b, c, &floors))); } } } Trials { - prompted: prompted_counts, + prompted: r_candidate.plus(r_base).plus(r_free), + r_candidate, + r_base, + r_free, degraded: degraded_counts, candidate: candidate_counts, noise: noise / f64::from(noises), @@ -3066,6 +3090,9 @@ mod tests { draws.extend((0..3).map(|s| draw(2, 8, 0.8, 0.02, s))); let t = calibration_trials(&draws, 6, 2, 3); assert_eq!(t.prompted.trials, 120); + // R is port 0: 20 pairs with R as candidate, 20 as base, 20 between the other five, 4 floors each. + assert_eq!((t.r_candidate.trials, t.r_base.trials, t.r_free.trials), (20, 20, 80)); + assert_eq!(t.prompted.trials, t.r_candidate.trials + t.r_base.trials + t.r_free.trials); assert_eq!(t.degraded.iter().map(|d| d.trials).collect::>(), [60, 60]); assert_eq!(t.candidate.trials, 6 * 10 * 3); assert!(t.noise > 0.0); diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 99ddc0e..33cd6a3 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -1953,6 +1953,9 @@ fn cli_loop_calibrate_reads_answered_draws_and_refuses_unanswered_and_bad_labels assert!(text.starts_with("calibrate draws prompted=4 degraded=1 candidate=0 floor-arms=3 unusable=0"), "{text}"); assert!(text.contains("calibrate class=all items en-US n=2 skipped=0"), "{text}"); assert!(text.contains("calibrate class=all prompted trials=20 nogain=20 accepted=0 regressed=0"), "{text}"); + for side in ["r-candidate trials=4", "r-base trials=4", "r-free trials=12"] { + assert!(text.contains(&format!("calibrate class=all prompted-{side} nogain=")), "{text}"); + } assert!(text.contains("calibrate class=old degraded=bad trials=20 regressed=0"), "{text}"); for leaked in ["item:daodejing", "packet:", "highest excellence", "canaryroot"] { assert!(!text.contains(leaked), "{text}"); diff --git a/docs/calibration.md b/docs/calibration.md index 5c00ae8..dc07411 100644 --- a/docs/calibration.md +++ b/docs/calibration.md @@ -9,7 +9,9 @@ reads the answered draws through `RunStores::read_draws` (the scoring read and skipped per language, the trial totals with their Accepted, NoGain and Regressed counts, and the mean panel noise. That is all it produces: the Accepted/NoGain/Regressed counts and the noise mean in the map's "Recalibration -on the grown corpus" block are reproduced by it; the other figures there (the +on the grown corpus" block are reproduced by it, and the prompted trials are +also printed split by the recorded lowering `R`'s role (`prompted-r-candidate`, +`prompted-r-base`, `prompted-r-free`, see below); the other figures there (the 35% with R removed, the noise range, the 0.0185 gain and its 0.41 ratio, the bootstrap intervals, the per-item SDs, R's +0.024, the mild-arm duplicate counts and mean composite offsets, the Train-vocabulary overlap, the old/new @@ -27,7 +29,15 @@ draws; the floor size is `FLOOR_ARMS_MIN` (3), the live rule's. 3-subset of the rest of `P` as the floor. A repeat of the same lowering should read NoGain; Accepted is a false accept, Regressed a false reject. (`R` comes from the round, not from `relower`, so it may not be - exchangeable with a repeat; read the pairs that involve it with that in mind.) + exchangeable with a repeat.) The prompted line is therefore printed three + more times, by `R`'s role in the trial: `prompted-r-candidate` (`R` is the + candidate), `prompted-r-base` (`R` is the base) and `prompted-r-free` (`R` is + neither; it may sit in the floor). The three sum to the prompted line. A false + accept is read from the first, a false reject from the second, and the third + is the exchangeable rate: on `growth-scene` it was 0 Accepted and 0 Regressed + of 80 while all 11 false accepts and 9 false rejects involved `R`. Which of + the two rates the live rule should be held to is undecided: the live loop + meets `R` as the head's establishing lowering. - Degraded trials, per `--degraded` arm: every base in `P` against every 3-subset of the rest of `P`, the degraded draw as the candidate. These should read Regressed. -- 2.47.3 From 6c263f0398e421f9bc40688977402c555915cf52 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 13:24:19 +0000 Subject: [PATCH 2/5] map: the R split is now printed by calibrate, not yet re-run on growth-scene Co-Authored-By: Claude Sonnet 5.5 --- docs/targets/interlingua-lowering-map.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 1a0b836..f418329 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -543,8 +543,12 @@ mild 60, Train-guide 240; Accepted / NoGain / Regressed): | old (14) | 0/105/15 | 0.0844 | 60 | 60 | 40/80/120 | | new (21) | 10/97/13 | 0.0340 | 3 | 4 | 50/129/61 | -Measured by Soul scratch scripts outside git (not reproduced by the harness; -`calibrate` prints class totals only): +Measured by Soul scratch scripts outside git (not reproduced by the harness, +except the R split in the first bullet, which `calibrate` prints as +`prompted-r-candidate`, `prompted-r-base` and `prompted-r-free` since the +`calibrate-report-r-involvement` cut; the growth-scene run has not been +re-run with it, so those printed lines are not yet checked against these +figures): - Every one of the 11 false accepts has R as the candidate (bases prompted-2, p4, p6); every one of the 9 false rejects has R as the base. The -- 2.47.3 From 5a681b2c3afd1f8eab031c0673156f4c7220abbd Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 13:48:54 +0000 Subject: [PATCH 3/5] calibrate: pin R roles and line columns in the unit test; doc: false rejects can have R as candidate Soul rinv s1 findings r-roles-unpinned and reject-in-r-candidate. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/main.rs | 7 ++----- crates/weksa/src/run.rs | 27 +++++++++++++++++++++++++++ docs/calibration.md | 6 ++++-- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 873a2ec..ff444d8 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -1293,13 +1293,10 @@ fn main() -> Result<()> { c.name, t.prompted.trials, t.prompted.nogain, t.prompted.accepted, t.prompted.regressed, t.noise )) ); - for (side, k) in [("r-candidate", t.r_candidate), ("r-base", t.r_base), ("r-free", t.r_free)] { + for (side, k) in t.r_roles() { println!( "{}", - run.labelled(&format!( - "calibrate class={} prompted-{side} trials={} nogain={} accepted={} regressed={}", - c.name, k.trials, k.nogain, k.accepted, k.regressed - )) + run.labelled(&format!("calibrate class={} prompted-{side} {}", c.name, k.line())) ); } for (label, d) in degraded.iter().zip(&t.degraded) { diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index a6bc735..32b29bd 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -1886,6 +1886,11 @@ impl Counts { } } + /// The `trials=.. nogain=.. accepted=.. regressed=..` text of a calibrate line. + pub fn line(&self) -> String { + format!("trials={} nogain={} accepted={} regressed={}", self.trials, self.nogain, self.accepted, self.regressed) + } + fn plus(self, o: Counts) -> Counts { Counts { trials: self.trials + o.trials, @@ -1919,6 +1924,14 @@ pub struct Trials { pub noise: f64, } +impl Trials { + /// The prompted trials by the recorded lowering `R`'s role, labelled as the + /// calibrate lines print them. + pub fn r_roles(&self) -> [(&'static str, Counts); 3] { + [("r-candidate", self.r_candidate), ("r-base", self.r_base), ("r-free", self.r_free)] + } +} + /// Every `k`-subset of `pool`, in order. fn subsets(pool: &[usize], k: usize) -> Vec> { if k == 0 { @@ -3101,6 +3114,20 @@ mod tests { // A far better draw is accepted in every trial. assert_eq!(t.candidate.accepted, t.candidate.trials); + // R (port 0) scores above the other prompted draws: it is Accepted as a + // candidate and Regressed as a base, and the trials without it read NoGain. + let mut r_high: Vec<_> = (0..6).map(|s| draw(2, 8, if s == 0 { 0.8 } else { 0.6 }, 0.02, s)).collect(); + let t = calibration_trials(&r_high, 6, 0, 0); + let [(cl, c), (bl, b), (fl, f)] = t.r_roles(); + assert_eq!((cl, bl, fl), ("r-candidate", "r-base", "r-free")); + assert_eq!((c.accepted, c.regressed), (c.trials, 0)); + assert_eq!((b.regressed, b.accepted), (b.trials, 0)); + assert_eq!((f.nogain, f.trials), (80, 80)); + assert_eq!(b.line(), "trials=20 nogain=0 accepted=0 regressed=20"); + assert_eq!(c.line(), "trials=20 nogain=0 accepted=20 regressed=0"); + assert_eq!(f.line(), "trials=80 nogain=80 accepted=0 regressed=0"); + r_high.clear(); + // Identical prompted draws: nothing to gain, nothing regressed, no noise. let same: Vec<_> = (0..6).map(|_| draw(2, 8, 0.6, 0.02, 1)).collect(); let t = calibration_trials(&same, 6, 0, 0); diff --git a/docs/calibration.md b/docs/calibration.md index dc07411..128b669 100644 --- a/docs/calibration.md +++ b/docs/calibration.md @@ -32,8 +32,10 @@ draws; the floor size is `FLOOR_ARMS_MIN` (3), the live rule's. exchangeable with a repeat.) The prompted line is therefore printed three more times, by `R`'s role in the trial: `prompted-r-candidate` (`R` is the candidate), `prompted-r-base` (`R` is the base) and `prompted-r-free` (`R` is - neither; it may sit in the floor). The three sum to the prompted line. A false - accept is read from the first, a false reject from the second, and the third + neither; it may sit in the floor). The three sum to the prompted line. Read the Accepted + and Regressed columns of each line: a false accept sits mostly under the + first and a false reject under the second, but `R`-as-candidate trials can + Regress too (class `old` on `growth-scene`: 5 of its 15). The third is the exchangeable rate: on `growth-scene` it was 0 Accepted and 0 Regressed of 80 while all 11 false accepts and 9 false rejects involved `R`. Which of the two rates the live rule should be held to is undecided: the live loop -- 2.47.3 From cca0664c17e810ef43a45b6401018f18c8c3c596 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 13:49:00 +0000 Subject: [PATCH 4/5] map: growth-scene re-run with the R split matches the Soul figures Co-Authored-By: Claude Sonnet 5.5 --- docs/targets/interlingua-lowering-map.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index f418329..2afa018 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -546,9 +546,9 @@ mild 60, Train-guide 240; Accepted / NoGain / Regressed): Measured by Soul scratch scripts outside git (not reproduced by the harness, except the R split in the first bullet, which `calibrate` prints as `prompted-r-candidate`, `prompted-r-base` and `prompted-r-free` since the -`calibrate-report-r-involvement` cut; the growth-scene run has not been -re-run with it, so those printed lines are not yet checked against these -figures): +`calibrate-report-r-involvement` cut; Soul re-ran growth-scene with it and the +printed `all` lines read r-candidate 11/9/0, r-base 0/11/9, r-free 0/80/0, as +below, in Accepted/NoGain/Regressed): - Every one of the 11 false accepts has R as the candidate (bases prompted-2, p4, p6); every one of the 9 false rejects has R as the base. The -- 2.47.3 From 74edaadfbfefc3fa46859f94527e76ae9e3fdf75 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 13:50:39 +0000 Subject: [PATCH 5/5] map: which R rate the rule is held to is undecided Soul finding r-draw-owns-every-misread, failure scenario. Co-Authored-By: Claude Sonnet 5.5 --- docs/targets/interlingua-lowering-map.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 2afa018..6b1ecb1 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -555,6 +555,10 @@ below, in Accepted/NoGain/Regressed): 80 trials without R read 0 Accepted and 0 Regressed in every class (95% upper bound 4.5%). R scores above the other prompted draws (+0.012 en-US and +0.023 ja-JP on all items), so R is not exchangeable with a repeat. + Which of the two rates the live rule should be held to is undecided: the + live loop meets R as the head's establishing lowering, so the R-candidate + rate may be the real one for cycles, while the 80 R-free trials are the + exchangeable repeat rate; neither number stands for the other. - The old/new split of the false-accept lives only in the 20 trials where R is the candidate, from 5 bases: new 10/20 [0.27, 0.73], old 0/20 [0.00, 0.17] (Clopper-Pearson); by base 3 of 5 against 0 of 5 (Fisher p 0.17). The -- 2.47.3