calibrate: split prompted trials by the recorded lowering R's role

Prints prompted-r-candidate, prompted-r-base and prompted-r-free beside the
prompted total, so the false accepts and rejects that involve R and the
R-free trials are visible from the harness, not from scratch scripts.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Weksa 2026-11-13 13:24:09 +00:00
commit 3b72f9b2be
4 changed files with 55 additions and 6 deletions

View file

@ -1293,6 +1293,15 @@ fn main() -> Result<()> {
c.name, t.prompted.trials, t.prompted.nogain, t.prompted.accepted, t.prompted.regressed, t.noise
))
);
for (side, k) in [("r-candidate", t.r_candidate), ("r-base", t.r_base), ("r-free", t.r_free)] {
println!(
"{}",
run.labelled(&format!(
"calibrate class={} prompted-{side} trials={} nogain={} accepted={} regressed={}",
c.name, k.trials, k.nogain, k.accepted, k.regressed
))
);
}
for (label, d) in degraded.iter().zip(&t.degraded) {
println!(
"{}",

View file

@ -1885,13 +1885,32 @@ impl Counts {
Verdict::Rejected(_) => self.nogain += 1,
}
}
fn plus(self, o: Counts) -> Counts {
Counts {
trials: self.trials + o.trials,
accepted: self.accepted + o.accepted,
nogain: self.nogain + o.nogain,
regressed: self.regressed + o.regressed,
}
}
}
/// What [`calibration_trials`] counted over one set of items.
#[derive(Clone, Debug, PartialEq)]
pub struct Trials {
/// Ordered (base, candidate) pairs of prompted draws, each over every floor.
/// Ordered (base, candidate) pairs of prompted draws, each over every floor:
/// the sum of the three counts below.
pub prompted: Counts,
/// The prompted trials whose candidate is the recorded lowering `R` (port 0).
/// `R` comes from the round, not from `relower`, so it is not exchangeable
/// with a repeat: every false accept of a grown-corpus calibration lived here.
pub r_candidate: Counts,
/// The prompted trials whose base is `R`.
pub r_base: Counts,
/// The prompted trials with `R` neither base nor candidate (it may sit in
/// the floor): the exchangeable repeats.
pub r_free: Counts,
/// Per degraded arm in the order given: the arm as the candidate.
pub degraded: Vec<Counts>,
/// Every candidate arm as the candidate, pooled.
@ -1942,7 +1961,8 @@ pub fn calibration_trials(
};
let pool: Vec<usize> = (0..prompted).collect();
let without = |out: &[usize]| -> Vec<usize> { pool.iter().copied().filter(|p| !out.contains(p)).collect() };
let (mut prompted_counts, mut candidate_counts) = (Counts::default(), Counts::default());
let (mut r_candidate, mut r_base, mut r_free) = (Counts::default(), Counts::default(), Counts::default());
let mut candidate_counts = Counts::default();
let mut degraded_counts = vec![Counts::default(); degraded];
let (mut noise, mut noises) = (0.0, 0u32);
for &b in &pool {
@ -1958,12 +1978,16 @@ pub fn calibration_trials(
}
for &c in pool.iter().filter(|&&c| c != b) {
for floors in subsets(&without(&[b, c]), FLOOR_ARMS_MIN) {
prompted_counts.tally(decide_panel(&rows(b, c, &floors)));
let side = if c == 0 { &mut r_candidate } else if b == 0 { &mut r_base } else { &mut r_free };
side.tally(decide_panel(&rows(b, c, &floors)));
}
}
}
Trials {
prompted: prompted_counts,
prompted: r_candidate.plus(r_base).plus(r_free),
r_candidate,
r_base,
r_free,
degraded: degraded_counts,
candidate: candidate_counts,
noise: noise / f64::from(noises),
@ -3066,6 +3090,9 @@ mod tests {
draws.extend((0..3).map(|s| draw(2, 8, 0.8, 0.02, s)));
let t = calibration_trials(&draws, 6, 2, 3);
assert_eq!(t.prompted.trials, 120);
// R is port 0: 20 pairs with R as candidate, 20 as base, 20 between the other five, 4 floors each.
assert_eq!((t.r_candidate.trials, t.r_base.trials, t.r_free.trials), (20, 20, 80));
assert_eq!(t.prompted.trials, t.r_candidate.trials + t.r_base.trials + t.r_free.trials);
assert_eq!(t.degraded.iter().map(|d| d.trials).collect::<Vec<_>>(), [60, 60]);
assert_eq!(t.candidate.trials, 6 * 10 * 3);
assert!(t.noise > 0.0);

View file

@ -1953,6 +1953,9 @@ fn cli_loop_calibrate_reads_answered_draws_and_refuses_unanswered_and_bad_labels
assert!(text.starts_with("calibrate draws prompted=4 degraded=1 candidate=0 floor-arms=3 unusable=0"), "{text}");
assert!(text.contains("calibrate class=all items en-US n=2 skipped=0"), "{text}");
assert!(text.contains("calibrate class=all prompted trials=20 nogain=20 accepted=0 regressed=0"), "{text}");
for side in ["r-candidate trials=4", "r-base trials=4", "r-free trials=12"] {
assert!(text.contains(&format!("calibrate class=all prompted-{side} nogain=")), "{text}");
}
assert!(text.contains("calibrate class=old degraded=bad trials=20 regressed=0"), "{text}");
for leaked in ["item:daodejing", "packet:", "highest excellence", "canaryroot"] {
assert!(!text.contains(leaked), "{text}");

View file

@ -9,7 +9,9 @@ reads the answered draws through `RunStores::read_draws` (the scoring
read and skipped per language, the trial totals with their Accepted, NoGain and
Regressed counts, and the mean panel noise. That is all it produces: the
Accepted/NoGain/Regressed counts and the noise mean in the map's "Recalibration
on the grown corpus" block are reproduced by it; the other figures there (the
on the grown corpus" block are reproduced by it, and the prompted trials are
also printed split by the recorded lowering `R`'s role (`prompted-r-candidate`,
`prompted-r-base`, `prompted-r-free`, see below); the other figures there (the
35% with R removed, the noise range, the 0.0185 gain and its 0.41 ratio, the
bootstrap intervals, the per-item SDs, R's +0.024, the mild-arm duplicate
counts and mean composite offsets, the Train-vocabulary overlap, the old/new
@ -27,7 +29,15 @@ draws; the floor size is `FLOOR_ARMS_MIN` (3), the live rule's.
3-subset of the rest of `P` as the floor. A repeat of the same lowering
should read NoGain; Accepted is a false accept, Regressed a false reject.
(`R` comes from the round, not from `relower`, so it may not be
exchangeable with a repeat; read the pairs that involve it with that in mind.)
exchangeable with a repeat.) The prompted line is therefore printed three
more times, by `R`'s role in the trial: `prompted-r-candidate` (`R` is the
candidate), `prompted-r-base` (`R` is the base) and `prompted-r-free` (`R` is
neither; it may sit in the floor). The three sum to the prompted line. A false
accept is read from the first, a false reject from the second, and the third
is the exchangeable rate: on `growth-scene` it was 0 Accepted and 0 Regressed
of 80 while all 11 false accepts and 9 false rejects involved `R`. Which of
the two rates the live rule should be held to is undecided: the live loop
meets `R` as the head's establishing lowering.
- Degraded trials, per `--degraded` arm: every base in `P` against every
3-subset of the rest of `P`, the degraded draw as the candidate. These should
read Regressed.