calibrate: split prompted trials by the recorded lowering R's role
Prints prompted-r-candidate, prompted-r-base and prompted-r-free beside the prompted total, so the false accepts and rejects that involve R and the R-free trials are visible from the harness, not from scratch scripts. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
d77a83e637
commit
3b72f9b2be
4 changed files with 55 additions and 6 deletions
|
|
@ -1293,6 +1293,15 @@ fn main() -> Result<()> {
|
|||
c.name, t.prompted.trials, t.prompted.nogain, t.prompted.accepted, t.prompted.regressed, t.noise
|
||||
))
|
||||
);
|
||||
for (side, k) in [("r-candidate", t.r_candidate), ("r-base", t.r_base), ("r-free", t.r_free)] {
|
||||
println!(
|
||||
"{}",
|
||||
run.labelled(&format!(
|
||||
"calibrate class={} prompted-{side} trials={} nogain={} accepted={} regressed={}",
|
||||
c.name, k.trials, k.nogain, k.accepted, k.regressed
|
||||
))
|
||||
);
|
||||
}
|
||||
for (label, d) in degraded.iter().zip(&t.degraded) {
|
||||
println!(
|
||||
"{}",
|
||||
|
|
|
|||
|
|
@ -1885,13 +1885,32 @@ impl Counts {
|
|||
Verdict::Rejected(_) => self.nogain += 1,
|
||||
}
|
||||
}
|
||||
|
||||
fn plus(self, o: Counts) -> Counts {
|
||||
Counts {
|
||||
trials: self.trials + o.trials,
|
||||
accepted: self.accepted + o.accepted,
|
||||
nogain: self.nogain + o.nogain,
|
||||
regressed: self.regressed + o.regressed,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What [`calibration_trials`] counted over one set of items.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Trials {
|
||||
/// Ordered (base, candidate) pairs of prompted draws, each over every floor.
|
||||
/// Ordered (base, candidate) pairs of prompted draws, each over every floor:
|
||||
/// the sum of the three counts below.
|
||||
pub prompted: Counts,
|
||||
/// The prompted trials whose candidate is the recorded lowering `R` (port 0).
|
||||
/// `R` comes from the round, not from `relower`, so it is not exchangeable
|
||||
/// with a repeat: every false accept of a grown-corpus calibration lived here.
|
||||
pub r_candidate: Counts,
|
||||
/// The prompted trials whose base is `R`.
|
||||
pub r_base: Counts,
|
||||
/// The prompted trials with `R` neither base nor candidate (it may sit in
|
||||
/// the floor): the exchangeable repeats.
|
||||
pub r_free: Counts,
|
||||
/// Per degraded arm in the order given: the arm as the candidate.
|
||||
pub degraded: Vec<Counts>,
|
||||
/// Every candidate arm as the candidate, pooled.
|
||||
|
|
@ -1942,7 +1961,8 @@ pub fn calibration_trials(
|
|||
};
|
||||
let pool: Vec<usize> = (0..prompted).collect();
|
||||
let without = |out: &[usize]| -> Vec<usize> { pool.iter().copied().filter(|p| !out.contains(p)).collect() };
|
||||
let (mut prompted_counts, mut candidate_counts) = (Counts::default(), Counts::default());
|
||||
let (mut r_candidate, mut r_base, mut r_free) = (Counts::default(), Counts::default(), Counts::default());
|
||||
let mut candidate_counts = Counts::default();
|
||||
let mut degraded_counts = vec![Counts::default(); degraded];
|
||||
let (mut noise, mut noises) = (0.0, 0u32);
|
||||
for &b in &pool {
|
||||
|
|
@ -1958,12 +1978,16 @@ pub fn calibration_trials(
|
|||
}
|
||||
for &c in pool.iter().filter(|&&c| c != b) {
|
||||
for floors in subsets(&without(&[b, c]), FLOOR_ARMS_MIN) {
|
||||
prompted_counts.tally(decide_panel(&rows(b, c, &floors)));
|
||||
let side = if c == 0 { &mut r_candidate } else if b == 0 { &mut r_base } else { &mut r_free };
|
||||
side.tally(decide_panel(&rows(b, c, &floors)));
|
||||
}
|
||||
}
|
||||
}
|
||||
Trials {
|
||||
prompted: prompted_counts,
|
||||
prompted: r_candidate.plus(r_base).plus(r_free),
|
||||
r_candidate,
|
||||
r_base,
|
||||
r_free,
|
||||
degraded: degraded_counts,
|
||||
candidate: candidate_counts,
|
||||
noise: noise / f64::from(noises),
|
||||
|
|
@ -3066,6 +3090,9 @@ mod tests {
|
|||
draws.extend((0..3).map(|s| draw(2, 8, 0.8, 0.02, s)));
|
||||
let t = calibration_trials(&draws, 6, 2, 3);
|
||||
assert_eq!(t.prompted.trials, 120);
|
||||
// R is port 0: 20 pairs with R as candidate, 20 as base, 20 between the other five, 4 floors each.
|
||||
assert_eq!((t.r_candidate.trials, t.r_base.trials, t.r_free.trials), (20, 20, 80));
|
||||
assert_eq!(t.prompted.trials, t.r_candidate.trials + t.r_base.trials + t.r_free.trials);
|
||||
assert_eq!(t.degraded.iter().map(|d| d.trials).collect::<Vec<_>>(), [60, 60]);
|
||||
assert_eq!(t.candidate.trials, 6 * 10 * 3);
|
||||
assert!(t.noise > 0.0);
|
||||
|
|
|
|||
|
|
@ -1953,6 +1953,9 @@ fn cli_loop_calibrate_reads_answered_draws_and_refuses_unanswered_and_bad_labels
|
|||
assert!(text.starts_with("calibrate draws prompted=4 degraded=1 candidate=0 floor-arms=3 unusable=0"), "{text}");
|
||||
assert!(text.contains("calibrate class=all items en-US n=2 skipped=0"), "{text}");
|
||||
assert!(text.contains("calibrate class=all prompted trials=20 nogain=20 accepted=0 regressed=0"), "{text}");
|
||||
for side in ["r-candidate trials=4", "r-base trials=4", "r-free trials=12"] {
|
||||
assert!(text.contains(&format!("calibrate class=all prompted-{side} nogain=")), "{text}");
|
||||
}
|
||||
assert!(text.contains("calibrate class=old degraded=bad trials=20 regressed=0"), "{text}");
|
||||
for leaked in ["item:daodejing", "packet:", "highest excellence", "canaryroot"] {
|
||||
assert!(!text.contains(leaked), "{text}");
|
||||
|
|
|
|||
|
|
@ -9,7 +9,9 @@ reads the answered draws through `RunStores::read_draws` (the scoring
|
|||
read and skipped per language, the trial totals with their Accepted, NoGain and
|
||||
Regressed counts, and the mean panel noise. That is all it produces: the
|
||||
Accepted/NoGain/Regressed counts and the noise mean in the map's "Recalibration
|
||||
on the grown corpus" block are reproduced by it; the other figures there (the
|
||||
on the grown corpus" block are reproduced by it, and the prompted trials are
|
||||
also printed split by the recorded lowering `R`'s role (`prompted-r-candidate`,
|
||||
`prompted-r-base`, `prompted-r-free`, see below); the other figures there (the
|
||||
35% with R removed, the noise range, the 0.0185 gain and its 0.41 ratio, the
|
||||
bootstrap intervals, the per-item SDs, R's +0.024, the mild-arm duplicate
|
||||
counts and mean composite offsets, the Train-vocabulary overlap, the old/new
|
||||
|
|
@ -27,7 +29,15 @@ draws; the floor size is `FLOOR_ARMS_MIN` (3), the live rule's.
|
|||
3-subset of the rest of `P` as the floor. A repeat of the same lowering
|
||||
should read NoGain; Accepted is a false accept, Regressed a false reject.
|
||||
(`R` comes from the round, not from `relower`, so it may not be
|
||||
exchangeable with a repeat; read the pairs that involve it with that in mind.)
|
||||
exchangeable with a repeat.) The prompted line is therefore printed three
|
||||
more times, by `R`'s role in the trial: `prompted-r-candidate` (`R` is the
|
||||
candidate), `prompted-r-base` (`R` is the base) and `prompted-r-free` (`R` is
|
||||
neither; it may sit in the floor). The three sum to the prompted line. A false
|
||||
accept is read from the first, a false reject from the second, and the third
|
||||
is the exchangeable rate: on `growth-scene` it was 0 Accepted and 0 Regressed
|
||||
of 80 while all 11 false accepts and 9 false rejects involved `R`. Which of
|
||||
the two rates the live rule should be held to is undecided: the live loop
|
||||
meets `R` as the head's establishing lowering.
|
||||
- Degraded trials, per `--degraded` arm: every base in `P` against every
|
||||
3-subset of the rest of `P`, the degraded draw as the candidate. These should
|
||||
read Regressed.
|
||||
|
|
|
|||
Loading…
Reference in a new issue