calibrate: split prompted trials by R's role #81
5 changed files with 91 additions and 8 deletions
|
|
@ -1293,6 +1293,12 @@ fn main() -> Result<()> {
|
|||
c.name, t.prompted.trials, t.prompted.nogain, t.prompted.accepted, t.prompted.regressed, t.noise
|
||||
))
|
||||
);
|
||||
for (side, k) in t.r_roles() {
|
||||
println!(
|
||||
"{}",
|
||||
run.labelled(&format!("calibrate class={} prompted-{side} {}", c.name, k.line()))
|
||||
);
|
||||
}
|
||||
for (label, d) in degraded.iter().zip(&t.degraded) {
|
||||
println!(
|
||||
"{}",
|
||||
|
|
|
|||
|
|
@ -1885,13 +1885,37 @@ impl Counts {
|
|||
Verdict::Rejected(_) => self.nogain += 1,
|
||||
}
|
||||
}
|
||||
|
||||
/// The `trials=.. nogain=.. accepted=.. regressed=..` text of a calibrate line.
|
||||
pub fn line(&self) -> String {
|
||||
format!("trials={} nogain={} accepted={} regressed={}", self.trials, self.nogain, self.accepted, self.regressed)
|
||||
}
|
||||
|
||||
fn plus(self, o: Counts) -> Counts {
|
||||
Counts {
|
||||
trials: self.trials + o.trials,
|
||||
accepted: self.accepted + o.accepted,
|
||||
nogain: self.nogain + o.nogain,
|
||||
regressed: self.regressed + o.regressed,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What [`calibration_trials`] counted over one set of items.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Trials {
|
||||
/// Ordered (base, candidate) pairs of prompted draws, each over every floor.
|
||||
/// Ordered (base, candidate) pairs of prompted draws, each over every floor:
|
||||
/// the sum of the three counts below.
|
||||
pub prompted: Counts,
|
||||
/// The prompted trials whose candidate is the recorded lowering `R` (port 0).
|
||||
/// `R` comes from the round, not from `relower`, so it is not exchangeable
|
||||
/// with a repeat: every false accept of a grown-corpus calibration lived here.
|
||||
pub r_candidate: Counts,
|
||||
/// The prompted trials whose base is `R`.
|
||||
pub r_base: Counts,
|
||||
/// The prompted trials with `R` neither base nor candidate (it may sit in
|
||||
/// the floor): the exchangeable repeats.
|
||||
pub r_free: Counts,
|
||||
/// Per degraded arm in the order given: the arm as the candidate.
|
||||
pub degraded: Vec<Counts>,
|
||||
/// Every candidate arm as the candidate, pooled.
|
||||
|
|
@ -1900,6 +1924,14 @@ pub struct Trials {
|
|||
pub noise: f64,
|
||||
}
|
||||
|
||||
impl Trials {
|
||||
/// The prompted trials by the recorded lowering `R`'s role, labelled as the
|
||||
/// calibrate lines print them.
|
||||
pub fn r_roles(&self) -> [(&'static str, Counts); 3] {
|
||||
[("r-candidate", self.r_candidate), ("r-base", self.r_base), ("r-free", self.r_free)]
|
||||
}
|
||||
}
|
||||
|
||||
/// Every `k`-subset of `pool`, in order.
|
||||
fn subsets(pool: &[usize], k: usize) -> Vec<Vec<usize>> {
|
||||
if k == 0 {
|
||||
|
|
@ -1942,7 +1974,8 @@ pub fn calibration_trials(
|
|||
};
|
||||
let pool: Vec<usize> = (0..prompted).collect();
|
||||
let without = |out: &[usize]| -> Vec<usize> { pool.iter().copied().filter(|p| !out.contains(p)).collect() };
|
||||
let (mut prompted_counts, mut candidate_counts) = (Counts::default(), Counts::default());
|
||||
let (mut r_candidate, mut r_base, mut r_free) = (Counts::default(), Counts::default(), Counts::default());
|
||||
let mut candidate_counts = Counts::default();
|
||||
let mut degraded_counts = vec![Counts::default(); degraded];
|
||||
let (mut noise, mut noises) = (0.0, 0u32);
|
||||
for &b in &pool {
|
||||
|
|
@ -1958,12 +1991,16 @@ pub fn calibration_trials(
|
|||
}
|
||||
for &c in pool.iter().filter(|&&c| c != b) {
|
||||
for floors in subsets(&without(&[b, c]), FLOOR_ARMS_MIN) {
|
||||
prompted_counts.tally(decide_panel(&rows(b, c, &floors)));
|
||||
let side = if c == 0 { &mut r_candidate } else if b == 0 { &mut r_base } else { &mut r_free };
|
||||
side.tally(decide_panel(&rows(b, c, &floors)));
|
||||
}
|
||||
}
|
||||
}
|
||||
Trials {
|
||||
prompted: prompted_counts,
|
||||
prompted: r_candidate.plus(r_base).plus(r_free),
|
||||
r_candidate,
|
||||
r_base,
|
||||
r_free,
|
||||
degraded: degraded_counts,
|
||||
candidate: candidate_counts,
|
||||
noise: noise / f64::from(noises),
|
||||
|
|
@ -3066,6 +3103,9 @@ mod tests {
|
|||
draws.extend((0..3).map(|s| draw(2, 8, 0.8, 0.02, s)));
|
||||
let t = calibration_trials(&draws, 6, 2, 3);
|
||||
assert_eq!(t.prompted.trials, 120);
|
||||
// R is port 0: 20 pairs with R as candidate, 20 as base, 20 between the other five, 4 floors each.
|
||||
assert_eq!((t.r_candidate.trials, t.r_base.trials, t.r_free.trials), (20, 20, 80));
|
||||
assert_eq!(t.prompted.trials, t.r_candidate.trials + t.r_base.trials + t.r_free.trials);
|
||||
assert_eq!(t.degraded.iter().map(|d| d.trials).collect::<Vec<_>>(), [60, 60]);
|
||||
assert_eq!(t.candidate.trials, 6 * 10 * 3);
|
||||
assert!(t.noise > 0.0);
|
||||
|
|
@ -3074,6 +3114,20 @@ mod tests {
|
|||
// A far better draw is accepted in every trial.
|
||||
assert_eq!(t.candidate.accepted, t.candidate.trials);
|
||||
|
||||
// R (port 0) scores above the other prompted draws: it is Accepted as a
|
||||
// candidate and Regressed as a base, and the trials without it read NoGain.
|
||||
let mut r_high: Vec<_> = (0..6).map(|s| draw(2, 8, if s == 0 { 0.8 } else { 0.6 }, 0.02, s)).collect();
|
||||
let t = calibration_trials(&r_high, 6, 0, 0);
|
||||
let [(cl, c), (bl, b), (fl, f)] = t.r_roles();
|
||||
assert_eq!((cl, bl, fl), ("r-candidate", "r-base", "r-free"));
|
||||
assert_eq!((c.accepted, c.regressed), (c.trials, 0));
|
||||
assert_eq!((b.regressed, b.accepted), (b.trials, 0));
|
||||
assert_eq!((f.nogain, f.trials), (80, 80));
|
||||
assert_eq!(b.line(), "trials=20 nogain=0 accepted=0 regressed=20");
|
||||
assert_eq!(c.line(), "trials=20 nogain=0 accepted=20 regressed=0");
|
||||
assert_eq!(f.line(), "trials=80 nogain=80 accepted=0 regressed=0");
|
||||
r_high.clear();
|
||||
|
||||
// Identical prompted draws: nothing to gain, nothing regressed, no noise.
|
||||
let same: Vec<_> = (0..6).map(|_| draw(2, 8, 0.6, 0.02, 1)).collect();
|
||||
let t = calibration_trials(&same, 6, 0, 0);
|
||||
|
|
|
|||
|
|
@ -1953,6 +1953,9 @@ fn cli_loop_calibrate_reads_answered_draws_and_refuses_unanswered_and_bad_labels
|
|||
assert!(text.starts_with("calibrate draws prompted=4 degraded=1 candidate=0 floor-arms=3 unusable=0"), "{text}");
|
||||
assert!(text.contains("calibrate class=all items en-US n=2 skipped=0"), "{text}");
|
||||
assert!(text.contains("calibrate class=all prompted trials=20 nogain=20 accepted=0 regressed=0"), "{text}");
|
||||
for side in ["r-candidate trials=4", "r-base trials=4", "r-free trials=12"] {
|
||||
assert!(text.contains(&format!("calibrate class=all prompted-{side} nogain=")), "{text}");
|
||||
}
|
||||
assert!(text.contains("calibrate class=old degraded=bad trials=20 regressed=0"), "{text}");
|
||||
for leaked in ["item:daodejing", "packet:", "highest excellence", "canaryroot"] {
|
||||
assert!(!text.contains(leaked), "{text}");
|
||||
|
|
|
|||
|
|
@ -9,7 +9,9 @@ reads the answered draws through `RunStores::read_draws` (the scoring
|
|||
read and skipped per language, the trial totals with their Accepted, NoGain and
|
||||
Regressed counts, and the mean panel noise. That is all it produces: the
|
||||
Accepted/NoGain/Regressed counts and the noise mean in the map's "Recalibration
|
||||
on the grown corpus" block are reproduced by it; the other figures there (the
|
||||
on the grown corpus" block are reproduced by it, and the prompted trials are
|
||||
also printed split by the recorded lowering `R`'s role (`prompted-r-candidate`,
|
||||
`prompted-r-base`, `prompted-r-free`, see below); the other figures there (the
|
||||
35% with R removed, the noise range, the 0.0185 gain and its 0.41 ratio, the
|
||||
bootstrap intervals, the per-item SDs, R's +0.024, the mild-arm duplicate
|
||||
counts and mean composite offsets, the Train-vocabulary overlap, the old/new
|
||||
|
|
@ -27,7 +29,17 @@ draws; the floor size is `FLOOR_ARMS_MIN` (3), the live rule's.
|
|||
3-subset of the rest of `P` as the floor. A repeat of the same lowering
|
||||
should read NoGain; Accepted is a false accept, Regressed a false reject.
|
||||
(`R` comes from the round, not from `relower`, so it may not be
|
||||
exchangeable with a repeat; read the pairs that involve it with that in mind.)
|
||||
exchangeable with a repeat.) The prompted line is therefore printed three
|
||||
more times, by `R`'s role in the trial: `prompted-r-candidate` (`R` is the
|
||||
candidate), `prompted-r-base` (`R` is the base) and `prompted-r-free` (`R` is
|
||||
neither; it may sit in the floor). The three sum to the prompted line. Read the Accepted
|
||||
and Regressed columns of each line: a false accept sits mostly under the
|
||||
first and a false reject under the second, but `R`-as-candidate trials can
|
||||
Regress too (class `old` on `growth-scene`: 5 of its 15). The third
|
||||
is the exchangeable rate: on `growth-scene` it was 0 Accepted and 0 Regressed
|
||||
of 80 while all 11 false accepts and 9 false rejects involved `R`. Which of
|
||||
the two rates the live rule should be held to is undecided: the live loop
|
||||
meets `R` as the head's establishing lowering.
|
||||
- Degraded trials, per `--degraded` arm: every base in `P` against every
|
||||
3-subset of the rest of `P`, the degraded draw as the candidate. These should
|
||||
read Regressed.
|
||||
|
|
|
|||
|
|
@ -543,14 +543,22 @@ mild 60, Train-guide 240; Accepted / NoGain / Regressed):
|
|||
| old (14) | 0/105/15 | 0.0844 | 60 | 60 | 40/80/120 |
|
||||
| new (21) | 10/97/13 | 0.0340 | 3 | 4 | 50/129/61 |
|
||||
|
||||
Measured by Soul scratch scripts outside git (not reproduced by the harness;
|
||||
`calibrate` prints class totals only):
|
||||
Measured by Soul scratch scripts outside git (not reproduced by the harness,
|
||||
except the R split in the first bullet, which `calibrate` prints as
|
||||
`prompted-r-candidate`, `prompted-r-base` and `prompted-r-free` since the
|
||||
`calibrate-report-r-involvement` cut; Soul re-ran growth-scene with it and the
|
||||
printed `all` lines read r-candidate 11/9/0, r-base 0/11/9, r-free 0/80/0, as
|
||||
below, in Accepted/NoGain/Regressed):
|
||||
|
||||
- Every one of the 11 false accepts has R as the candidate (bases
|
||||
prompted-2, p4, p6); every one of the 9 false rejects has R as the base. The
|
||||
80 trials without R read 0 Accepted and 0 Regressed in every class (95% upper
|
||||
bound 4.5%). R scores above the other prompted draws (+0.012 en-US and +0.023
|
||||
ja-JP on all items), so R is not exchangeable with a repeat.
|
||||
Which of the two rates the live rule should be held to is undecided: the
|
||||
live loop meets R as the head's establishing lowering, so the R-candidate
|
||||
rate may be the real one for cycles, while the 80 R-free trials are the
|
||||
exchangeable repeat rate; neither number stands for the other.
|
||||
- The old/new split of the false-accept lives only in the 20 trials where R is
|
||||
the candidate, from 5 bases: new 10/20 [0.27, 0.73], old 0/20 [0.00, 0.17]
|
||||
(Clopper-Pearson); by base 3 of 5 against 0 of 5 (Fisher p 0.17). The
|
||||
|
|
|
|||
Loading…
Reference in a new issue