calibrate: per-arm candidate counts, mild-arm composite offsets, line order pinned
Trials gains candidate_arms (pooled candidate is their sum) and degraded_offset (arm composite minus the prompted mean, per language). The CLI test pins the order of the class lines and the per-arm candidate lines. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
0761f4ae7c
commit
064483dba7
4 changed files with 71 additions and 13 deletions
|
|
@ -1299,11 +1299,17 @@ fn main() -> Result<()> {
|
||||||
run.labelled(&format!("calibrate class={} prompted-{side} {}", c.name, k.line()))
|
run.labelled(&format!("calibrate class={} prompted-{side} {}", c.name, k.line()))
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
for (label, d) in degraded.iter().zip(&t.degraded) {
|
for ((label, d), offsets) in degraded.iter().zip(&t.degraded).zip(&t.degraded_offset) {
|
||||||
|
let offsets: String = c
|
||||||
|
.languages
|
||||||
|
.iter()
|
||||||
|
.zip(offsets)
|
||||||
|
.map(|(l, o)| format!(" offset {}={o:+.4}", l.language))
|
||||||
|
.collect();
|
||||||
println!(
|
println!(
|
||||||
"{}",
|
"{}",
|
||||||
run.labelled(&format!(
|
run.labelled(&format!(
|
||||||
"calibrate class={} degraded={label} trials={} regressed={}",
|
"calibrate class={} degraded={label} trials={} regressed={}{offsets}",
|
||||||
c.name, d.trials, d.regressed
|
c.name, d.trials, d.regressed
|
||||||
))
|
))
|
||||||
);
|
);
|
||||||
|
|
@ -1315,6 +1321,12 @@ fn main() -> Result<()> {
|
||||||
c.name, t.candidate.trials, t.candidate.accepted, t.candidate.nogain, t.candidate.regressed
|
c.name, t.candidate.trials, t.candidate.accepted, t.candidate.nogain, t.candidate.regressed
|
||||||
))
|
))
|
||||||
);
|
);
|
||||||
|
for (label, k) in candidate.iter().zip(&t.candidate_arms) {
|
||||||
|
println!(
|
||||||
|
"{}",
|
||||||
|
run.labelled(&format!("calibrate class={} candidate={label} {}", c.name, k.line()))
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1918,8 +1918,15 @@ pub struct Trials {
|
||||||
pub r_free: Counts,
|
pub r_free: Counts,
|
||||||
/// Per degraded arm in the order given: the arm as the candidate.
|
/// Per degraded arm in the order given: the arm as the candidate.
|
||||||
pub degraded: Vec<Counts>,
|
pub degraded: Vec<Counts>,
|
||||||
/// Every candidate arm as the candidate, pooled.
|
/// Every candidate arm as the candidate, pooled: the sum of `candidate_arms`.
|
||||||
pub candidate: Counts,
|
pub candidate: Counts,
|
||||||
|
/// Per candidate arm in the order given: the arm as the candidate. Arms of
|
||||||
|
/// one kind can pool unlike draws, so the pooled count is not a power.
|
||||||
|
pub candidate_arms: Vec<Counts>,
|
||||||
|
/// Per degraded arm in the order given, per language: the arm's composite
|
||||||
|
/// minus the mean composite of the prompted draws (the recorded lowering
|
||||||
|
/// included), over the items read. NaN where a language has no items.
|
||||||
|
pub degraded_offset: Vec<Vec<f64>>,
|
||||||
/// The mean of [`panel_noise`] over every (base, floor) choice.
|
/// The mean of [`panel_noise`] over every (base, floor) choice.
|
||||||
pub noise: f64,
|
pub noise: f64,
|
||||||
}
|
}
|
||||||
|
|
@ -1975,7 +1982,7 @@ pub fn calibration_trials(
|
||||||
let pool: Vec<usize> = (0..prompted).collect();
|
let pool: Vec<usize> = (0..prompted).collect();
|
||||||
let without = |out: &[usize]| -> Vec<usize> { pool.iter().copied().filter(|p| !out.contains(p)).collect() };
|
let without = |out: &[usize]| -> Vec<usize> { pool.iter().copied().filter(|p| !out.contains(p)).collect() };
|
||||||
let (mut r_candidate, mut r_base, mut r_free) = (Counts::default(), Counts::default(), Counts::default());
|
let (mut r_candidate, mut r_base, mut r_free) = (Counts::default(), Counts::default(), Counts::default());
|
||||||
let mut candidate_counts = Counts::default();
|
let mut candidate_arms = vec![Counts::default(); candidate];
|
||||||
let mut degraded_counts = vec![Counts::default(); degraded];
|
let mut degraded_counts = vec![Counts::default(); degraded];
|
||||||
let (mut noise, mut noises) = (0.0, 0u32);
|
let (mut noise, mut noises) = (0.0, 0u32);
|
||||||
for &b in &pool {
|
for &b in &pool {
|
||||||
|
|
@ -1984,7 +1991,7 @@ pub fn calibration_trials(
|
||||||
counts.tally(decide_panel(&rows(b, prompted + d, &floors)));
|
counts.tally(decide_panel(&rows(b, prompted + d, &floors)));
|
||||||
}
|
}
|
||||||
for c in prompted + degraded..prompted + degraded + candidate {
|
for c in prompted + degraded..prompted + degraded + candidate {
|
||||||
candidate_counts.tally(decide_panel(&rows(b, c, &floors)));
|
candidate_arms[c - prompted - degraded].tally(decide_panel(&rows(b, c, &floors)));
|
||||||
}
|
}
|
||||||
noise += panel_noise(&rows(b, b, &floors));
|
noise += panel_noise(&rows(b, b, &floors));
|
||||||
noises += 1;
|
noises += 1;
|
||||||
|
|
@ -2002,7 +2009,20 @@ pub fn calibration_trials(
|
||||||
r_base,
|
r_base,
|
||||||
r_free,
|
r_free,
|
||||||
degraded: degraded_counts,
|
degraded: degraded_counts,
|
||||||
candidate: candidate_counts,
|
candidate: candidate_arms.iter().fold(Counts::default(), |a, &k| a.plus(k)),
|
||||||
|
candidate_arms,
|
||||||
|
degraded_offset: (0..degraded)
|
||||||
|
.map(|d| {
|
||||||
|
(0..languages)
|
||||||
|
.map(|l| {
|
||||||
|
let composite = |p: usize| {
|
||||||
|
reading_of(&draws[p][l]).map_or(f64::NAN, |r| r.composite)
|
||||||
|
};
|
||||||
|
composite(prompted + d) - pool.iter().map(|&p| composite(p)).sum::<f64>() / prompted as f64
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
noise: noise / f64::from(noises),
|
noise: noise / f64::from(noises),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -3114,6 +3134,13 @@ mod tests {
|
||||||
// A far better draw is accepted in every trial.
|
// A far better draw is accepted in every trial.
|
||||||
assert_eq!(t.candidate.accepted, t.candidate.trials);
|
assert_eq!(t.candidate.accepted, t.candidate.trials);
|
||||||
|
|
||||||
|
// Per-arm candidate counts sum to the pooled ones; a degraded draw of
|
||||||
|
// all zeros sits below the prompted mean (0.6 chrF and meaning) in both languages.
|
||||||
|
assert_eq!(t.candidate_arms.len(), 3);
|
||||||
|
assert_eq!(t.candidate_arms.iter().map(|k| k.trials).sum::<u32>(), t.candidate.trials);
|
||||||
|
assert_eq!(t.candidate_arms.iter().fold(Counts::default(), |a, &k| a.plus(k)), t.candidate);
|
||||||
|
assert_eq!((t.degraded_offset.len(), t.degraded_offset[0].len()), (2, 2));
|
||||||
|
assert!(t.degraded_offset.iter().flatten().all(|o| (*o + 0.6).abs() < 0.05), "{:?}", t.degraded_offset);
|
||||||
// R (port 0) scores above the other prompted draws: it is Accepted as a
|
// R (port 0) scores above the other prompted draws: it is Accepted as a
|
||||||
// candidate and Regressed as a base, and the trials without it read NoGain.
|
// candidate and Regressed as a base, and the trials without it read NoGain.
|
||||||
let mut r_high: Vec<_> = (0..6).map(|s| draw(2, 8, if s == 0 { 0.8 } else { 0.6 }, 0.02, s)).collect();
|
let mut r_high: Vec<_> = (0..6).map(|s| draw(2, 8, if s == 0 { 0.8 } else { 0.6 }, 0.02, s)).collect();
|
||||||
|
|
|
||||||
|
|
@ -1914,7 +1914,7 @@ fn cli_loop_calibrate_reads_answered_draws_and_refuses_unanswered_and_bad_labels
|
||||||
.collect();
|
.collect();
|
||||||
assert_eq!(digests.len(), 2);
|
assert_eq!(digests.len(), 2);
|
||||||
// `P1` and `a_b` are answered too, so only the label check can refuse them.
|
// `P1` and `a_b` are answered too, so only the label check can refuse them.
|
||||||
for label in ["p1", "p2", "p3", "p4", "bad", "P1", "a_b"] {
|
for label in ["p1", "p2", "p3", "p4", "bad", "cand1", "cand2", "P1", "a_b"] {
|
||||||
std::fs::create_dir_all(root.join(format!("relower-1-{label}"))).unwrap();
|
std::fs::create_dir_all(root.join(format!("relower-1-{label}"))).unwrap();
|
||||||
for digest in &digests {
|
for digest in &digests {
|
||||||
std::fs::write(root.join(format!("relower-1-{label}/{digest}.answer")), said).unwrap();
|
std::fs::write(root.join(format!("relower-1-{label}/{digest}.answer")), said).unwrap();
|
||||||
|
|
@ -1947,16 +1947,26 @@ fn cli_loop_calibrate_reads_answered_draws_and_refuses_unanswered_and_bad_labels
|
||||||
assert!(!out(&o).contains("unanswered"), "{args:?} must be refused for itself: {}", out(&o));
|
assert!(!out(&o).contains("unanswered"), "{args:?} must be refused for itself: {}", out(&o));
|
||||||
}
|
}
|
||||||
|
|
||||||
let o = calibrate(&[&four[..], &["--degraded", "bad", "--class", "old=item:daodejing"]].concat());
|
let o = calibrate(
|
||||||
|
&[&four[..], &["--degraded", "bad", "--candidate", "cand1", "--candidate", "cand2", "--class", "old=item:daodejing"]].concat(),
|
||||||
|
);
|
||||||
assert!(o.status.success(), "{}", out(&o));
|
assert!(o.status.success(), "{}", out(&o));
|
||||||
let text = out(&o);
|
let text = out(&o);
|
||||||
assert!(text.starts_with("calibrate draws prompted=4 degraded=1 candidate=0 floor-arms=3 unusable=0"), "{text}");
|
assert!(text.starts_with("calibrate draws prompted=4 degraded=1 candidate=2 floor-arms=3 unusable=0"), "{text}");
|
||||||
assert!(text.contains("calibrate class=all items en-US n=2 skipped=0"), "{text}");
|
assert!(text.contains("calibrate class=all items en-US n=2 skipped=0"), "{text}");
|
||||||
assert!(text.contains("calibrate class=all prompted trials=20 nogain=20 accepted=0 regressed=0"), "{text}");
|
assert!(text.contains("calibrate class=all prompted trials=20 nogain=20 accepted=0 regressed=0"), "{text}");
|
||||||
for side in ["r-candidate trials=4", "r-base trials=4", "r-free trials=12"] {
|
for side in ["r-candidate trials=4", "r-base trials=4", "r-free trials=12"] {
|
||||||
assert!(text.contains(&format!("calibrate class=all prompted-{side} nogain=")), "{text}");
|
assert!(text.contains(&format!("calibrate class=all prompted-{side} nogain=")), "{text}");
|
||||||
}
|
}
|
||||||
assert!(text.contains("calibrate class=old degraded=bad trials=20 regressed=0"), "{text}");
|
assert!(text.contains("calibrate class=old degraded=bad trials=20 regressed=0 offset en-US=+0.0000"), "{text}");
|
||||||
|
// The class=all lines in print order: prompted, its three R roles, degraded, pooled candidate, each candidate arm.
|
||||||
|
let all: Vec<&str> = text.lines().filter(|l| l.contains("calibrate class=all ")).collect();
|
||||||
|
let kinds = ["items", "prompted trials", "prompted-r-candidate", "prompted-r-base", "prompted-r-free", "degraded=bad", "candidate trials", "candidate=cand1", "candidate=cand2"];
|
||||||
|
assert_eq!(all.len(), kinds.len(), "{text}");
|
||||||
|
for (line, kind) in all.iter().zip(kinds) {
|
||||||
|
assert!(line.contains(&format!("class=all {kind}")), "{kind}: {line}");
|
||||||
|
}
|
||||||
|
assert!(all[7].contains("candidate=cand1 trials=20 nogain=20 accepted=0 regressed=0"), "{}", all[7]);
|
||||||
for leaked in ["item:daodejing", "packet:", "highest excellence", "canaryroot"] {
|
for leaked in ["item:daodejing", "packet:", "highest excellence", "canaryroot"] {
|
||||||
assert!(!text.contains(leaked), "{text}");
|
assert!(!text.contains(leaked), "{text}");
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -11,10 +11,12 @@ Regressed counts, and the mean panel noise. That is all it produces: the
|
||||||
Accepted/NoGain/Regressed counts and the noise mean in the map's "Recalibration
|
Accepted/NoGain/Regressed counts and the noise mean in the map's "Recalibration
|
||||||
on the grown corpus" block are reproduced by it, and the prompted trials are
|
on the grown corpus" block are reproduced by it, and the prompted trials are
|
||||||
also printed split by the recorded lowering `R`'s role (`prompted-r-candidate`,
|
also printed split by the recorded lowering `R`'s role (`prompted-r-candidate`,
|
||||||
`prompted-r-base`, `prompted-r-free`, see below); the other figures there (the
|
`prompted-r-base`, `prompted-r-free`, see below), the candidate counts per arm
|
||||||
|
and the mild arms' composite offsets from the prompted mean (see below); the
|
||||||
|
other figures there (the
|
||||||
35% with R removed, the noise range, the 0.0185 gain and its 0.41 ratio, the
|
35% with R removed, the noise range, the 0.0185 gain and its 0.41 ratio, the
|
||||||
bootstrap intervals, the per-item SDs, R's +0.024, the mild-arm duplicate
|
bootstrap intervals, the per-item SDs, R's +0.024, the mild-arm duplicate
|
||||||
counts and mean composite offsets, the Train-vocabulary overlap, the old/new
|
counts, the Train-vocabulary overlap, the old/new
|
||||||
class composites and the ten parse-refused item ids) were computed by scratch scripts that are not in the repo. It is a
|
class composites and the ten parse-refused item ids) were computed by scratch scripts that are not in the repo. It is a
|
||||||
diagnostic: it records nothing, writes no prompts, and never touches Test or
|
diagnostic: it records nothing, writes no prompts, and never touches Test or
|
||||||
Train.
|
Train.
|
||||||
|
|
@ -44,7 +46,14 @@ draws; the floor size is `FLOOR_ARMS_MIN` (3), the live rule's.
|
||||||
3-subset of the rest of `P`, the degraded draw as the candidate. These should
|
3-subset of the rest of `P`, the degraded draw as the candidate. These should
|
||||||
read Regressed.
|
read Regressed.
|
||||||
- Candidate trials, pooled over the `--candidate` arms: the same, tallied
|
- Candidate trials, pooled over the `--candidate` arms: the same, tallied
|
||||||
Accepted, NoGain, Regressed. This is the rule's power against that arm.
|
Accepted, NoGain, Regressed. This is the rule's power against that arm. The
|
||||||
|
pooled line is followed by one `candidate=LABEL` line per arm: arms of one
|
||||||
|
kind can differ completely (Train-guide h1 and h2 Regressed in every trial
|
||||||
|
while h3 and h4 gained), and then the pooled figure is not a power.
|
||||||
|
- Mild offset: each `degraded=LABEL` line ends with `offset LANGUAGE=+x.xxxx`, the
|
||||||
|
arm's composite minus the mean composite of the prompted draws (`R` included),
|
||||||
|
per language over the items read. A degraded arm that is barely below the
|
||||||
|
prompted mean is a weak probe of the rule's sensitivity.
|
||||||
- Noise: the mean `panel_noise` over every (base, floor) choice.
|
- Noise: the mean `panel_noise` over every (base, floor) choice.
|
||||||
|
|
||||||
With six prompted draws including `R` that is 120 prompted trials, 60 per
|
With six prompted draws including `R` that is 120 prompted trials, 60 per
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue