calibrate: tests that tell arms and languages apart; map notes the harness now prints them
Soul arms s1 finding: unit test could not distinguish arms, languages or the R-included mean. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
064483dba7
commit
a85d410e77
2 changed files with 37 additions and 1 deletions
|
|
@ -3115,6 +3115,38 @@ mod tests {
|
|||
.collect()
|
||||
}
|
||||
|
||||
/// A wobble-free draw whose items all score `bases[language]` (chrF as a percentage, meaning as a fraction).
|
||||
fn flat(bases: [f64; 2], items: usize) -> Vec<Vec<ItemScore>> {
|
||||
bases.iter().map(|&b| (0..items).map(|_| scored(100.0 * b, b)).collect()).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn arm_reports_attribute_each_arm_and_language() {
|
||||
// R (port 0) is above the five other prompted draws; the two languages differ everywhere.
|
||||
let mut draws = vec![flat([0.8, 0.7], 8)];
|
||||
draws.extend((0..5).map(|_| flat([0.6, 0.6], 8)));
|
||||
// Degraded arms: composites differ per arm and per language.
|
||||
draws.push(flat([0.5, 0.2], 8));
|
||||
draws.push(flat([0.1, 0.4], 8));
|
||||
// Candidate arms: far above, level with the prompted, far below.
|
||||
draws.push(flat([0.95, 0.95], 8));
|
||||
draws.push(flat([0.6, 0.6], 8));
|
||||
draws.push(flat([0.05, 0.05], 8));
|
||||
let t = calibration_trials(&draws, 6, 2, 3);
|
||||
// Offsets: arm minus the mean over all six prompted ports (R included): en 0.6333, ja 0.6167.
|
||||
let want = [[0.5 - 3.8 / 6.0, 0.2 - 3.7 / 6.0], [0.1 - 3.8 / 6.0, 0.4 - 3.7 / 6.0]];
|
||||
for (got, want) in t.degraded_offset.iter().flatten().zip(want.iter().flatten()) {
|
||||
assert!((got - want).abs() < 1e-9, "{:?} against {want:?}", t.degraded_offset);
|
||||
}
|
||||
// Each arm keeps its own counts: the high arm gains, the level one gains nothing, the low one regresses.
|
||||
let [high, level, low] = [t.candidate_arms[0], t.candidate_arms[1], t.candidate_arms[2]];
|
||||
assert_eq!((high.trials, level.trials, low.trials), (60, 60, 60));
|
||||
assert_eq!(high.accepted, 60);
|
||||
assert_eq!(level.accepted, 0);
|
||||
assert_eq!(low.regressed, 60);
|
||||
assert_eq!(t.candidate, high.plus(level).plus(low));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn calibration_trials_count_every_choice_and_read_the_fixed_cases() {
|
||||
// Six prompted draws, two degraded, three candidates, two languages of 8 items.
|
||||
|
|
|
|||
|
|
@ -548,7 +548,11 @@ except the R split in the first bullet, which `calibrate` prints as
|
|||
`prompted-r-candidate`, `prompted-r-base` and `prompted-r-free` since the
|
||||
`calibrate-report-r-involvement` cut; Soul re-ran growth-scene with it and the
|
||||
printed `all` lines read r-candidate 11/9/0, r-base 0/11/9, r-free 0/80/0, as
|
||||
below, in Accepted/NoGain/Regressed):
|
||||
below, in Accepted/NoGain/Regressed; likewise the per-arm Train-guide counts
|
||||
(`candidate=h1..h4`) and the mild arms' composite offsets from the prompted
|
||||
mean, R included (`offset`), which `calibrate` prints since
|
||||
`calibrate-arm-counts-and-mild-offsets` and which reproduce the figures below,
|
||||
Soul re-run on growth-scene):
|
||||
|
||||
- Every one of the 11 false accepts has R as the candidate (bases
|
||||
prompted-2, p4, p6); every one of the 9 false rejects has R as the base. The
|
||||
|
|
|
|||
Loading…
Reference in a new issue