relower: paired per-item noise floor over >=3 prompted repeats; panel gain clears the noise of the sum (#70)
This commit is contained in:
parent
97aa026498
commit
353dd2138f
5 changed files with 246 additions and 101 deletions
|
|
@ -154,8 +154,9 @@ enum LoopCommand {
|
|||
ollama_url: String,
|
||||
},
|
||||
/// Re-lower round ROUND's Validation packets through arm ARM, score them
|
||||
/// against the items, and, with --floor ARM, decide against that second arm
|
||||
/// by the panel accept rule. Prompts and answers live in
|
||||
/// against the items, and, with --floor ARM given two or more times (prompted
|
||||
/// repeats of the lowering), decide against the paired noise floor of those
|
||||
/// repeats by the panel accept rule. Prompts and answers live in
|
||||
/// `<ROOT>/relower-<ROUND>-<ARM>`; the command exits 3 until the arm is
|
||||
/// answered. --lowerer names the arm's model in the printed line. A
|
||||
/// diagnostic, never a gate; records nothing; Test and Train are never
|
||||
|
|
@ -169,7 +170,7 @@ enum LoopCommand {
|
|||
#[arg(long)]
|
||||
lowerer: String,
|
||||
#[arg(long)]
|
||||
floor: Option<String>,
|
||||
floor: Vec<String>,
|
||||
#[arg(long, default_value = "http://127.0.0.1:11434")]
|
||||
ollama_url: String,
|
||||
},
|
||||
|
|
@ -1147,7 +1148,10 @@ fn main() -> Result<()> {
|
|||
(1..=32).contains(&l.len())
|
||||
&& l.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-')
|
||||
};
|
||||
if !label(&arm) || floor.as_deref().is_some_and(|f| !label(f) || f == arm) {
|
||||
if !label(&arm)
|
||||
|| floor.iter().any(|f| !label(f) || *f == arm)
|
||||
|| floor.iter().enumerate().any(|(i, f)| floor[..i].contains(f))
|
||||
{
|
||||
anyhow::bail!("arm and floor labels must match [a-z0-9-]{{1,32}} and differ");
|
||||
}
|
||||
let run = RunStores::open(&root)?;
|
||||
|
|
@ -1156,16 +1160,19 @@ fn main() -> Result<()> {
|
|||
run.record().model.clone(),
|
||||
);
|
||||
let mut files = run.answer_files(run.relower_dir(round, &arm))?;
|
||||
let mut floor_port = floor.as_deref().map(|f| {
|
||||
RecordedAnswers::new(run.relower_dir(round, f), run.record().model.clone())
|
||||
});
|
||||
let mut floor_ports: Vec<RecordedAnswers> = floor
|
||||
.iter()
|
||||
.map(|f| RecordedAnswers::new(run.relower_dir(round, f), run.record().model.clone()))
|
||||
.collect();
|
||||
let mut floor_dyn: Vec<&mut dyn ModelPort> =
|
||||
floor_ports.iter_mut().map(|p| p as &mut dyn ModelPort).collect();
|
||||
let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder));
|
||||
match run.relower(
|
||||
round,
|
||||
&arm,
|
||||
&mut recorded,
|
||||
&mut files,
|
||||
floor_port.as_mut().map(|p| p as &mut dyn ModelPort),
|
||||
&mut floor_dyn,
|
||||
&scorer,
|
||||
)? {
|
||||
RelowerOutcome::Unanswered { prompts } => {
|
||||
|
|
|
|||
|
|
@ -498,6 +498,8 @@ pub enum RunError {
|
|||
/// The floor arm of a relower holds no answer to a prompt; it is a
|
||||
/// reference, never written to, and must be answered whole first.
|
||||
FloorUnanswered,
|
||||
/// A relower was given fewer floor arms than [`FLOOR_ARMS_MIN`].
|
||||
FloorArms,
|
||||
}
|
||||
|
||||
impl fmt::Display for RunError {
|
||||
|
|
@ -526,6 +528,7 @@ impl fmt::Display for RunError {
|
|||
RunError::SpeakerState => {
|
||||
f.write_str("a scene does not read against the run's inventory")
|
||||
}
|
||||
RunError::FloorArms => f.write_str("a floor needs three or more floor arms; fewer measure no noise"),
|
||||
RunError::FloorUnanswered => {
|
||||
f.write_str("the floor arm holds no answer to a lowering prompt; answer it whole first")
|
||||
}
|
||||
|
|
@ -1586,13 +1589,14 @@ pub struct RelowerReading {
|
|||
pub skipped: u32,
|
||||
/// Arm answers that were not a usable lowering; each scored 0.
|
||||
pub unusable: u32,
|
||||
/// Floor-arm answers that were not a usable lowering; each scored 0, which
|
||||
/// widens `floor`. Zero without a floor arm.
|
||||
/// Floor-arm answers that were not a usable lowering, over every floor arm;
|
||||
/// each scored 0, which widens `floor`. Zero without floor arms.
|
||||
pub floor_unusable: u32,
|
||||
/// The round's own lowering, read against the items.
|
||||
pub recorded: Reading,
|
||||
pub arm: Reading,
|
||||
/// `spread(recorded, floor arm)`, when a floor arm was given.
|
||||
/// [`paired_floor`] of the recorded lowering against the floor arms, when
|
||||
/// floor arms were given.
|
||||
pub floor: Option<Reading>,
|
||||
}
|
||||
|
||||
|
|
@ -1637,9 +1641,10 @@ impl RunStores {
|
|||
/// distil decision, whether a tuned lowerer is worth keeping against the
|
||||
/// prompted one (calibration-pending: see the map). Asks `arm` the byte-identical lowering prompts of round `n`'s live
|
||||
/// Validation items, reads the arm and the round's own lowering
|
||||
/// (`recorded`) against the items, and, given a `floor` arm (a second
|
||||
/// prompted lowering, so the floor is lowering-only noise), decides by
|
||||
/// [`decide_panel`] over (recorded, arm, spread(recorded, floor)). Only
|
||||
/// (`recorded`) against the items, and, given `floors` (three or more
|
||||
/// prompted repeats of the recorded lowering, so the floor is lowering-only
|
||||
/// noise), decides by [`decide_panel`] over (recorded, arm,
|
||||
/// [`paired_floor`]). Only
|
||||
/// the model of `arm` is unpinned; the embedder is checked. An arm answer
|
||||
/// that is not a usable lowering scores 0 and is counted in `unusable`.
|
||||
/// Records nothing; `arm_label` names only where the scratch packet store
|
||||
|
|
@ -1650,7 +1655,7 @@ impl RunStores {
|
|||
arm_label: &str,
|
||||
recorded: &mut dyn ModelPort,
|
||||
arm: &mut dyn ModelPort,
|
||||
mut floor: Option<&mut dyn ModelPort>,
|
||||
floors: &mut [&mut dyn ModelPort],
|
||||
scorer: &Scorer<E>,
|
||||
) -> Result<RelowerOutcome, RunError> {
|
||||
let store = |_| RunError::Store;
|
||||
|
|
@ -1666,9 +1671,12 @@ impl RunStores {
|
|||
}
|
||||
let mut scratch = PacketStore::open(&dir.join("base.cc")).map_err(store)?;
|
||||
|
||||
// Per language, per item: the item text, then the recorded, arm and floor
|
||||
// lowerings (`None` when unusable).
|
||||
type Row = (String, Option<String>, Option<String>, Option<String>);
|
||||
if (1..FLOOR_ARMS_MIN).contains(&floors.len()) {
|
||||
return Err(RunError::FloorArms);
|
||||
}
|
||||
// Per language, per item: the item text, then the recorded and arm
|
||||
// lowerings and one per floor arm (`None` when unusable).
|
||||
type Row = (String, Option<String>, Option<String>, Vec<Option<String>>);
|
||||
let mut waiting: Vec<String> = Vec::new();
|
||||
let mut floor_waiting = false;
|
||||
let mut rows: Vec<Vec<Row>> = Vec::new();
|
||||
|
|
@ -1690,16 +1698,16 @@ impl RunStores {
|
|||
Asked::Waiting => return Err(RunError::Unrecorded),
|
||||
other => text(other),
|
||||
};
|
||||
let floor_text = match floor.as_deref_mut() {
|
||||
Some(f) => match asked(f, &prompt)? {
|
||||
let mut floor_texts = Vec::new();
|
||||
for f in floors.iter_mut() {
|
||||
floor_texts.push(match asked(&mut **f, &prompt)? {
|
||||
Asked::Waiting => {
|
||||
floor_waiting = true;
|
||||
None
|
||||
}
|
||||
other => text(other),
|
||||
},
|
||||
None => None,
|
||||
};
|
||||
});
|
||||
}
|
||||
let arm_text = match asked(arm, &prompt)? {
|
||||
Asked::Waiting => {
|
||||
let d = prompt.digest();
|
||||
|
|
@ -1710,7 +1718,7 @@ impl RunStores {
|
|||
}
|
||||
other => text(other),
|
||||
};
|
||||
language_rows.push((item.text.clone(), recorded_text, arm_text, floor_text));
|
||||
language_rows.push((item.text.clone(), recorded_text, arm_text, floor_texts));
|
||||
}
|
||||
rows.push(language_rows);
|
||||
}
|
||||
|
|
@ -1723,8 +1731,8 @@ impl RunStores {
|
|||
|
||||
// One score call: each usable lowering against its item, in row order.
|
||||
let mut inputs: Vec<ScoreInput> = Vec::new();
|
||||
for (reference, recorded, arm, floor_text) in rows.iter().flatten() {
|
||||
for output in [recorded, arm, floor_text].into_iter().flatten() {
|
||||
for (reference, recorded, arm, floor_texts) in rows.iter().flatten() {
|
||||
for output in [recorded, arm].into_iter().chain(floor_texts).flatten() {
|
||||
inputs.push(ScoreInput { reference, output });
|
||||
}
|
||||
}
|
||||
|
|
@ -1751,7 +1759,8 @@ impl RunStores {
|
|||
let mut languages = Vec::new();
|
||||
let live = self.corpus.all().map_err(store)?;
|
||||
for ((language, _, _), language_rows) in held.iter().zip(&rows) {
|
||||
let (mut rec, mut arm_scores, mut floor_scores) = (Vec::new(), Vec::new(), Vec::new());
|
||||
let (mut rec, mut arm_scores) = (Vec::new(), Vec::new());
|
||||
let mut floor_scores: Vec<Vec<ItemScore>> = vec![Vec::new(); floors.len()];
|
||||
let (mut unusable, mut floor_unusable) = (0u32, 0u32);
|
||||
for (_, r, a, f) in language_rows {
|
||||
rec.push(next(r)?);
|
||||
|
|
@ -1759,9 +1768,9 @@ impl RunStores {
|
|||
if a.is_none() {
|
||||
unusable += 1;
|
||||
}
|
||||
if floor.is_some() {
|
||||
floor_scores.push(next(f)?);
|
||||
if f.is_none() {
|
||||
for (scores, t) in floor_scores.iter_mut().zip(f) {
|
||||
scores.push(next(t)?);
|
||||
if t.is_none() {
|
||||
floor_unusable += 1;
|
||||
}
|
||||
}
|
||||
|
|
@ -1781,14 +1790,10 @@ impl RunStores {
|
|||
floor_unusable,
|
||||
recorded: recorded_reading,
|
||||
arm: reading_of(&arm_scores)?,
|
||||
floor: if floor.is_some() {
|
||||
Some(spread(&recorded_reading, &reading_of(&floor_scores)?))
|
||||
} else {
|
||||
None
|
||||
},
|
||||
floor: if floors.is_empty() { None } else { Some(paired_floor(&rec, &floor_scores)) },
|
||||
});
|
||||
}
|
||||
let verdict = if floor.is_some() {
|
||||
let verdict = if !floors.is_empty() {
|
||||
let panel: Vec<(Reading, Reading, Reading)> = languages
|
||||
.iter()
|
||||
.filter_map(|l| Some((l.recorded, l.arm, l.floor?)))
|
||||
|
|
@ -2261,7 +2266,11 @@ pub fn decide(base: &Reading, candidate: &Reading, floor: &Reading) -> Verdict {
|
|||
/// decides whether a language regressed (any regression rejects; the first in
|
||||
/// panel order names the metric), and the gain test is [`decide`]'s for one
|
||||
/// language and, for two or more, the panel's: the sum over languages of the
|
||||
/// composite deltas (billionths) must exceed one [`LADDER_STEP`]. A language
|
||||
/// composite deltas (billionths) must exceed the larger of one [`LADDER_STEP`]
|
||||
/// and the noise of that sum, the root of the languages' composite floors
|
||||
/// squared (independent languages; a one-language panel's noise is its floor,
|
||||
/// as in [`decide`]). Without that noise term a prompted repeat of the base
|
||||
/// lowering read Accepted in 40% of the calibration trials on `distil-base` at `FLOOR_Z` 3. A language
|
||||
/// may lose up to its own noise floor if the panel nets more than a step; the
|
||||
/// rule gives up strict Pareto non-regression, which on noisy per-language
|
||||
/// readings ratchets every base to a favourable draw and strangles the loop
|
||||
|
|
@ -2289,7 +2298,10 @@ pub fn decide_panel(rows: &[(Reading, Reading, Reading)]) -> Verdict {
|
|||
.iter()
|
||||
.map(|(base, candidate, _)| nano(candidate.composite - base.composite))
|
||||
.sum();
|
||||
if net > nano(LADDER_STEP) {
|
||||
// The noise of a sum of independent languages' differences: the root of the
|
||||
// sum of their floors squared.
|
||||
let noise = rows.iter().map(|(_, _, floor)| floor.composite.powi(2)).sum::<f64>().sqrt();
|
||||
if net > nano(noise.max(LADDER_STEP)) {
|
||||
Verdict::Accepted
|
||||
} else {
|
||||
Verdict::Rejected(Why::NoGain)
|
||||
|
|
@ -2488,6 +2500,50 @@ fn floor_of(
|
|||
.collect()
|
||||
}
|
||||
|
||||
/// How many standard errors of a prompted-vs-prompted mean difference the
|
||||
/// relower floor is wide, and the fewest floor arms (prompted repeats of the
|
||||
/// recorded lowering) it is measured over. Calibrated on `runs/distil-base`
|
||||
/// (eight prompted draws of 14 en-US and 14 ja-JP Validation items, 28
|
||||
/// unordered pairs): with three floor arms a prompted repeat reads NoGain in
|
||||
/// 92% of the trials (bootstrap 95% over draws 80 to 100%), Regressed 8% and
|
||||
/// Accepted 0%; a "Yes." arm and a mild degradation (one detail blurred per
|
||||
/// utterance) read Regressed in 100%, and a subtler one in 75% (50 to 100%): at
|
||||
/// 14 items a degradation under about three standard errors cannot be told from
|
||||
/// noise. `FLOOR_Z` was chosen on that data, so the rates are in-sample.
|
||||
pub const FLOOR_Z: f64 = 3.0;
|
||||
pub const FLOOR_ARMS_MIN: usize = 3;
|
||||
|
||||
/// The lowering-only noise floor of a relower: `FLOOR_Z` standard errors of the
|
||||
/// mean per-item difference between the recorded lowering and a repeat of it,
|
||||
/// metric by metric. Each floor arm gives one paired sample of per-item
|
||||
/// differences; their variances are pooled over the arms, so the floor rests on
|
||||
/// `arms * (n - 1)` degrees of freedom instead of the one coincidental
|
||||
/// difference of means a two-run spread is (which reads 0 whenever the two
|
||||
/// means happen to meet, and then a one-step dip regresses). Paired per item,
|
||||
/// after Koehn (2004). NaN (fail closed in [`decide`]) when there are fewer
|
||||
/// than two items or no arm (0/0), and when an arm is ragged.
|
||||
pub fn paired_floor(recorded: &[ItemScore], arms: &[Vec<ItemScore>]) -> Reading {
|
||||
let n = recorded.len();
|
||||
if arms.iter().any(|a| a.len() != n) {
|
||||
return Reading { chrf: f64::NAN, meaning: f64::NAN, composite: f64::NAN };
|
||||
}
|
||||
let pooled = |of: &dyn Fn(&ItemScore) -> f64| {
|
||||
let variance = |arm: &Vec<ItemScore>| {
|
||||
let d: Vec<f64> = recorded.iter().zip(arm).map(|(r, a)| of(r) - of(a)).collect();
|
||||
let mean = d.iter().sum::<f64>() / n as f64;
|
||||
d.iter().map(|x| (x - mean).powi(2)).sum::<f64>() / (n - 1) as f64
|
||||
};
|
||||
let v = arms.iter().map(variance).sum::<f64>() / arms.len() as f64;
|
||||
FLOOR_Z * (v / n as f64).sqrt()
|
||||
};
|
||||
let meaning = |s: &ItemScore| s.meaning.unwrap_or(0.0);
|
||||
Reading {
|
||||
chrf: pooled(&|s| s.chrf),
|
||||
meaning: pooled(&meaning),
|
||||
composite: pooled(&|s| (s.chrf / 100.0 + meaning(s)) / 2.0),
|
||||
}
|
||||
}
|
||||
|
||||
/// The absolute difference of two readings, metric by metric.
|
||||
fn spread(a: &Reading, b: &Reading) -> Reading {
|
||||
Reading {
|
||||
|
|
@ -2869,6 +2925,52 @@ mod tests {
|
|||
assert_eq!(decide(&base, &gain(0.506), &floor), rejected(Why::NoGain));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn paired_floor_is_z_standard_errors_of_the_paired_difference() {
|
||||
let item = |chrf: f64, meaning: f64| ItemScore { chrf, meaning: Some(meaning) };
|
||||
let recorded = [item(60.0, 0.8), item(40.0, 0.6), item(50.0, 0.7), item(30.0, 0.5)];
|
||||
// An arm that differs by a constant has no spread of differences: a
|
||||
// shifted mean is the thing the floor is read against, not noise.
|
||||
let shifted: Vec<ItemScore> = recorded.iter().map(|s| item(s.chrf - 5.0, s.meaning.unwrap() - 0.1)).collect();
|
||||
let f = paired_floor(&recorded, &[shifted.clone(), shifted.clone(), shifted]);
|
||||
assert!(f.chrf.abs() < 1e-9 && f.meaning.abs() < 1e-9 && f.composite.abs() < 1e-9, "{f:?}");
|
||||
// Differences 2, -2, 2, -2 on chrF: sample variance 16/3, standard error
|
||||
// sqrt(16/3/4), FLOOR_Z of it; arms pool, so a second equal arm changes nothing.
|
||||
let wobble_arm = |recorded: &[ItemScore]| -> Vec<ItemScore> {
|
||||
recorded
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, s)| item(s.chrf - if i % 2 == 0 { 2.0 } else { -2.0 }, s.meaning.unwrap()))
|
||||
.collect()
|
||||
};
|
||||
let wobble = wobble_arm(&recorded);
|
||||
let want = FLOOR_Z * (16.0_f64 / 3.0 / 4.0).sqrt();
|
||||
let one = paired_floor(&recorded, std::slice::from_ref(&wobble));
|
||||
let two = paired_floor(&recorded, &[wobble.clone(), wobble.clone()]);
|
||||
assert!((one.chrf - want).abs() < 1e-9 && (two.chrf - want).abs() < 1e-9, "{one:?} {two:?}");
|
||||
assert!(one.meaning.abs() < 1e-9);
|
||||
// Arms pool by the mean of their variances: one wobbling arm and one
|
||||
// exact repeat give the root of half the variance.
|
||||
let exact = recorded.to_vec();
|
||||
let mixed = paired_floor(&recorded, &[wobble.clone(), exact]);
|
||||
assert!((mixed.chrf - FLOOR_Z * (16.0_f64 / 3.0 / 2.0 / 4.0).sqrt()).abs() < 1e-9, "{mixed:?}");
|
||||
// The calibrated constants, pinned apart from the formula.
|
||||
assert_eq!((FLOOR_Z, FLOOR_ARMS_MIN), (3.0, 3));
|
||||
// Under two items, no arms or a ragged arm there is no floor: NaN fails
|
||||
// closed in `decide`.
|
||||
for f in [
|
||||
paired_floor(&recorded[..1], &[vec![item(1.0, 1.0)]]),
|
||||
paired_floor(&recorded, &[]),
|
||||
paired_floor(&recorded, &[vec![item(1.0, 1.0)]]),
|
||||
] {
|
||||
assert!(f.composite.is_nan());
|
||||
assert_eq!(
|
||||
decide(&reading(50.0, 0.5, 0.5), &reading(50.0, 0.5, 0.9), &f),
|
||||
Verdict::Rejected(Why::Regressed(Metric::Composite))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decide_panel_table() {
|
||||
let zero = reading(0.0, 0.0, 0.0);
|
||||
|
|
@ -2922,9 +3024,8 @@ mod tests {
|
|||
assert_eq!(net(LADDER_STEP + 6e-10), Verdict::Accepted);
|
||||
assert_eq!(net(LADDER_STEP + 2e-9), Verdict::Accepted);
|
||||
// Liveness: no language need pass its own threshold. Two languages each
|
||||
// gaining 0.6 of a step at zero floor accept; so does one gaining two
|
||||
// steps under a 0.02 floor (its own threshold) with the others flat; so
|
||||
// does a gain with another language falling inside its own floor.
|
||||
// gaining 0.6 of a step at zero floor accept; so does a gain past the
|
||||
// noise of the sum with another language falling inside its own floor.
|
||||
let each = 0.6 * LADDER_STEP;
|
||||
assert_eq!(
|
||||
decide_panel(&[
|
||||
|
|
@ -2933,27 +3034,32 @@ mod tests {
|
|||
]),
|
||||
Verdict::Accepted
|
||||
);
|
||||
// The sum must also clear its own noise: the root of the floors squared
|
||||
// (0.02 and 0.02 give 0.0283), not just a step. A repeat of the base
|
||||
// lowering gains that much by chance.
|
||||
let big_floor = reading(0.0, 0.0, 0.02);
|
||||
assert_eq!(
|
||||
decide_panel(&[(base, reading(50.0, 0.5, 0.5 + 2.0 * LADDER_STEP), big_floor), (base, flat, big_floor)]),
|
||||
Verdict::Accepted
|
||||
);
|
||||
let sum_of = |gain: f64| {
|
||||
decide_panel(&[(base, reading(50.0, 0.5, 0.5 + gain), big_floor), (base, flat, big_floor)])
|
||||
};
|
||||
assert_eq!(sum_of(0.02), rejected(Why::NoGain), "past a step, under the noise of the sum");
|
||||
assert_eq!(sum_of(0.028), rejected(Why::NoGain));
|
||||
assert_eq!(sum_of(0.029), Verdict::Accepted);
|
||||
let own_floor = reading(0.0, 0.0, 0.004);
|
||||
assert_eq!(
|
||||
decide_panel(&[
|
||||
(base, reading(50.0, 0.5, 0.55), reading(0.0, 0.0, 0.03)),
|
||||
(base, reading(50.0, 0.5, 0.497), own_floor)
|
||||
]),
|
||||
Verdict::Accepted,
|
||||
"a fall inside the language's floor, net 0.047 over a noise of 0.0303"
|
||||
);
|
||||
assert_eq!(
|
||||
decide_panel(&[
|
||||
(base, reading(50.0, 0.5, 0.51), reading(0.0, 0.0, 0.03)),
|
||||
(base, reading(50.0, 0.5, 0.497), own_floor)
|
||||
]),
|
||||
Verdict::Accepted,
|
||||
"a fall inside the language's floor, net 0.007"
|
||||
);
|
||||
// The same fall with the net exactly a step rejects.
|
||||
assert_eq!(
|
||||
decide_panel(&[
|
||||
(base, reading(50.0, 0.5, 0.508), reading(0.0, 0.0, 0.03)),
|
||||
(base, reading(50.0, 0.5, 0.497), own_floor)
|
||||
]),
|
||||
rejected(Why::NoGain)
|
||||
rejected(Why::NoGain),
|
||||
"net 0.007 is under the noise"
|
||||
);
|
||||
// The sum is over the deltas, with each language's fall counted.
|
||||
assert_eq!(
|
||||
|
|
|
|||
|
|
@ -1833,12 +1833,19 @@ fn cli_loop_relower_exits_3_until_answered_and_refuses_bad_labels() {
|
|||
assert_eq!(o.status.code(), Some(1), "{}", out(&o));
|
||||
assert!(!root.join("relower-1-second").exists());
|
||||
|
||||
// Answered floor: a verdict line.
|
||||
std::fs::create_dir(root.join("relower-1-second")).unwrap();
|
||||
for digest in &digests {
|
||||
std::fs::write(root.join(format!("relower-1-second/{digest}.answer")), said).unwrap();
|
||||
}
|
||||
// Fewer than three floor arms is refused: one pair measures no noise.
|
||||
let o = relower(&["--arm", "tuned", "--floor", "second"]);
|
||||
assert_eq!(o.status.code(), Some(1), "{}", out(&o));
|
||||
assert!(out(&o).contains("three or more floor arms"), "{}", out(&o));
|
||||
|
||||
// Answered floors: a verdict line.
|
||||
for label in ["second", "third", "fourth"] {
|
||||
std::fs::create_dir_all(root.join(format!("relower-1-{label}"))).unwrap();
|
||||
for digest in &digests {
|
||||
std::fs::write(root.join(format!("relower-1-{label}/{digest}.answer")), said).unwrap();
|
||||
}
|
||||
}
|
||||
let o = relower(&["--arm", "tuned", "--floor", "second", "--floor", "third", "--floor", "fourth"]);
|
||||
assert!(o.status.success(), "{}", out(&o));
|
||||
let text = out(&o);
|
||||
assert!(text.contains("| floor chrf=0.00") && text.trim_end().ends_with("verdict Rejected(NoGain)"), "{text}");
|
||||
|
|
@ -1852,6 +1859,7 @@ fn cli_loop_relower_exits_3_until_answered_and_refuses_bad_labels() {
|
|||
vec!["--arm", &long],
|
||||
vec!["--arm", "../x"],
|
||||
vec!["--arm", "tuned", "--floor", "Bad"],
|
||||
vec!["--arm", "tuned", "--floor", "b", "--floor", "b", "--floor", "c"],
|
||||
] {
|
||||
let o = relower(&args);
|
||||
assert_eq!(o.status.code(), Some(1), "{args:?}: {}", out(&o));
|
||||
|
|
|
|||
|
|
@ -5335,14 +5335,14 @@ fn the_cycles_candidate_round_stops_on_a_withheld_lowering_prompt() {
|
|||
|
||||
use weksa::run::{RelowerOutcome, RelowerReading};
|
||||
|
||||
/// `relower` of round 1 with the Bag scorer; `floor` is an optional second arm.
|
||||
/// `relower` of round 1 with the Bag scorer; `floors` are the floor arms (none, or three or more).
|
||||
fn relower(
|
||||
dao: &Dao,
|
||||
recorded: &mut dyn ModelPort,
|
||||
arm: &mut dyn ModelPort,
|
||||
floor: Option<&mut dyn ModelPort>,
|
||||
floors: &mut [&mut dyn ModelPort],
|
||||
) -> Result<RelowerOutcome, RunError> {
|
||||
dao.run.relower(1, "arm", recorded, arm, floor, &Scorer::new(Bag))
|
||||
dao.run.relower(1, "arm", recorded, arm, floors, &Scorer::new(Bag))
|
||||
}
|
||||
|
||||
fn relowered(outcome: Result<RelowerOutcome, RunError>) -> (Vec<RelowerReading>, Option<weksa::run::Verdict>) {
|
||||
|
|
@ -5360,7 +5360,7 @@ fn relower_asks_only_validation_lowering_prompts_of_the_round() {
|
|||
// which digest is which item's.
|
||||
let (mut every, mut rec, mut arm) = (Lowerer::new(reader), Lowerer::new(reader), Lowerer::new(reader));
|
||||
read(ablate(&dao, &mut every, &mut Lowerer::new(reader)).unwrap());
|
||||
relowered(relower(&dao, &mut rec, &mut arm, None));
|
||||
relowered(relower(&dao, &mut rec, &mut arm, &mut []));
|
||||
let of = |keys: &[&str]| -> Vec<String> {
|
||||
every
|
||||
.seen
|
||||
|
|
@ -5400,7 +5400,7 @@ fn relower_reads_recorded_and_arm_against_the_item() {
|
|||
&dao,
|
||||
&mut Lowerer::new(reader),
|
||||
&mut Lowerer::new(blind),
|
||||
None,
|
||||
&mut [],
|
||||
));
|
||||
let l = &langs[0];
|
||||
assert_eq!((l.language.as_str(), l.n, l.unusable), ("en-US", 3, 0));
|
||||
|
|
@ -5408,7 +5408,7 @@ fn relower_reads_recorded_and_arm_against_the_item() {
|
|||
assert!(l.arm.composite > l.recorded.composite);
|
||||
assert!(l.floor.is_none() && verdict.is_none(), "no floor arm, no verdict");
|
||||
// An arm equal to the recorded answers reads equal.
|
||||
let (langs, _) = relowered(relower(&dao, &mut Lowerer::new(reader), &mut Lowerer::new(reader), None));
|
||||
let (langs, _) = relowered(relower(&dao, &mut Lowerer::new(reader), &mut Lowerer::new(reader), &mut []));
|
||||
assert_eq!(langs[0].recorded, langs[0].arm);
|
||||
}
|
||||
|
||||
|
|
@ -5432,11 +5432,11 @@ impl ModelPort for Breaks {
|
|||
fn relower_counts_an_unusable_arm_answer_as_zero() {
|
||||
let (dao, _) = recorded_dao(|_| {});
|
||||
let keys = ["item:daodejing:2", "item:daodejing:3", WRITTEN];
|
||||
let (whole, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), None));
|
||||
let (whole, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), &mut []));
|
||||
let whole = whole[0].arm;
|
||||
let (mut chrf, mut meaning) = (0.0, 0.0);
|
||||
for key in keys {
|
||||
let (langs, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Breaks(key), None));
|
||||
let (langs, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Breaks(key), &mut []));
|
||||
let l = &langs[0];
|
||||
assert_eq!((l.n, l.unusable), (3, 1), "{}", l.line());
|
||||
assert!(l.arm.chrf < whole.chrf && l.arm.meaning < whole.meaning, "{}", l.line());
|
||||
|
|
@ -5452,8 +5452,8 @@ fn relower_counts_an_unusable_arm_answer_as_zero() {
|
|||
#[test]
|
||||
fn relower_scores_an_unusable_recorded_answer_as_zero_without_counting_it_against_the_arm() {
|
||||
let (dao, _) = recorded_dao(|_| {});
|
||||
let (whole, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), None));
|
||||
let (langs, _) = relowered(relower(&dao, &mut Breaks("item:daodejing:3"), &mut Lowerer::new(blind), None));
|
||||
let (whole, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), &mut []));
|
||||
let (langs, _) = relowered(relower(&dao, &mut Breaks("item:daodejing:3"), &mut Lowerer::new(blind), &mut []));
|
||||
let l = &langs[0];
|
||||
assert_eq!(l.unusable, 0, "{}", l.line());
|
||||
assert_eq!(l.arm, whole[0].arm);
|
||||
|
|
@ -5465,16 +5465,16 @@ fn relower_decides_by_decide_panel_against_the_floor_arm() {
|
|||
use weksa::run::decide_panel;
|
||||
let (dao, _) = recorded_dao(|_| {});
|
||||
let run = |recorded: fn(&str, &str) -> String, arm: fn(&str, &str) -> String| {
|
||||
let mut floor = Lowerer::new(recorded);
|
||||
let (mut f1, mut f2, mut f3) = (Lowerer::new(recorded), Lowerer::new(recorded), Lowerer::new(recorded));
|
||||
let (langs, verdict) = relowered(relower(
|
||||
&dao,
|
||||
&mut Lowerer::new(recorded),
|
||||
&mut Lowerer::new(arm),
|
||||
Some(&mut floor),
|
||||
&mut [&mut f1, &mut f2, &mut f3],
|
||||
));
|
||||
let l = &langs[0];
|
||||
let floor_reading = l.floor.expect("a floor arm gives a floor");
|
||||
// A floor arm that equals the recorded answers has no spread.
|
||||
// Floor arms that equal the recorded answers have no spread.
|
||||
assert_eq!((floor_reading.chrf, floor_reading.meaning, floor_reading.composite), (0.0, 0.0, 0.0));
|
||||
assert_eq!(verdict, Some(decide_panel(&[(l.recorded, l.arm, floor_reading)])));
|
||||
verdict.unwrap()
|
||||
|
|
@ -5485,29 +5485,34 @@ fn relower_decides_by_decide_panel_against_the_floor_arm() {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn relower_floor_is_the_spread_of_recorded_and_the_floor_arm() {
|
||||
fn relower_floor_is_the_paired_noise_of_recorded_against_the_floor_arms() {
|
||||
let (dao, _) = recorded_dao(|_| {});
|
||||
let (langs, _) = relowered(relower(
|
||||
&dao,
|
||||
&mut Lowerer::new(blind),
|
||||
&mut Lowerer::new(blind),
|
||||
Some(&mut Breaks("item:daodejing:3")),
|
||||
&mut [&mut Lowerer::new(blind), &mut Lowerer::new(blind), &mut Breaks("item:daodejing:3")],
|
||||
));
|
||||
let l = &langs[0];
|
||||
let f = l.floor.unwrap();
|
||||
// The floor arm read as an arm against the same recorded answers.
|
||||
let (as_arm, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Breaks("item:daodejing:3"), None));
|
||||
let a = as_arm[0].arm;
|
||||
assert!(f.chrf > 0.0 && f.meaning > 0.0 && f.composite > 0.0);
|
||||
assert!((f.chrf - (l.recorded.chrf - a.chrf).abs()).abs() < 1e-9, "{}", l.line());
|
||||
assert!((f.composite - (l.recorded.composite - a.composite).abs()).abs() < 1e-9);
|
||||
assert!(f.chrf > 0.0 && f.meaning > 0.0 && f.composite > 0.0, "{}", l.line());
|
||||
assert!(l.line().contains(&format!("| floor chrf={:.2}", f.chrf)), "{}", l.line());
|
||||
assert_eq!(l.unusable, 0, "unusable counts the arm, not the floor");
|
||||
assert_eq!(l.floor_unusable, 1, "the floor arm's refusal is counted apart");
|
||||
assert_eq!(l.floor_unusable, 1, "a floor arm's refusal is counted apart");
|
||||
assert!(l.line().contains(" unusable=0 recorded "), "{}", l.line());
|
||||
assert!(l.line().ends_with(" unusable=1"), "the floor's refusal is on the line: {}", l.line());
|
||||
// Without a floor arm there is none to count.
|
||||
assert_eq!(as_arm[0].floor_unusable, 0);
|
||||
// A floor needs three arms: fewer measure no noise, and nothing is asked of the arm.
|
||||
for arms in [1usize, 2] {
|
||||
let mut fs: Vec<Lowerer> = (0..arms).map(|_| Lowerer::new(blind)).collect();
|
||||
let mut dyns: Vec<&mut dyn ModelPort> = fs.iter_mut().map(|f| f as &mut dyn ModelPort).collect();
|
||||
let mut arm = Lowerer::new(blind);
|
||||
let r = relower(&dao, &mut Lowerer::new(blind), &mut arm, &mut dyns);
|
||||
assert!(matches!(r, Err(RunError::FloorArms)));
|
||||
assert!(arm.seen.is_empty());
|
||||
}
|
||||
// Without floor arms there is none to count.
|
||||
let (none, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), &mut []));
|
||||
assert_eq!(none[0].floor_unusable, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -5517,7 +5522,7 @@ fn relower_skips_and_counts_an_item_the_round_refused_before_admission() {
|
|||
answers.insert("item:daodejing:3".into(), "not a draft".into());
|
||||
});
|
||||
let (mut rec, mut arm) = (Lowerer::new(blind), Lowerer::new(blind));
|
||||
let (langs, _) = relowered(relower(&dao, &mut rec, &mut arm, None));
|
||||
let (langs, _) = relowered(relower(&dao, &mut rec, &mut arm, &mut []));
|
||||
let l = &langs[0];
|
||||
assert_eq!((l.n, l.skipped, l.unusable), (2, 1, 0), "{}", l.line());
|
||||
assert!(l.line().starts_with("relower en-US n=2 skipped=1 unusable=0 "), "{}", l.line());
|
||||
|
|
@ -5536,7 +5541,7 @@ fn relower_skips_and_counts_an_item_whose_lowering_prompt_the_round_refused() {
|
|||
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
|
||||
assert_eq!(r.language("ja-JP").unwrap().validation_refused.lower, 1);
|
||||
let (mut rec, mut arm) = (Lowerer::new(bi_blind), Lowerer::new(bi_blind));
|
||||
let (langs, _) = relowered(relower(&dao, &mut rec, &mut arm, None));
|
||||
let (langs, _) = relowered(relower(&dao, &mut rec, &mut arm, &mut []));
|
||||
let ja = langs.iter().find(|l| l.language == "ja-JP").unwrap();
|
||||
assert_eq!((ja.n, ja.skipped), (1, 1), "{}", ja.line());
|
||||
assert_eq!(ja.recorded, ja.arm);
|
||||
|
|
@ -5579,7 +5584,7 @@ fn relower_decides_over_the_whole_panel_not_its_first_language() {
|
|||
&dao,
|
||||
&mut Lowerer::new(bi_reader),
|
||||
&mut Lowerer::new(bi_ja_lost),
|
||||
Some(&mut Lowerer::new(bi_reader)),
|
||||
&mut [&mut Lowerer::new(bi_reader), &mut Lowerer::new(bi_reader), &mut Lowerer::new(bi_reader)],
|
||||
));
|
||||
assert_eq!(
|
||||
langs.iter().map(|l| (l.language.as_str(), l.n)).collect::<Vec<_>>(),
|
||||
|
|
@ -5603,16 +5608,16 @@ fn relower_refuses_a_cycle_round_and_a_missing_recorded_answer() {
|
|||
run_cycle(&mut dao, &mut cycle_fake(ONE_CONCEPT));
|
||||
let candidate = dao.run.rounds().unwrap().into_iter().find(|r| r.cycle().is_some()).unwrap().n();
|
||||
let mut arm = Lowerer::new(reader);
|
||||
let r = dao.run.relower(candidate, "arm", &mut Lowerer::new(reader), &mut arm, None, &Scorer::new(Bag));
|
||||
let r = dao.run.relower(candidate, "arm", &mut Lowerer::new(reader), &mut arm, &mut [], &Scorer::new(Bag));
|
||||
assert!(matches!(r, Err(RunError::Malformed("round"))));
|
||||
let r = dao.run.relower(99, "arm", &mut Lowerer::new(reader), &mut arm, None, &Scorer::new(Bag));
|
||||
let r = dao.run.relower(99, "arm", &mut Lowerer::new(reader), &mut arm, &mut [], &Scorer::new(Bag));
|
||||
assert!(matches!(r, Err(RunError::Malformed("round"))));
|
||||
assert!(arm.seen.is_empty());
|
||||
|
||||
let (dao, _) = recorded_dao(|_| {});
|
||||
let mut empty = RecordedAnswers::new(dao.root.join("round-1/nowhere"), "fake-model".into());
|
||||
let mut arm = Lowerer::new(reader);
|
||||
let r = relower(&dao, &mut empty, &mut arm, None);
|
||||
let r = relower(&dao, &mut empty, &mut arm, &mut []);
|
||||
assert!(matches!(r, Err(RunError::Unrecorded)));
|
||||
assert!(arm.seen.is_empty(), "nothing is asked of the arm before the round's answer exists");
|
||||
}
|
||||
|
|
@ -5622,7 +5627,12 @@ fn relower_floor_arm_with_an_unanswered_prompt_is_refused() {
|
|||
use weksa::model::RecordedAnswers;
|
||||
let (dao, _) = recorded_dao(|_| {});
|
||||
let mut floor = RecordedAnswers::new(dao.root.join("relower-1-floor"), "fake-model".into());
|
||||
let r = relower(&dao, &mut Lowerer::new(reader), &mut Lowerer::new(reader), Some(&mut floor));
|
||||
let r = relower(
|
||||
&dao,
|
||||
&mut Lowerer::new(reader),
|
||||
&mut Lowerer::new(reader),
|
||||
&mut [&mut floor, &mut Lowerer::new(reader), &mut Lowerer::new(reader)],
|
||||
);
|
||||
assert!(matches!(r, Err(RunError::FloorUnanswered)));
|
||||
assert!(!dao.root.join("relower-1-floor").exists(), "a floor arm is never written to");
|
||||
}
|
||||
|
|
@ -5640,9 +5650,9 @@ fn relower_checks_the_embedder_not_the_model() {
|
|||
}
|
||||
let (dao, _) = recorded_dao(|_| {});
|
||||
let mut other = Named(Lowerer::new(blind));
|
||||
relowered(relower(&dao, &mut Lowerer::new(reader), &mut other, None));
|
||||
relowered(relower(&dao, &mut Lowerer::new(reader), &mut other, &mut []));
|
||||
let mut arm = Lowerer::new(reader);
|
||||
let r = dao.run.relower(1, "arm", &mut Lowerer::new(reader), &mut arm, None, &Scorer::new(OtherEmbedder));
|
||||
let r = dao.run.relower(1, "arm", &mut Lowerer::new(reader), &mut arm, &mut [], &Scorer::new(OtherEmbedder));
|
||||
assert!(matches!(r, Err(RunError::Drift(DriftOf::Embedder))));
|
||||
assert!(arm.seen.is_empty());
|
||||
}
|
||||
|
|
@ -5676,7 +5686,7 @@ fn relower_records_nothing() {
|
|||
// An unanswered arm writes its prompts in its own directory only.
|
||||
let mut files = dao.run.answer_files(dao.run.relower_dir(1, "arm")).unwrap();
|
||||
// The recorded answers are files here, as the relay leaves them.
|
||||
let r = relower(&dao, &mut recorded, &mut files, None);
|
||||
let r = relower(&dao, &mut recorded, &mut files, &mut []);
|
||||
assert!(matches!(r, Err(RunError::Unrecorded)), "no round answer files yet");
|
||||
let after = tree(&dao.root);
|
||||
for (path, bytes) in &before {
|
||||
|
|
@ -5699,7 +5709,7 @@ fn relower_records_nothing() {
|
|||
let before = tree(&dao.root);
|
||||
let mut recorded = RecordedAnswers::new(dao.root.join("round-1"), "fake-model".into());
|
||||
let mut files = dao.run.answer_files(dao.run.relower_dir(1, "arm")).unwrap();
|
||||
let RelowerOutcome::Unanswered { prompts } = relower(&dao, &mut recorded, &mut files, None).unwrap() else {
|
||||
let RelowerOutcome::Unanswered { prompts } = relower(&dao, &mut recorded, &mut files, &mut []).unwrap() else {
|
||||
panic!("the arm is unanswered")
|
||||
};
|
||||
assert_eq!(prompts.len(), 3);
|
||||
|
|
@ -5727,7 +5737,7 @@ fn relower_arm_prompt_holding_test_text_is_withheld() {
|
|||
|_| {},
|
||||
);
|
||||
let mut files = dao.run.answer_files(dao.run.relower_dir(1, "arm")).unwrap();
|
||||
let r = relower(&dao, &mut Lowerer::new(reader), &mut files, None);
|
||||
let r = relower(&dao, &mut Lowerer::new(reader), &mut files, &mut []);
|
||||
assert!(matches!(r, Err(RunError::Withheld)), "{:?}", r.map(|_| ()));
|
||||
let dir = dao.run.relower_dir(1, "arm");
|
||||
let prompts = std::fs::read_dir(&dir)
|
||||
|
|
|
|||
|
|
@ -359,7 +359,7 @@ the training set is the whole corpus re-encoded under the frozen revision.
|
|||
|
||||
A lowerer is compared on round n's Validation packets with the prompts
|
||||
byte-identical to the round's, by `weksa loop relower ROOT --round N --arm ARM
|
||||
--lowerer LABEL [--floor ARM2]`. Every arm is answered through the one prompt
|
||||
--lowerer LABEL [--floor ARM2 --floor ARM3 --floor ARM4]`. Every arm is answered through the one prompt
|
||||
door (`RunStores::answer_files`, the Test-text guard); a tuned model is a script
|
||||
that reads `<digest>.prompt` and writes `<digest>.answer` in
|
||||
`<ROOT>/relower-<N>-<ARM>`, not a new adapter. The recorded reading is the
|
||||
|
|
@ -369,15 +369,29 @@ ask: it is skipped, so the readings exclude it, and it is counted as `skipped=`
|
|||
on the line; `n` is the items read, so `recorded` can differ from the round's
|
||||
own Validation reading, which scores those items 0. An arm answer that is not a
|
||||
usable lowering scores 0 and counts in `unusable=`; the same for the floor arm,
|
||||
counted after `floor ... unusable=`. The floor is the spread of the recorded
|
||||
lowering and a second prompted arm (lowering-only noise, no encoder noise). The
|
||||
verdict is `decide_panel` over (recorded, arm, floor): Accepted is the rule's
|
||||
name for a gain past the floor, Rejected(Regressed) for a loss past it,
|
||||
Rejected(NoGain) for neither. None is yet a finding about the arm. The rule is calibration-pending: a one-pair floor is noisy
|
||||
(a simulated prompted repeat reads NoGain only about 7-15% of the time, Regressed
|
||||
44-60%, Accepted 33-41%), so NoGain is not yet evidence of non-inferiority and
|
||||
is not yet the bar for cut 13; the calibration arms (a prompted repeat, a
|
||||
control) decide what the floor must be before the verdict is read as a ruling.
|
||||
counted (over all floor arms) after `floor ... unusable=`. The floor needs three
|
||||
or more floor arms, each a prompted repeat of the recorded lowering on the same
|
||||
prompts: it is `FLOOR_Z` (3) standard errors of the mean per-item difference
|
||||
between the recorded lowering and a repeat, per metric, with the variances
|
||||
pooled over the arms (`paired_floor`; lowering-only noise, no encoder noise). The
|
||||
verdict is `decide_panel` over (recorded, arm, floor): Accepted is a composite
|
||||
gain past the noise of the panel's sum (the root of the languages' composite
|
||||
floors squared, at least one step), Rejected(Regressed) a loss past a language's
|
||||
floor, Rejected(NoGain) neither. The cycle's own floor is still the spread of two
|
||||
round replicates (`floor_of`); it has no per-item scores to pair.
|
||||
Calibration (`runs/distil-base`, 8 prompted draws, each its own subagent, 14
|
||||
en-US and 14 ja-JP Validation items; every base, candidate and 3-arm floor
|
||||
combination, 1120 trials from 28 unordered pairs; intervals bootstrap over
|
||||
draws, each degraded arm one fixed draw): a prompted repeat reads NoGain 92%
|
||||
(80 to 100), Regressed 8% (0 to 20), Accepted 0%, which with 28 pairs bounds
|
||||
false-accept only to about 10%; an all-"Yes." arm reads Regressed 100%; a mild
|
||||
degradation (one detail blurred per utterance) 100%; a subtler one 75% (50 to
|
||||
100). At 14 items a degradation under about three standard errors (a composite
|
||||
near 0.06) cannot be told from noise, and no arm known to be better exists, so
|
||||
the rule's power to accept a real gain is unmeasured. `FLOOR_Z` was chosen on
|
||||
this data (2.5 gives NoGain 84% and 93% on the subtler arm), so the rates are
|
||||
in-sample. Without the noise term in the gain test a prompted repeat reads
|
||||
Accepted in 40% of the trials.
|
||||
Training pairs are Train only, at a frozen revision; the export is 12b.
|
||||
|
||||
## Lowerer first cut (`golf-loop-lowerer-cut`)
|
||||
|
|
|
|||
Loading…
Reference in a new issue