calibrate-harness: weksa loop calibrate over answered draws (#78)
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
ba42639c3b
commit
d8afc4ab85
6 changed files with 999 additions and 118 deletions
|
|
@ -15,7 +15,7 @@ use weksa::packet::{FeatValue, FeatureValue, Grade};
|
|||
use weksa::packet::proof_packet;
|
||||
use weksa::model::{ModelPort, RecordedAnswers};
|
||||
use weksa::run::{
|
||||
AblationOutcome, CycleOutcome, LowererReach, OntologyMintOutcome, RecallOutcome, RelowerOutcome, RoundOutcome, RunPaths, RunStores, cycle, round,
|
||||
AblationOutcome, CalibrateOutcome, CycleOutcome, FLOOR_ARMS_MIN, LowererReach, OntologyMintOutcome, RecallOutcome, RelowerOutcome, RoundOutcome, RunPaths, RunStores, cycle, round,
|
||||
};
|
||||
use weksa::score::ScoreInput;
|
||||
use weksa::{
|
||||
|
|
@ -174,6 +174,31 @@ enum LoopCommand {
|
|||
#[arg(long, default_value = "http://127.0.0.1:11434")]
|
||||
ollama_url: String,
|
||||
},
|
||||
/// Replay the accept rule over answered relower draws of round ROUND, to
|
||||
/// measure how often it accepts, rejects or regresses what it should. Reads
|
||||
/// the recorded lowering and each draw (--prompted: repeats of it, at least
|
||||
/// four; --degraded: arms that should regress; --candidate: arms to rate),
|
||||
/// answered through `weksa loop relower ROOT --round N --arm LABEL`, and
|
||||
/// prints counts over all Validation items and over each --class
|
||||
/// NAME=PREFIX[,PREFIX...] (items whose key starts with a prefix). Writes
|
||||
/// no prompts; exits 1 naming the draws still unanswered. Procedure:
|
||||
/// docs/calibration.md. A diagnostic, never a gate; records nothing; Test
|
||||
/// and Train are never touched. Labels match [a-z0-9-]{1,32} and differ.
|
||||
Calibrate {
|
||||
root: PathBuf,
|
||||
#[arg(long)]
|
||||
round: u32,
|
||||
#[arg(long)]
|
||||
prompted: Vec<String>,
|
||||
#[arg(long)]
|
||||
degraded: Vec<String>,
|
||||
#[arg(long)]
|
||||
candidate: Vec<String>,
|
||||
#[arg(long)]
|
||||
class: Vec<String>,
|
||||
#[arg(long, default_value = "http://127.0.0.1:11434")]
|
||||
ollama_url: String,
|
||||
},
|
||||
/// Run one edit-accept cycle of the run at ROOT. The stores are read only
|
||||
/// from the run's record. The editor's prompt, then the candidate round's
|
||||
/// encode prompts, then its lowering prompts are written in turn; the
|
||||
|
|
@ -660,6 +685,12 @@ impl AddArgs {
|
|||
}
|
||||
}
|
||||
|
||||
/// A draw or arm label: [a-z0-9-]{1,32}.
|
||||
fn is_label(l: &str) -> bool {
|
||||
(1..=32).contains(&l.len())
|
||||
&& l.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-')
|
||||
}
|
||||
|
||||
/// The split of an item as `Train`, `Validation`, `Test` or `excluded`.
|
||||
fn split_text(item: &CorpusItem) -> String {
|
||||
SplitScheme::V1
|
||||
|
|
@ -1146,12 +1177,8 @@ fn main() -> Result<()> {
|
|||
floor,
|
||||
ollama_url,
|
||||
}) => {
|
||||
let label = |l: &str| {
|
||||
(1..=32).contains(&l.len())
|
||||
&& l.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-')
|
||||
};
|
||||
if !label(&arm)
|
||||
|| floor.iter().any(|f| !label(f) || *f == arm)
|
||||
if !is_label(&arm)
|
||||
|| floor.iter().any(|f| !is_label(f) || *f == arm)
|
||||
|| floor.iter().enumerate().any(|(i, f)| floor[..i].contains(f))
|
||||
{
|
||||
anyhow::bail!("arm and floor labels must match [a-z0-9-]{{1,32}} and differ");
|
||||
|
|
@ -1197,6 +1224,95 @@ fn main() -> Result<()> {
|
|||
}
|
||||
}
|
||||
}
|
||||
Command::Loop(LoopCommand::Calibrate {
|
||||
root,
|
||||
round,
|
||||
prompted,
|
||||
degraded,
|
||||
candidate,
|
||||
class,
|
||||
ollama_url,
|
||||
}) => {
|
||||
let labels: Vec<&String> = prompted.iter().chain(°raded).chain(&candidate).collect();
|
||||
if labels.iter().any(|l| !is_label(l))
|
||||
|| labels.iter().enumerate().any(|(i, l)| labels[..i].contains(l))
|
||||
{
|
||||
anyhow::bail!("draw labels must match [a-z0-9-]{{1,32}} and differ");
|
||||
}
|
||||
if prompted.len() < FLOOR_ARMS_MIN + 1 {
|
||||
anyhow::bail!("at least {} --prompted labels are needed", FLOOR_ARMS_MIN + 1);
|
||||
}
|
||||
let mut classes: Vec<(String, Vec<String>)> = Vec::new();
|
||||
for c in &class {
|
||||
let Some((name, prefixes)) = c.split_once('=') else {
|
||||
anyhow::bail!("--class is NAME=PREFIX[,PREFIX...]");
|
||||
};
|
||||
if !is_label(name)
|
||||
|| name == "all"
|
||||
|| classes.iter().any(|(n, _)| n == name)
|
||||
|| prefixes.split(',').any(str::is_empty)
|
||||
{
|
||||
anyhow::bail!("--class is NAME=PREFIX[,PREFIX...] with a new label NAME other than all");
|
||||
}
|
||||
classes.push((name.to_string(), prefixes.split(',').map(String::from).collect()));
|
||||
}
|
||||
let run = RunStores::open(&root)?;
|
||||
let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder));
|
||||
match run.calibrate(round, &prompted, °raded, &candidate, &classes, &scorer)? {
|
||||
CalibrateOutcome::Unanswered(missing) => {
|
||||
for (label, n) in &missing {
|
||||
println!("unanswered {label} {n}");
|
||||
}
|
||||
eprintln!(
|
||||
"answer it through weksa loop relower ROOT --round N --arm LABEL first"
|
||||
);
|
||||
std::process::exit(1);
|
||||
}
|
||||
CalibrateOutcome::Read { unusable, classes } => {
|
||||
println!(
|
||||
"{}",
|
||||
run.labelled(&format!(
|
||||
"calibrate draws prompted={} degraded={} candidate={} floor-arms={FLOOR_ARMS_MIN} unusable={unusable}",
|
||||
prompted.len(),
|
||||
degraded.len(),
|
||||
candidate.len(),
|
||||
))
|
||||
);
|
||||
for c in classes {
|
||||
let items: String = c
|
||||
.languages
|
||||
.iter()
|
||||
.map(|l| format!(" {} n={} skipped={}", l.language, l.n, l.skipped))
|
||||
.collect();
|
||||
let t = &c.trials;
|
||||
println!("{}", run.labelled(&format!("calibrate class={} items{items}", c.name)));
|
||||
println!(
|
||||
"{}",
|
||||
run.labelled(&format!(
|
||||
"calibrate class={} prompted trials={} nogain={} accepted={} regressed={} noise={:.4}",
|
||||
c.name, t.prompted.trials, t.prompted.nogain, t.prompted.accepted, t.prompted.regressed, t.noise
|
||||
))
|
||||
);
|
||||
for (label, d) in degraded.iter().zip(&t.degraded) {
|
||||
println!(
|
||||
"{}",
|
||||
run.labelled(&format!(
|
||||
"calibrate class={} degraded={label} trials={} regressed={}",
|
||||
c.name, d.trials, d.regressed
|
||||
))
|
||||
);
|
||||
}
|
||||
println!(
|
||||
"{}",
|
||||
run.labelled(&format!(
|
||||
"calibrate class={} candidate trials={} accepted={} nogain={} regressed={}",
|
||||
c.name, t.candidate.trials, t.candidate.accepted, t.candidate.nogain, t.candidate.regressed
|
||||
))
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Command::Loop(LoopCommand::Recall { root }) => {
|
||||
let run = RunStores::open(&root)?;
|
||||
let mut files = run.answer_files(run.recall_dir())?;
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ use crate::encode::{
|
|||
EditDraft, LoweringDraft, MAX_EDIT_CHANGES, OntologyDraft, PacketDraft, encoder_id,
|
||||
};
|
||||
use crate::form::{is_attestation, is_language_tag, is_slug};
|
||||
use crate::model::{AnswerFiles, ModelPort, PortError};
|
||||
use crate::model::{AnswerFiles, ModelPort, PortError, RecordedAnswers};
|
||||
use crate::packet::{Packet, Stance};
|
||||
use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate, validation_items};
|
||||
use crate::split::{Split, SplitScheme, fnv1a64, fnv1a64_of, hex16};
|
||||
|
|
@ -1675,27 +1675,90 @@ impl RunStores {
|
|||
floors: &mut [&mut dyn ModelPort],
|
||||
scorer: &Scorer<E>,
|
||||
) -> Result<RelowerOutcome, RunError> {
|
||||
let store = |_| RunError::Store;
|
||||
check_embedder(self, scorer)?;
|
||||
let held = self.held_splits(n, &[Split::Validation])?;
|
||||
let inventory = self.inventory.inventory().map_err(store)?;
|
||||
let ontology = self.ontology.ontology().map_err(store)?;
|
||||
|
||||
let dir = self.relower_dir(n, arm_label);
|
||||
std::fs::create_dir_all(&dir).map_err(|e| RunError::Io(e.kind()))?;
|
||||
for suffix in ["cc", "cc.lock"] {
|
||||
let _ = std::fs::remove_file(dir.join(format!("base.{suffix}")));
|
||||
}
|
||||
let mut scratch = PacketStore::open(&dir.join("base.cc")).map_err(store)?;
|
||||
|
||||
if (1..FLOOR_ARMS_MIN).contains(&floors.len()) {
|
||||
return Err(RunError::FloorArms);
|
||||
}
|
||||
// Per language, per item: the item text, then the recorded and arm
|
||||
// lowerings and one per floor arm (`None` when unusable).
|
||||
type Row = (String, Option<String>, Option<String>, Vec<Option<String>>);
|
||||
let mut waiting: Vec<String> = Vec::new();
|
||||
let mut floor_waiting = false;
|
||||
let dir = self.relower_dir(n, arm_label);
|
||||
// Ports in the order the prompts are asked: recorded, floors, arm.
|
||||
let floor_count = floors.len();
|
||||
let mut ports: Vec<&mut dyn ModelPort> = vec![recorded];
|
||||
for f in floors.iter_mut() {
|
||||
let f: &mut dyn ModelPort = &mut **f;
|
||||
ports.push(f);
|
||||
}
|
||||
ports.push(arm);
|
||||
let drawn = self.read_draws(n, &dir.join("base.cc"), &mut ports, scorer)?;
|
||||
let arm_at = floor_count + 1;
|
||||
if drawn.waiting[1..arm_at].iter().any(|w| !w.is_empty()) {
|
||||
return Err(RunError::FloorUnanswered);
|
||||
}
|
||||
if !drawn.waiting[arm_at].is_empty() {
|
||||
return Ok(RelowerOutcome::Unanswered { prompts: drawn.waiting[arm_at].clone() });
|
||||
}
|
||||
let mut languages = Vec::new();
|
||||
for l in &drawn.languages {
|
||||
let of_port = |p: usize| -> Vec<ItemScore> { l.items.iter().map(|(_, s)| s[p].clone()).collect() };
|
||||
let rec = of_port(0);
|
||||
let floor_scores: Vec<Vec<ItemScore>> = (1..arm_at).map(of_port).collect();
|
||||
languages.push(RelowerReading {
|
||||
language: l.language.clone(),
|
||||
n: l.items.len() as u32,
|
||||
skipped: l.skipped,
|
||||
unusable: l.unusable[arm_at],
|
||||
floor_unusable: l.unusable[1..arm_at].iter().sum(),
|
||||
recorded: reading_of(&rec)?,
|
||||
arm: reading_of(&of_port(arm_at))?,
|
||||
floor: if floor_count == 0 { None } else { Some(paired_floor(&rec, &floor_scores)) },
|
||||
});
|
||||
}
|
||||
let verdict = if floor_count > 0 {
|
||||
let panel: Vec<(Reading, Reading, Reading)> = languages
|
||||
.iter()
|
||||
.filter_map(|l| Some((l.recorded, l.arm, l.floor?)))
|
||||
.collect();
|
||||
Some(decide_panel(&panel))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
Ok(RelowerOutcome::Read { languages, verdict })
|
||||
}
|
||||
|
||||
/// Asks every port, in the order given, each live Validation item's
|
||||
/// byte-identical lowering prompt of round `n` and scores the answers against
|
||||
/// the items in one [`Scorer::score`] call: the one owner of per-item
|
||||
/// scoring for [`relower`](Self::relower) and
|
||||
/// [`calibrate`](Self::calibrate). An answer that is not a usable lowering
|
||||
/// scores 0. Port 0 is the recorded lowering, which must be answered
|
||||
/// (`Unrecorded` at once); any other port that is not is listed in
|
||||
/// `waiting`, and nothing is scored. `scratch` is the packet store the
|
||||
/// prompts are built in, removed and reopened here; its directory is
|
||||
/// created here, after the round is found.
|
||||
fn read_draws<E: Embedder>(
|
||||
&self,
|
||||
n: u32,
|
||||
scratch: &Path,
|
||||
ports: &mut [&mut dyn ModelPort],
|
||||
scorer: &Scorer<E>,
|
||||
) -> Result<Drawn, RunError> {
|
||||
let store = |_| RunError::Store;
|
||||
let held = self.held_splits(n, &[Split::Validation])?;
|
||||
let inventory = self.inventory.inventory().map_err(store)?;
|
||||
let ontology = self.ontology.ontology().map_err(store)?;
|
||||
if let Some(dir) = scratch.parent() {
|
||||
std::fs::create_dir_all(dir).map_err(|e| RunError::Io(e.kind()))?;
|
||||
}
|
||||
for suffix in ["", ".lock"] {
|
||||
let mut name = scratch.as_os_str().to_owned();
|
||||
name.push(suffix);
|
||||
let _ = std::fs::remove_file(PathBuf::from(name));
|
||||
}
|
||||
let mut scratch = PacketStore::open(scratch).map_err(store)?;
|
||||
|
||||
// Per language, per item: the item id and text, then each port's lowering
|
||||
// (`None` when unusable or unanswered).
|
||||
let mut waiting: Vec<Vec<String>> = vec![Vec::new(); ports.len()];
|
||||
type Row = (String, String, Vec<Option<String>>);
|
||||
let mut rows: Vec<Vec<Row>> = Vec::new();
|
||||
for (_, _, items) in &held {
|
||||
let mut language_rows = Vec::new();
|
||||
|
|
@ -1707,49 +1770,33 @@ impl RunStores {
|
|||
else {
|
||||
continue;
|
||||
};
|
||||
let text = |a: Asked| match a {
|
||||
Asked::Text(t) => Some(t),
|
||||
_ => None,
|
||||
};
|
||||
let recorded_text = match asked(recorded, &prompt)? {
|
||||
Asked::Waiting => return Err(RunError::Unrecorded),
|
||||
other => text(other),
|
||||
};
|
||||
let mut floor_texts = Vec::new();
|
||||
for f in floors.iter_mut() {
|
||||
floor_texts.push(match asked(&mut **f, &prompt)? {
|
||||
let mut texts = Vec::new();
|
||||
for (p, port) in ports.iter_mut().enumerate() {
|
||||
texts.push(match asked(&mut **port, &prompt)? {
|
||||
Asked::Text(t) => Some(t),
|
||||
Asked::Unusable => None,
|
||||
Asked::Waiting if p == 0 => return Err(RunError::Unrecorded),
|
||||
Asked::Waiting => {
|
||||
floor_waiting = true;
|
||||
let d = prompt.digest();
|
||||
if !waiting[p].contains(&d) {
|
||||
waiting[p].push(d);
|
||||
}
|
||||
None
|
||||
}
|
||||
other => text(other),
|
||||
});
|
||||
}
|
||||
let arm_text = match asked(arm, &prompt)? {
|
||||
Asked::Waiting => {
|
||||
let d = prompt.digest();
|
||||
if !waiting.contains(&d) {
|
||||
waiting.push(d);
|
||||
}
|
||||
None
|
||||
}
|
||||
other => text(other),
|
||||
};
|
||||
language_rows.push((item.text.clone(), recorded_text, arm_text, floor_texts));
|
||||
language_rows.push((item.id.as_str().to_string(), item.text.clone(), texts));
|
||||
}
|
||||
rows.push(language_rows);
|
||||
}
|
||||
if floor_waiting {
|
||||
return Err(RunError::FloorUnanswered);
|
||||
}
|
||||
if !waiting.is_empty() {
|
||||
return Ok(RelowerOutcome::Unanswered { prompts: waiting });
|
||||
if waiting.iter().any(|w| !w.is_empty()) {
|
||||
return Ok(Drawn { languages: Vec::new(), waiting });
|
||||
}
|
||||
|
||||
// One score call: each usable lowering against its item, in row order.
|
||||
let mut inputs: Vec<ScoreInput> = Vec::new();
|
||||
for (reference, recorded, arm, floor_texts) in rows.iter().flatten() {
|
||||
for output in [recorded, arm].into_iter().chain(floor_texts).flatten() {
|
||||
for (_, reference, texts) in rows.iter().flatten() {
|
||||
for output in texts.iter().flatten() {
|
||||
inputs.push(ScoreInput { reference, output });
|
||||
}
|
||||
}
|
||||
|
|
@ -1759,67 +1806,272 @@ impl RunStores {
|
|||
scorer.score(&inputs).map_err(|_| RunError::Embed)?
|
||||
}
|
||||
.into_iter();
|
||||
let mut next = |t: &Option<String>| -> Result<ItemScore, RunError> {
|
||||
match t {
|
||||
Some(_) => scores.next().ok_or(RunError::Store),
|
||||
None => Ok(ItemScore { chrf: 0.0, meaning: Some(0.0) }),
|
||||
}
|
||||
};
|
||||
let reading_of = |scored: &[ItemScore]| {
|
||||
Reading::of(&SplitMean {
|
||||
count: scored.len(),
|
||||
chrf: mean(scored.iter().map(|s| s.chrf)),
|
||||
meaning: mean(scored.iter().filter_map(|s| s.meaning)),
|
||||
})
|
||||
.ok_or(RunError::NoValidation)
|
||||
};
|
||||
let mut languages = Vec::new();
|
||||
let live = self.corpus.all().map_err(store)?;
|
||||
for ((language, _, _), language_rows) in held.iter().zip(&rows) {
|
||||
let (mut rec, mut arm_scores) = (Vec::new(), Vec::new());
|
||||
let mut floor_scores: Vec<Vec<ItemScore>> = vec![Vec::new(); floors.len()];
|
||||
let (mut unusable, mut floor_unusable) = (0u32, 0u32);
|
||||
for (_, r, a, f) in language_rows {
|
||||
rec.push(next(r)?);
|
||||
arm_scores.push(next(a)?);
|
||||
if a.is_none() {
|
||||
unusable += 1;
|
||||
}
|
||||
for (scores, t) in floor_scores.iter_mut().zip(f) {
|
||||
scores.push(next(t)?);
|
||||
if t.is_none() {
|
||||
floor_unusable += 1;
|
||||
}
|
||||
let mut languages = Vec::new();
|
||||
for ((language, _, _), language_rows) in held.iter().zip(rows) {
|
||||
let mut unusable = vec![0u32; ports.len()];
|
||||
let mut items = Vec::new();
|
||||
for (id, _, texts) in language_rows {
|
||||
let mut item_scores = Vec::new();
|
||||
for (p, t) in texts.iter().enumerate() {
|
||||
item_scores.push(match t {
|
||||
Some(_) => scores.next().ok_or(RunError::Store)?,
|
||||
None => {
|
||||
unusable[p] += 1;
|
||||
ItemScore { chrf: 0.0, meaning: Some(0.0) }
|
||||
}
|
||||
});
|
||||
}
|
||||
items.push((id, item_scores));
|
||||
}
|
||||
let recorded_reading = reading_of(&rec)?;
|
||||
languages.push(RelowerReading {
|
||||
let live_validation = live
|
||||
.iter()
|
||||
.filter(|i| &i.language == language && SplitScheme::V1.live(i) == Some(Split::Validation))
|
||||
.count() as u32;
|
||||
languages.push(DrawnLanguage {
|
||||
language: language.clone(),
|
||||
n: language_rows.len() as u32,
|
||||
skipped: live
|
||||
.iter()
|
||||
.filter(|i| {
|
||||
&i.language == language && SplitScheme::V1.live(i) == Some(Split::Validation)
|
||||
})
|
||||
.count() as u32
|
||||
- language_rows.len() as u32,
|
||||
skipped: live_validation - items.len() as u32,
|
||||
items,
|
||||
unusable,
|
||||
floor_unusable,
|
||||
recorded: recorded_reading,
|
||||
arm: reading_of(&arm_scores)?,
|
||||
floor: if floors.is_empty() { None } else { Some(paired_floor(&rec, &floor_scores)) },
|
||||
});
|
||||
}
|
||||
let verdict = if !floors.is_empty() {
|
||||
let panel: Vec<(Reading, Reading, Reading)> = languages
|
||||
.iter()
|
||||
.filter_map(|l| Some((l.recorded, l.arm, l.floor?)))
|
||||
.collect();
|
||||
Some(decide_panel(&panel))
|
||||
} else {
|
||||
None
|
||||
Ok(Drawn { languages, waiting })
|
||||
}
|
||||
}
|
||||
|
||||
/// One panel language of [`RunStores::read_draws`].
|
||||
struct DrawnLanguage {
|
||||
language: String,
|
||||
/// Live Validation items of the language with no lowering prompt.
|
||||
skipped: u32,
|
||||
/// Per read item in key order: its id and one score per port.
|
||||
items: Vec<(String, Vec<ItemScore>)>,
|
||||
/// Per port, answers that were not a usable lowering.
|
||||
unusable: Vec<u32>,
|
||||
}
|
||||
|
||||
struct Drawn {
|
||||
/// Per panel language in panel order; empty while any port is waiting.
|
||||
languages: Vec<DrawnLanguage>,
|
||||
/// Per port, the digests of the prompts it holds no answer to.
|
||||
waiting: Vec<Vec<String>>,
|
||||
}
|
||||
|
||||
/// A reading over per-item scores; `NoValidation` without items.
|
||||
fn reading_of(scored: &[ItemScore]) -> Result<Reading, RunError> {
|
||||
Reading::of(&SplitMean {
|
||||
count: scored.len(),
|
||||
chrf: mean(scored.iter().map(|s| s.chrf)),
|
||||
meaning: mean(scored.iter().filter_map(|s| s.meaning)),
|
||||
})
|
||||
.ok_or(RunError::NoValidation)
|
||||
}
|
||||
|
||||
/// How trials of one kind read: counts, not rates.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub struct Counts {
|
||||
pub trials: u32,
|
||||
pub accepted: u32,
|
||||
pub nogain: u32,
|
||||
pub regressed: u32,
|
||||
}
|
||||
|
||||
impl Counts {
|
||||
fn tally(&mut self, v: Verdict) {
|
||||
self.trials += 1;
|
||||
match v {
|
||||
Verdict::Accepted => self.accepted += 1,
|
||||
Verdict::Rejected(Why::Regressed(_)) => self.regressed += 1,
|
||||
Verdict::Rejected(_) => self.nogain += 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What [`calibration_trials`] counted over one set of items.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Trials {
|
||||
/// Ordered (base, candidate) pairs of prompted draws, each over every floor.
|
||||
pub prompted: Counts,
|
||||
/// Per degraded arm in the order given: the arm as the candidate.
|
||||
pub degraded: Vec<Counts>,
|
||||
/// Every candidate arm as the candidate, pooled.
|
||||
pub candidate: Counts,
|
||||
/// The mean of [`panel_noise`] over every (base, floor) choice.
|
||||
pub noise: f64,
|
||||
}
|
||||
|
||||
/// Every `k`-subset of `pool`, in order.
|
||||
fn subsets(pool: &[usize], k: usize) -> Vec<Vec<usize>> {
|
||||
if k == 0 {
|
||||
return vec![Vec::new()];
|
||||
}
|
||||
let mut out = Vec::new();
|
||||
for (i, &first) in pool.iter().enumerate() {
|
||||
for mut rest in subsets(&pool[i + 1..], k - 1) {
|
||||
rest.insert(0, first);
|
||||
out.push(rest);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The accept rule replayed over draws that are all lowerings of one round, as
|
||||
/// the live loop would meet them: [`decide_panel`] over (base, candidate,
|
||||
/// [`paired_floor`] of the base against `FLOOR_ARMS_MIN` other prompted draws),
|
||||
/// one row per language. `draws` is `[port][language][item]`; ports
|
||||
/// `0..prompted` are prompted draws (the recorded lowering first), then
|
||||
/// `degraded` and `candidate` ports follow. A language with no items reads
|
||||
/// NaN, which fails closed.
|
||||
pub fn calibration_trials(
|
||||
draws: &[Vec<Vec<ItemScore>>],
|
||||
prompted: usize,
|
||||
degraded: usize,
|
||||
candidate: usize,
|
||||
) -> Trials {
|
||||
let languages = draws.first().map_or(0, Vec::len);
|
||||
let rows = |b: usize, c: usize, floors: &[usize]| -> Vec<(Reading, Reading, Reading)> {
|
||||
let read = |scores: &[ItemScore]| {
|
||||
reading_of(scores).unwrap_or(Reading { chrf: f64::NAN, meaning: f64::NAN, composite: f64::NAN })
|
||||
};
|
||||
Ok(RelowerOutcome::Read { languages, verdict })
|
||||
(0..languages)
|
||||
.map(|l| {
|
||||
let arms: Vec<Vec<ItemScore>> = floors.iter().map(|&f| draws[f][l].clone()).collect();
|
||||
(read(&draws[b][l]), read(&draws[c][l]), paired_floor(&draws[b][l], &arms))
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
let pool: Vec<usize> = (0..prompted).collect();
|
||||
let without = |out: &[usize]| -> Vec<usize> { pool.iter().copied().filter(|p| !out.contains(p)).collect() };
|
||||
let (mut prompted_counts, mut candidate_counts) = (Counts::default(), Counts::default());
|
||||
let mut degraded_counts = vec![Counts::default(); degraded];
|
||||
let (mut noise, mut noises) = (0.0, 0u32);
|
||||
for &b in &pool {
|
||||
for floors in subsets(&without(&[b]), FLOOR_ARMS_MIN) {
|
||||
for (d, counts) in degraded_counts.iter_mut().enumerate() {
|
||||
counts.tally(decide_panel(&rows(b, prompted + d, &floors)));
|
||||
}
|
||||
for c in prompted + degraded..prompted + degraded + candidate {
|
||||
candidate_counts.tally(decide_panel(&rows(b, c, &floors)));
|
||||
}
|
||||
noise += panel_noise(&rows(b, b, &floors));
|
||||
noises += 1;
|
||||
}
|
||||
for &c in pool.iter().filter(|&&c| c != b) {
|
||||
for floors in subsets(&without(&[b, c]), FLOOR_ARMS_MIN) {
|
||||
prompted_counts.tally(decide_panel(&rows(b, c, &floors)));
|
||||
}
|
||||
}
|
||||
}
|
||||
Trials {
|
||||
prompted: prompted_counts,
|
||||
degraded: degraded_counts,
|
||||
candidate: candidate_counts,
|
||||
noise: noise / f64::from(noises),
|
||||
}
|
||||
}
|
||||
|
||||
/// One panel language of a class: items read and live Validation items of the
|
||||
/// class not read.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ClassLanguage {
|
||||
pub language: String,
|
||||
pub n: u32,
|
||||
pub skipped: u32,
|
||||
}
|
||||
|
||||
/// The trials over one class of items.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct ClassTrials {
|
||||
pub name: String,
|
||||
pub languages: Vec<ClassLanguage>,
|
||||
pub trials: Trials,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum CalibrateOutcome {
|
||||
/// Draw labels with the number of prompts each has no answer to.
|
||||
Unanswered(Vec<(String, u32)>),
|
||||
/// `unusable` over every prompted, degraded and candidate draw; `all` first,
|
||||
/// then the classes in the order given.
|
||||
Read { unusable: u32, classes: Vec<ClassTrials> },
|
||||
}
|
||||
|
||||
impl RunStores {
|
||||
/// Replays the accept rule over answered relower draws: reads every draw of
|
||||
/// round `n` (the recorded lowering and the `relower-<n>-<label>`
|
||||
/// directories, through [`RecordedAnswers`] only, so nothing is written
|
||||
/// there) with [`read_draws`](Self::read_draws), and counts
|
||||
/// [`calibration_trials`] over all items and over each class, a class being
|
||||
/// the items whose key starts with any of its prefixes. See
|
||||
/// `docs/calibration.md`. Records nothing; the scratch packet store is
|
||||
/// `<root>/calibrate-<n>/base.cc`.
|
||||
pub fn calibrate<E: Embedder>(
|
||||
&self,
|
||||
n: u32,
|
||||
prompted: &[String],
|
||||
degraded: &[String],
|
||||
candidate: &[String],
|
||||
classes: &[(String, Vec<String>)],
|
||||
scorer: &Scorer<E>,
|
||||
) -> Result<CalibrateOutcome, RunError> {
|
||||
check_embedder(self, scorer)?;
|
||||
let labels: Vec<&String> = prompted.iter().chain(degraded).chain(candidate).collect();
|
||||
let mut recorded =
|
||||
RecordedAnswers::new(self.root.join(format!("round-{n}")), self.record.model.clone());
|
||||
let mut draws: Vec<RecordedAnswers> = labels
|
||||
.iter()
|
||||
.map(|l| RecordedAnswers::new(self.relower_dir(n, l), self.record.model.clone()))
|
||||
.collect();
|
||||
let mut ports: Vec<&mut dyn ModelPort> = vec![&mut recorded];
|
||||
for d in draws.iter_mut() {
|
||||
ports.push(d);
|
||||
}
|
||||
let dir = self.root.join(format!("calibrate-{n}"));
|
||||
let drawn = self.read_draws(n, &dir.join("base.cc"), &mut ports, scorer)?;
|
||||
let missing: Vec<(String, u32)> = labels
|
||||
.iter()
|
||||
.zip(&drawn.waiting[1..])
|
||||
.filter(|(_, w)| !w.is_empty())
|
||||
.map(|(l, w)| ((*l).clone(), w.len() as u32))
|
||||
.collect();
|
||||
if !missing.is_empty() {
|
||||
return Ok(CalibrateOutcome::Unanswered(missing));
|
||||
}
|
||||
let live = self.corpus.all().map_err(|_| RunError::Store)?;
|
||||
let everything = [("all".to_string(), vec![String::new()])];
|
||||
let mut out = Vec::new();
|
||||
for (name, prefixes) in everything.iter().chain(classes) {
|
||||
let keep = |id: &str| prefixes.iter().any(|p| id.starts_with(p.as_str()));
|
||||
let mut by_port: Vec<Vec<Vec<ItemScore>>> = vec![Vec::new(); ports.len()];
|
||||
let mut languages = Vec::new();
|
||||
for l in &drawn.languages {
|
||||
let items: Vec<&Vec<ItemScore>> =
|
||||
l.items.iter().filter(|(id, _)| keep(id)).map(|(_, s)| s).collect();
|
||||
for (p, port) in by_port.iter_mut().enumerate() {
|
||||
port.push(items.iter().map(|s| s[p].clone()).collect());
|
||||
}
|
||||
let in_class = live
|
||||
.iter()
|
||||
.filter(|i| {
|
||||
i.language == l.language
|
||||
&& SplitScheme::V1.live(i) == Some(Split::Validation)
|
||||
&& keep(i.id.as_str())
|
||||
})
|
||||
.count() as u32;
|
||||
languages.push(ClassLanguage {
|
||||
language: l.language.clone(),
|
||||
n: items.len() as u32,
|
||||
skipped: in_class - items.len() as u32,
|
||||
});
|
||||
}
|
||||
out.push(ClassTrials {
|
||||
name: name.clone(),
|
||||
languages,
|
||||
trials: calibration_trials(&by_port, prompted.len() + 1, degraded.len(), candidate.len()),
|
||||
});
|
||||
}
|
||||
let unusable = drawn.languages.iter().map(|l| l.unusable[1..].iter().sum::<u32>()).sum();
|
||||
Ok(CalibrateOutcome::Read { unusable, classes: out })
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2258,8 +2510,11 @@ fn nano(x: f64) -> i64 {
|
|||
/// otherwise read as unchanged); a metric that falls by more than its
|
||||
/// tolerance (the floor, at least one step) regresses that metric; the
|
||||
/// composite falling by more than max(its floor, one [`LADDER_STEP`])
|
||||
/// regresses the composite. Means count a refused Validation item as 0, so
|
||||
/// refusing a weak item cannot raise a reading. Differences are compared in
|
||||
/// regresses the composite. Regression compares the readings it is given
|
||||
/// and counts nothing itself: a round's reading counts a refused item as 0
|
||||
/// ([`round`], `score_round`), so refusing a weak item cannot
|
||||
/// raise it, while a relower or calibrate reading is over the items that have a
|
||||
/// lowering prompt only and counts the rest in `skipped`, not as 0. Differences are compared in
|
||||
/// billionths ([`nano`]), so a fall of exactly the tolerance is a tie.
|
||||
fn regression(base: &Reading, candidate: &Reading, floor: &Reading) -> Option<Metric> {
|
||||
let finite = |r: &Reading| r.chrf.is_finite() && r.meaning.is_finite() && r.composite.is_finite();
|
||||
|
|
@ -2327,16 +2582,21 @@ pub fn decide_panel(rows: &[(Reading, Reading, Reading)]) -> Verdict {
|
|||
.iter()
|
||||
.map(|(base, candidate, _)| nano(candidate.composite - base.composite))
|
||||
.sum();
|
||||
// The noise of a sum of independent languages' differences: the root of the
|
||||
// sum of their floors squared.
|
||||
let noise = rows.iter().map(|(_, _, floor)| floor.composite.powi(2)).sum::<f64>().sqrt();
|
||||
if net > nano(noise.max(LADDER_STEP)) {
|
||||
if net > nano(panel_noise(rows).max(LADDER_STEP)) {
|
||||
Verdict::Accepted
|
||||
} else {
|
||||
Verdict::Rejected(Why::NoGain)
|
||||
}
|
||||
}
|
||||
|
||||
/// The noise of a sum of independent languages' differences: the root of the
|
||||
/// sum of their composite floors squared, over one `(base, candidate, floor)`
|
||||
/// row per language. A one-language panel's noise is its floor. The one owner
|
||||
/// of the term, for [`decide_panel`] and the calibration.
|
||||
pub fn panel_noise(rows: &[(Reading, Reading, Reading)]) -> f64 {
|
||||
rows.iter().map(|(_, _, floor)| floor.composite.powi(2)).sum::<f64>().sqrt()
|
||||
}
|
||||
|
||||
/// One panel language's row of a cycle: the head's Validation reading, the
|
||||
/// candidate's (none when the edit never reached a round), the language's
|
||||
/// measured floor and the threshold derived from it. The threshold decides only
|
||||
|
|
@ -2570,7 +2830,7 @@ pub const FLOOR_ARMS_MIN: usize = 3;
|
|||
/// than two items or no arm (0/0), and when an arm is ragged.
|
||||
pub fn paired_floor(recorded: &[ItemScore], arms: &[Vec<ItemScore>]) -> Reading {
|
||||
let n = recorded.len();
|
||||
if arms.iter().any(|a| a.len() != n) {
|
||||
if n < 2 || arms.iter().any(|a| a.len() != n) {
|
||||
return Reading { chrf: f64::NAN, meaning: f64::NAN, composite: f64::NAN };
|
||||
}
|
||||
let pooled = |of: &dyn Fn(&ItemScore) -> f64| {
|
||||
|
|
@ -2780,6 +3040,52 @@ fn discard_inventory(dir: &Path) {
|
|||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn scored(chrf: f64, meaning: f64) -> ItemScore {
|
||||
ItemScore { chrf, meaning: Some(meaning) }
|
||||
}
|
||||
|
||||
/// A draw over `[language][item]`: `base` plus a per-draw, per-item wobble.
|
||||
fn draw(languages: usize, items: usize, base: f64, wobble: f64, seed: usize) -> Vec<Vec<ItemScore>> {
|
||||
(0..languages)
|
||||
.map(|l| {
|
||||
(0..items)
|
||||
.map(|i| {
|
||||
let w = wobble * (((i * 7 + seed * 3 + l) % 5) as f64 - 2.0) / 2.0;
|
||||
scored((100.0 * (base + w)).clamp(0.0, 100.0), (base + w).clamp(0.0, 1.0))
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn calibration_trials_count_every_choice_and_read_the_fixed_cases() {
|
||||
// Six prompted draws, two degraded, three candidates, two languages of 8 items.
|
||||
let mut draws: Vec<_> = (0..6).map(|s| draw(2, 8, 0.6, 0.02, s)).collect();
|
||||
draws.extend((0..2).map(|_| draw(2, 8, 0.0, 0.0, 0)));
|
||||
draws.extend((0..3).map(|s| draw(2, 8, 0.8, 0.02, s)));
|
||||
let t = calibration_trials(&draws, 6, 2, 3);
|
||||
assert_eq!(t.prompted.trials, 120);
|
||||
assert_eq!(t.degraded.iter().map(|d| d.trials).collect::<Vec<_>>(), [60, 60]);
|
||||
assert_eq!(t.candidate.trials, 6 * 10 * 3);
|
||||
assert!(t.noise > 0.0);
|
||||
// An all-zero draw regresses against any base, in every trial.
|
||||
assert!(t.degraded.iter().all(|d| d.regressed == d.trials));
|
||||
// A far better draw is accepted in every trial.
|
||||
assert_eq!(t.candidate.accepted, t.candidate.trials);
|
||||
|
||||
// Identical prompted draws: nothing to gain, nothing regressed, no noise.
|
||||
let same: Vec<_> = (0..6).map(|_| draw(2, 8, 0.6, 0.02, 1)).collect();
|
||||
let t = calibration_trials(&same, 6, 0, 0);
|
||||
assert_eq!((t.prompted.trials, t.prompted.nogain), (120, 120));
|
||||
assert_eq!(t.noise, 0.0);
|
||||
|
||||
// A class of one item has no floor: every trial fails closed.
|
||||
let one: Vec<_> = (0..6).map(|s| draw(2, 1, 0.6, 0.02, s)).collect();
|
||||
let t = calibration_trials(&one, 6, 0, 0);
|
||||
assert_eq!((t.prompted.trials, t.prompted.regressed), (120, 120));
|
||||
}
|
||||
|
||||
/// The twin of the `RoundRecord` literal in its compile_fail doctest: the
|
||||
/// same fields, compiling here because the fields are private to this module.
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -1866,6 +1866,99 @@ fn cli_loop_relower_exits_3_until_answered_and_refuses_bad_labels() {
|
|||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cli_loop_calibrate_reads_answered_draws_and_refuses_unanswered_and_bad_labels() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
open_dao_run(d.path());
|
||||
let url = fake_embedder();
|
||||
let root = d.path().join("canaryroot");
|
||||
let mut run = weksa::run::RunStores::open(&root).unwrap();
|
||||
let scorer = weksa::Scorer::new(weksa::OllamaEmbedder::new(&url, &run.record().embedder));
|
||||
let weksa::run::RoundOutcome::Recorded(r) = weksa::run::round(&mut run, &mut DaoModel, &scorer).unwrap()
|
||||
else {
|
||||
panic!("answered")
|
||||
};
|
||||
let said = r#"{"text":"The highest excellence is like the water."}"#;
|
||||
for call in &r.calls()[3..] {
|
||||
std::fs::write(root.join(format!("round-1/{}.answer", call.prompt)), said).unwrap();
|
||||
}
|
||||
let calibrate = |extra: &[&str]| {
|
||||
weksa(
|
||||
d.path(),
|
||||
&[&["loop", "calibrate", "canaryroot", "--round", "1", "--ollama-url", &url], extra].concat(),
|
||||
)
|
||||
};
|
||||
let four = ["--prompted", "p1", "--prompted", "p2", "--prompted", "p3", "--prompted", "p4"];
|
||||
|
||||
// Unanswered draws: exit 1, each label with its missing count, no relower directory created.
|
||||
let o = calibrate(&four);
|
||||
assert_eq!(o.status.code(), Some(1), "{}", out(&o));
|
||||
let text = out(&o);
|
||||
for label in ["p1", "p2", "p3", "p4"] {
|
||||
assert!(text.contains(&format!("unanswered {label} 2")), "{text}");
|
||||
assert!(!root.join(format!("relower-1-{label}")).exists());
|
||||
}
|
||||
assert!(text.contains("answer it through weksa loop relower"), "{text}");
|
||||
|
||||
// Answer every draw through relower's door; from here the only thing that can refuse is the argument.
|
||||
let o = weksa(
|
||||
d.path(),
|
||||
&["loop", "relower", "canaryroot", "--round", "1", "--lowerer", "x", "--arm", "p1", "--ollama-url", &url],
|
||||
);
|
||||
assert_eq!(o.status.code(), Some(3), "{}", out(&o));
|
||||
let digests: Vec<String> = out(&o)
|
||||
.lines()
|
||||
.filter_map(|l| l.strip_prefix("unanswered "))
|
||||
.filter(|x| x.len() == 16)
|
||||
.map(String::from)
|
||||
.collect();
|
||||
assert_eq!(digests.len(), 2);
|
||||
// `P1` and `a_b` are answered too, so only the label check can refuse them.
|
||||
for label in ["p1", "p2", "p3", "p4", "bad", "P1", "a_b"] {
|
||||
std::fs::create_dir_all(root.join(format!("relower-1-{label}"))).unwrap();
|
||||
for digest in &digests {
|
||||
std::fs::write(root.join(format!("relower-1-{label}/{digest}.answer")), said).unwrap();
|
||||
}
|
||||
}
|
||||
// A draw answered in part.
|
||||
std::fs::create_dir_all(root.join("relower-1-part")).unwrap();
|
||||
std::fs::write(root.join(format!("relower-1-part/{}.answer", digests[0])), said).unwrap();
|
||||
let o = calibrate(&["--prompted", "p1", "--prompted", "p2", "--prompted", "p3", "--prompted", "part"]);
|
||||
assert_eq!(o.status.code(), Some(1), "{}", out(&o));
|
||||
assert!(out(&o).contains("unanswered part 1") && !out(&o).contains("unanswered p1"), "{}", out(&o));
|
||||
|
||||
// Refusals, every draw answered: the baseline reads, each variant differs by one fault.
|
||||
assert!(calibrate(&four).status.success());
|
||||
for args in [
|
||||
vec!["--prompted", "p1", "--prompted", "p2", "--prompted", "p3"],
|
||||
vec!["--prompted", "p1", "--prompted", "p2", "--prompted", "p3", "--prompted", "p3"],
|
||||
vec!["--prompted", "p1", "--prompted", "p2", "--prompted", "p3", "--prompted", "p4", "--degraded", "p4"],
|
||||
vec!["--prompted", "p1", "--prompted", "p2", "--prompted", "p3", "--prompted", "p4", "--candidate", "bad", "--degraded", "bad"],
|
||||
vec!["--prompted", "P1", "--prompted", "p2", "--prompted", "p3", "--prompted", "p4"],
|
||||
vec!["--prompted", "a_b", "--prompted", "p2", "--prompted", "p3", "--prompted", "p4"],
|
||||
[&four[..], &["--class", "all=item:"]].concat(),
|
||||
[&four[..], &["--class", "noequals"]].concat(),
|
||||
[&four[..], &["--class", "Old=item:a"]].concat(),
|
||||
[&four[..], &["--class", "old=item:a,"]].concat(),
|
||||
[&four[..], &["--class", "old=item:a", "--class", "old=item:b"]].concat(),
|
||||
] {
|
||||
let o = calibrate(&args);
|
||||
assert_eq!(o.status.code(), Some(1), "{args:?}: {}", out(&o));
|
||||
assert!(!out(&o).contains("unanswered"), "{args:?} must be refused for itself: {}", out(&o));
|
||||
}
|
||||
|
||||
let o = calibrate(&[&four[..], &["--degraded", "bad", "--class", "old=item:daodejing"]].concat());
|
||||
assert!(o.status.success(), "{}", out(&o));
|
||||
let text = out(&o);
|
||||
assert!(text.starts_with("calibrate draws prompted=4 degraded=1 candidate=0 floor-arms=3 unusable=0"), "{text}");
|
||||
assert!(text.contains("calibrate class=all items en-US n=2 skipped=0"), "{text}");
|
||||
assert!(text.contains("calibrate class=all prompted trials=20 nogain=20 accepted=0 regressed=0"), "{text}");
|
||||
assert!(text.contains("calibrate class=old degraded=bad trials=20 regressed=0"), "{text}");
|
||||
for leaked in ["item:daodejing", "packet:", "highest excellence", "canaryroot"] {
|
||||
assert!(!text.contains(leaked), "{text}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn loop_ablate_without_the_rounds_answers_says_so() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
|
|
|
|||
|
|
@ -6174,3 +6174,257 @@ fn relower_arm_prompt_holding_test_text_is_withheld() {
|
|||
.unwrap_or(0);
|
||||
assert_eq!(prompts, 0, "no prompt file is written");
|
||||
}
|
||||
|
||||
// ---- calibrate: the accept rule replayed over answered relower draws ----
|
||||
|
||||
/// The cut a [`Tee`] answers with something that is no lowering.
|
||||
const NOT_A_LOWERING: usize = 99;
|
||||
|
||||
/// Answers each lowering prompt with the item's own text cut by `cut(key)`
|
||||
/// tenths, and leaves the answer where an answer file goes, as the relay does.
|
||||
struct Tee {
|
||||
dir: PathBuf,
|
||||
texts: HashMap<String, String>,
|
||||
cut: Box<dyn Fn(&str) -> usize>,
|
||||
}
|
||||
|
||||
impl ModelPort for Tee {
|
||||
fn model(&self) -> &str {
|
||||
"fake-model"
|
||||
}
|
||||
fn answer(&mut self, prompt: &Prompt) -> Result<String, PortError> {
|
||||
let key = prompt.item_keys()[0].clone();
|
||||
let text: Vec<char> = self.texts[&key].chars().collect();
|
||||
let cut = (self.cut)(&key);
|
||||
let keep = (text.len() * (10 - cut.min(10))).div_ceil(10);
|
||||
let answer = if cut == NOT_A_LOWERING {
|
||||
"this is no lowering".to_string()
|
||||
} else {
|
||||
serde_json::json!({ "text": text[..keep].iter().collect::<String>() }).to_string()
|
||||
};
|
||||
std::fs::create_dir_all(&self.dir).unwrap();
|
||||
std::fs::write(self.dir.join(format!("{}.answer", prompt.digest())), &answer).unwrap();
|
||||
Ok(answer)
|
||||
}
|
||||
}
|
||||
|
||||
/// A bilingual Dao run grown to seven live Validation items per language, five
|
||||
/// of them `item:vgrow:*`, with round 1 recorded.
|
||||
fn grown_bilingual() -> Dao {
|
||||
let mut dao = dao_panel(
|
||||
|corpus, entities| {
|
||||
add_languages(corpus, entities, LAOZI_JA);
|
||||
let entities = weksa::EntityStore::open(entities).unwrap();
|
||||
let mut store = CorpusStore::open(corpus).unwrap();
|
||||
for k in 0..5 {
|
||||
for (from, key, text, language) in [
|
||||
("item:daodejing:2", format!("item:vgrow:{}", k + 1), format!("The sage grows item {k} by letting the thing alone for {} seasons", k * 3 + 1), "en-US"),
|
||||
("item:daodejing:12", format!("item:vgrowja:{}", k + 1), format!("聖人は{}の季節を数えて物ごとに静かに任せる", "春夏秋冬".chars().cycle().skip(k).take(k + 2).collect::<String>()), "ja-JP"),
|
||||
] {
|
||||
let mut item = store.get(from).unwrap().unwrap();
|
||||
item.id = key.as_str().into();
|
||||
item.text = text;
|
||||
item.language = language.into();
|
||||
item.names = vec![];
|
||||
item.prior_turns = vec![];
|
||||
store.add(item, &entities).unwrap();
|
||||
}
|
||||
}
|
||||
},
|
||||
&["ja-JP", "en-US"],
|
||||
);
|
||||
let mut model = bilingual_fake();
|
||||
for k in 0..5 {
|
||||
for key in [format!("item:vgrow:{}", k + 1), format!("item:vgrowja:{}", k + 1)] {
|
||||
model.answers.insert(key.clone(), citing_laozi(2));
|
||||
model.lowered.insert(key, "placeholder".into());
|
||||
}
|
||||
}
|
||||
recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
|
||||
dao
|
||||
}
|
||||
|
||||
fn corpus_texts(dao: &Dao) -> HashMap<String, String> {
|
||||
CorpusStore::open(&dao.corpus)
|
||||
.unwrap()
|
||||
.all()
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|i| (i.id.as_str().to_string(), i.text))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A draw label's answers, written through relower's own door.
|
||||
fn answer_draw(dao: &Dao, label: &str, cut: impl Fn(&str) -> usize + 'static) {
|
||||
use weksa::model::RecordedAnswers;
|
||||
let mut recorded = RecordedAnswers::new(dao.root.join("round-1"), "fake-model".into());
|
||||
let mut tee = Tee { dir: dao.run.relower_dir(1, label), texts: corpus_texts(dao), cut: Box::new(cut) };
|
||||
dao.run.relower(1, label, &mut recorded, &mut tee, &mut [], &Scorer::new(Bag)).unwrap();
|
||||
}
|
||||
|
||||
/// A stable per-item wobble in 0..4 for draw `d`.
|
||||
fn wobble(d: usize, key: &str) -> usize {
|
||||
(key.bytes().map(usize::from).sum::<usize>() * (d + 3) + d) % 4
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn calibrate_reads_answered_draws_by_decide_panel() {
|
||||
use weksa::run::CalibrateOutcome;
|
||||
let dao = grown_bilingual();
|
||||
// The recorded lowering's answers, through a tee over the round directory.
|
||||
let mut round1 = Tee { dir: dao.root.join("round-1"), texts: corpus_texts(&dao), cut: Box::new(|k| wobble(0, k)) };
|
||||
let mut seed = Tee { dir: dao.run.relower_dir(1, "seed"), texts: corpus_texts(&dao), cut: Box::new(|_| 0) };
|
||||
dao.run.relower(1, "seed", &mut round1, &mut seed, &mut [], &Scorer::new(Bag)).unwrap();
|
||||
|
||||
let prompted: Vec<String> = (1..=5).map(|d| format!("p{d}")).collect();
|
||||
for (i, label) in prompted.iter().enumerate() {
|
||||
answer_draw(&dao, label, move |k| wobble(i + 1, k));
|
||||
}
|
||||
answer_draw(&dao, "bad", |_| 9);
|
||||
answer_draw(&dao, "mild", |k| 1 + wobble(7, k) % 2);
|
||||
for (i, label) in ["c1", "c2"].iter().enumerate() {
|
||||
answer_draw(&dao, label, move |k| wobble(i + 10, k) / 2);
|
||||
}
|
||||
let degraded = ["bad".to_string(), "mild".to_string()];
|
||||
let candidate = ["c1".to_string(), "c2".to_string()];
|
||||
let classes = [
|
||||
("old".to_string(), vec!["item:daodejing".to_string()]),
|
||||
("new".to_string(), vec!["item:vgrow".to_string()]),
|
||||
];
|
||||
let before = tree(&dao.root);
|
||||
let CalibrateOutcome::Read { unusable, classes } = dao
|
||||
.run
|
||||
.calibrate(1, &prompted, °raded, &candidate, &classes, &Scorer::new(Bag))
|
||||
.unwrap()
|
||||
else {
|
||||
panic!("every draw is answered")
|
||||
};
|
||||
let after = tree(&dao.root);
|
||||
assert!(
|
||||
after.keys().all(|p| before.contains_key(p) || p.starts_with("calibrate-1/")),
|
||||
"calibrate writes only its scratch store"
|
||||
);
|
||||
assert_eq!(unusable, 0);
|
||||
let got: Vec<String> = classes
|
||||
.iter()
|
||||
.map(|c| {
|
||||
let t = &c.trials;
|
||||
let items: Vec<(u32, u32)> = c.languages.iter().map(|l| (l.n, l.skipped)).collect();
|
||||
format!(
|
||||
"{} {items:?} prompted {}/{}/{}/{} noise {:.4} degraded {:?} candidate {}/{}/{}/{}",
|
||||
c.name,
|
||||
t.prompted.trials, t.prompted.nogain, t.prompted.accepted, t.prompted.regressed,
|
||||
t.noise,
|
||||
t.degraded.iter().map(|d| (d.trials, d.regressed)).collect::<Vec<_>>(),
|
||||
t.candidate.trials, t.candidate.accepted, t.candidate.nogain, t.candidate.regressed,
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(
|
||||
got,
|
||||
[
|
||||
"all [(7, 0), (7, 0)] prompted 120/30/24/66 noise 0.0468 degraded [(60, 60), (60, 43)] candidate 120/64/40/16",
|
||||
"old [(2, 0), (2, 0)] prompted 120/118/0/2 noise 0.1354 degraded [(60, 60), (60, 60)] candidate 120/0/120/0",
|
||||
"new [(5, 0), (5, 0)] prompted 120/16/24/80 noise 0.0138 degraded [(60, 60), (60, 50)] candidate 120/60/20/40",
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/// An item in each language whose packet, so whose prompt, no other item shares.
|
||||
const BAD_EN: &str = "item:daodejing:3";
|
||||
const BAD_JA: &str = "item:daodejing:13";
|
||||
|
||||
#[test]
|
||||
fn calibrate_classes_match_key_prefixes_exactly() {
|
||||
use weksa::run::CalibrateOutcome;
|
||||
let dao = grown_bilingual();
|
||||
let seed = |dao: &Dao| {
|
||||
let mut round1 = Tee { dir: dao.root.join("round-1"), texts: corpus_texts(dao), cut: Box::new(|_| 0) };
|
||||
let mut arm = Tee { dir: dao.run.relower_dir(1, "seed"), texts: corpus_texts(dao), cut: Box::new(|_| 0) };
|
||||
dao.run.relower(1, "seed", &mut round1, &mut arm, &mut [], &Scorer::new(Bag)).unwrap();
|
||||
};
|
||||
seed(&dao);
|
||||
let prompted: Vec<String> = (1..=4).map(|d| format!("p{d}")).collect();
|
||||
for label in &prompted {
|
||||
answer_draw(&dao, label, |_| 0);
|
||||
}
|
||||
let class = |name: &str, prefix: &str| (name.to_string(), vec![prefix.to_string()]);
|
||||
let classes = [
|
||||
class("en", "item:vgrow:"), // the en-US items only; a prefix one byte short also takes the ja-JP ones
|
||||
class("mid", "vgrow"), // a key does not start with it, though it contains it
|
||||
class("both", "item:vgrow"),
|
||||
("two".to_string(), vec!["item:vgrow:1".to_string(), "item:vgrowja:1".to_string()]),
|
||||
];
|
||||
let CalibrateOutcome::Read { classes, .. } =
|
||||
dao.run.calibrate(1, &prompted, &[], &[], &classes, &Scorer::new(Bag)).unwrap()
|
||||
else {
|
||||
panic!("answered")
|
||||
};
|
||||
let seen: Vec<(String, Vec<(u32, u32)>)> = classes
|
||||
.iter()
|
||||
.map(|c| (c.name.clone(), c.languages.iter().map(|l| (l.n, l.skipped)).collect()))
|
||||
.collect();
|
||||
// Panel order: en-US, ja-JP.
|
||||
assert_eq!(
|
||||
seen,
|
||||
[
|
||||
("all".to_string(), vec![(7, 0), (7, 0)]),
|
||||
("en".to_string(), vec![(5, 0), (0, 0)]),
|
||||
("mid".to_string(), vec![(0, 0), (0, 0)]),
|
||||
("both".to_string(), vec![(5, 0), (5, 0)]),
|
||||
("two".to_string(), vec![(1, 0), (1, 0)]),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unusable_answers_are_counted_for_draws_and_for_arm_and_floors_apart() {
|
||||
use weksa::model::RecordedAnswers;
|
||||
use weksa::run::{CalibrateOutcome, RelowerOutcome};
|
||||
let dao = grown_bilingual();
|
||||
let tee = |dir: PathBuf, bad: &'static [&'static str]| Tee {
|
||||
dir,
|
||||
texts: corpus_texts(&dao),
|
||||
cut: Box::new(move |k| if bad.contains(&k) { NOT_A_LOWERING } else { 0 }),
|
||||
};
|
||||
// The recorded lowering is unusable on one item; it is no draw and is not counted.
|
||||
let mut round1 = tee(dao.root.join("round-1"), &[BAD_EN]);
|
||||
let mut seed = tee(dao.run.relower_dir(1, "seed"), &[]);
|
||||
dao.run.relower(1, "seed", &mut round1, &mut seed, &mut [], &Scorer::new(Bag)).unwrap();
|
||||
|
||||
// relower: the arm is unusable on one item, the first floor on two; they are counted apart.
|
||||
let mut recorded = RecordedAnswers::new(dao.root.join("round-1"), "fake-model".into());
|
||||
let mut arm = tee(dao.run.relower_dir(1, "arm"), &[BAD_EN]);
|
||||
let mut f1 = tee(dao.run.relower_dir(1, "f1"), &[BAD_EN, BAD_JA]);
|
||||
let mut f2 = tee(dao.run.relower_dir(1, "f2"), &[]);
|
||||
let mut f3 = tee(dao.run.relower_dir(1, "f3"), &[]);
|
||||
let RelowerOutcome::Read { languages, .. } = dao
|
||||
.run
|
||||
.relower(1, "arm", &mut recorded, &mut arm, &mut [&mut f1, &mut f2, &mut f3], &Scorer::new(Bag))
|
||||
.unwrap()
|
||||
else {
|
||||
panic!("answered")
|
||||
};
|
||||
assert_eq!(languages.iter().map(|l| l.unusable).sum::<u32>(), 1);
|
||||
assert_eq!(languages.iter().map(|l| l.floor_unusable).sum::<u32>(), 2);
|
||||
|
||||
// calibrate: three unusable answers across the draws, none of them the recorded one.
|
||||
answer_draw_bad(&dao, "p1", &[BAD_EN, BAD_JA]);
|
||||
answer_draw_bad(&dao, "p2", &[]);
|
||||
answer_draw_bad(&dao, "p3", &[]);
|
||||
answer_draw_bad(&dao, "p4", &[]);
|
||||
answer_draw_bad(&dao, "bad", &[BAD_EN]);
|
||||
let prompted: Vec<String> = (1..=4).map(|d| format!("p{d}")).collect();
|
||||
let CalibrateOutcome::Read { unusable, .. } = dao
|
||||
.run
|
||||
.calibrate(1, &prompted, &["bad".to_string()], &[], &[], &Scorer::new(Bag))
|
||||
.unwrap()
|
||||
else {
|
||||
panic!("answered")
|
||||
};
|
||||
assert_eq!(unusable, 3);
|
||||
}
|
||||
|
||||
fn answer_draw_bad(dao: &Dao, label: &str, bad: &'static [&'static str]) {
|
||||
answer_draw(dao, label, move |k| if bad.contains(&k) { NOT_A_LOWERING } else { 0 });
|
||||
}
|
||||
|
|
|
|||
99
docs/calibration.md
Normal file
99
docs/calibration.md
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
# Calibrating the accept rule
|
||||
|
||||
`weksa loop calibrate` measures how the live accept rule behaves on a run's own
|
||||
Validation corpus: how often it accepts, rejects as no gain, or regresses a
|
||||
candidate that should be none of those. It does not reimplement the rule. It
|
||||
reads the answered draws through `RunStores::read_draws` (the scoring
|
||||
`weksa loop relower` uses), decides every trial with `decide_panel` over
|
||||
`paired_floor` (`crates/weksa/src/run.rs`), and prints, per class, the items
|
||||
read and skipped per language, the trial totals with their Accepted, NoGain and
|
||||
Regressed counts, and the mean panel noise. That is all it produces: the
|
||||
Accepted/NoGain/Regressed counts and the noise mean in the map's "Recalibration
|
||||
on the grown corpus" block are reproduced by it; the other figures there (the
|
||||
35% with R removed, the noise range, the 0.0185 gain and its 0.41 ratio, the
|
||||
bootstrap intervals, the per-item SDs, R's +0.024, the mild-arm duplicate
|
||||
counts and mean composite offsets, the Train-vocabulary overlap, the old/new
|
||||
class composites and the ten parse-refused item ids) were computed by scratch scripts that are not in the repo. It is a
|
||||
diagnostic: it records nothing, writes no prompts, and never touches Test or
|
||||
Train.
|
||||
|
||||
## What it counts
|
||||
|
||||
The draws are lowerings of one round's Validation packets: the round's own
|
||||
(recorded) lowering `R`, plus draws you label. `P` is `R` and the `--prompted`
|
||||
draws; the floor size is `FLOOR_ARMS_MIN` (3), the live rule's.
|
||||
|
||||
- Prompted trials: every ordered pair (base, candidate) in `P`, against every
|
||||
3-subset of the rest of `P` as the floor. A repeat of the same lowering
|
||||
should read NoGain; Accepted is a false accept, Regressed a false reject.
|
||||
(`R` comes from the round, not from `relower`, so it may not be
|
||||
exchangeable with a repeat; read the pairs that involve it with that in mind.)
|
||||
- Degraded trials, per `--degraded` arm: every base in `P` against every
|
||||
3-subset of the rest of `P`, the degraded draw as the candidate. These should
|
||||
read Regressed.
|
||||
- Candidate trials, pooled over the `--candidate` arms: the same, tallied
|
||||
Accepted, NoGain, Regressed. This is the rule's power against that arm.
|
||||
- Noise: the mean `panel_noise` over every (base, floor) choice.
|
||||
|
||||
With six prompted draws including `R` that is 120 prompted trials, 60 per
|
||||
degraded arm and 240 candidate trials. Trials share draws and are not
|
||||
independent. Each count is printed for `all` Validation items and for each
|
||||
`--class NAME=PREFIX[,PREFIX...]` (items whose key starts with any prefix), from
|
||||
the same per-item scores. A class with fewer than two read items in a language
|
||||
has no floor and reads Regressed in every trial; that is printed.
|
||||
|
||||
## Steps
|
||||
|
||||
1. `weksa loop open ROOT ...` and `weksa loop round ROOT` record round N and
|
||||
leave its answers in `ROOT/round-N/`.
|
||||
2. For each draw label: `weksa loop relower ROOT --round N --arm LABEL
|
||||
--lowerer NAME` writes the byte-identical lowering prompts to
|
||||
`ROOT/relower-N-LABEL/<digest>.prompt` and exits 3. Answer each prompt into
|
||||
`ROOT/relower-N-LABEL/<digest>.answer`.
|
||||
3. `weksa loop calibrate ROOT --round N --prompted p1 --prompted p2 ...
|
||||
[--degraded LABEL]... [--candidate LABEL]... [--class NAME=PREFIX]...`.
|
||||
At least four `--prompted` labels are needed (a pair and three floors with
|
||||
`R` make five draws). It exits 1 and lists `unanswered LABEL COUNT` for any
|
||||
draw with prompts still unanswered. Labels match `[a-z0-9-]{1,32}` and differ
|
||||
across the three kinds. It needs the Ollama embedder of the run; a full
|
||||
60-item, 13-draw calibration is on the order of 1500 embedded texts, so
|
||||
launch it detached with a log.
|
||||
|
||||
The scratch packet store is `ROOT/calibrate-N/base.cc`; delete it at will.
|
||||
|
||||
## Answer file format
|
||||
|
||||
`<digest>.answer` is one JSON object and nothing else, no code fence:
|
||||
`{"text": "..."}`, a JSON-escaped string. The answerer reads only the prompt
|
||||
(its first line says what to write) and writes the answer. Every prompt of the
|
||||
draw must be answered.
|
||||
|
||||
## Briefs per arm
|
||||
|
||||
Independence rule: a fresh agent per batch of answers; no batch sees another
|
||||
draw's answers. Only `mild` reads `round-N/*.answer`.
|
||||
|
||||
- Plain prompted: answer each prompt as its own first line asks; read nothing
|
||||
else in the run root.
|
||||
- Mild: for each prompt, read the recorded lowering in
|
||||
`ROOT/round-N/<digest>.answer` and write the same-format answer whose text is
|
||||
that utterance with exactly one meaning detail blurred: change or drop one of
|
||||
tense, polarity, modality or hedge, a referent, a quantity or a predicate's
|
||||
concept (replace it with a near but different word). Keep language, register,
|
||||
length and fluency; the result must read as a natural utterance and must not
|
||||
mention the change. Pick the detail that is not the most salient one. A
|
||||
second mild draw blurs a different detail than the first and is written
|
||||
without reading the first's answers. The wording of the distil-base original
|
||||
is lost, and the dispatch used for growth-base is not on disk, so this is the
|
||||
wording from now on. Warning: on growth-base two mild arms briefed this way
|
||||
came out identical in 27 of 60 items, so independently briefed mild arms
|
||||
are not independent degradations; read a mild draw's regress count with that
|
||||
in mind.
|
||||
- Train-guide: before answering, read the Train sentences of the corpus in the
|
||||
batch's language as a register guide and let it steer word choice; the guide
|
||||
gives phrasing only and each answer must still say exactly what its own packet
|
||||
says. The guide section is the live Train items of that language, built from
|
||||
`weksa corpus dump` filtered to Train, and is given to the answerer in the
|
||||
brief, never pasted into this document, and never includes Validation or Test
|
||||
text. Read nothing else in the run root.
|
||||
- Control: a script writes `{"text":"Yes."}` as the answer to every prompt.
|
||||
|
|
@ -505,11 +505,24 @@ findings shaped the wording.
|
|||
`validation-growth-writer-claude-checked`). The new scenes use a different
|
||||
`in_group` convention from the old ones (follow-up
|
||||
`vgrow-scene-in-group-convention`), so the new-item scores are for scenes that
|
||||
will be re-annotated; the corpus digest changes again then.
|
||||
will be re-annotated; the speaker-state store changes then (a run pins its
|
||||
digest), the corpus does not.
|
||||
|
||||
Evidence lives in the Self workspace (`cal2/`, `runs/growth-base`), not in git,
|
||||
and uses a DUMP patch kept out of the repo; no committed harness reruns these
|
||||
numbers (Soul finding `evidence-outside-git`, follow-up owed).
|
||||
`weksa loop calibrate` (`docs/calibration.md`) reproduces from `run.rs`'s own
|
||||
scoring and accept rule the counts per class, the trial totals and the mean
|
||||
noise of this block: prompted 116 NoGain, 4 Accepted, 0 Regressed of 120;
|
||||
control-yes 60 of 60 Regressed, mild-1 19 of 60, mild-2 0 of 60; the
|
||||
Train-guide arm 51 Accepted, 189 NoGain, 0 Regressed of 240; noise 0.0451;
|
||||
items en-US n=28 skipped=7, ja-JP n=32 skipped=3. By class it adds: old items
|
||||
(`item:original*`) prompted 111/9/0, noise 0.1069, Train-guide 71 Accepted of
|
||||
240; new items (`item:vgrow*`) prompted 116/4/0, noise 0.0334, Train-guide 0
|
||||
Accepted and 74 Regressed of 240. The other figures of this block (35% with R
|
||||
removed, the noise range, the 0.0185 gain and 0.41 ratio, the bootstrap
|
||||
intervals, the per-item SDs, R's +0.024, the mild-arm duplicate counts and
|
||||
mean composite offsets, the Train-vocabulary overlap, the old/new class
|
||||
composites, the ten parse-refused item ids) were
|
||||
computed by scratch scripts outside git (Soul finding `evidence-outside-git`)
|
||||
and are not reproduced by the harness.
|
||||
|
||||
### Smallest measurable distil slice (`distil-incontext`, not built)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue