diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index f70887a..3cd2235 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -15,7 +15,7 @@ use weksa::packet::{FeatValue, FeatureValue, Grade}; use weksa::packet::proof_packet; use weksa::model::{ModelPort, RecordedAnswers}; use weksa::run::{ - AblationOutcome, CycleOutcome, LowererReach, OntologyMintOutcome, RecallOutcome, RelowerOutcome, RoundOutcome, RunPaths, RunStores, cycle, round, + AblationOutcome, CalibrateOutcome, CycleOutcome, FLOOR_ARMS_MIN, LowererReach, OntologyMintOutcome, RecallOutcome, RelowerOutcome, RoundOutcome, RunPaths, RunStores, cycle, round, }; use weksa::score::ScoreInput; use weksa::{ @@ -174,6 +174,31 @@ enum LoopCommand { #[arg(long, default_value = "http://127.0.0.1:11434")] ollama_url: String, }, + /// Replay the accept rule over answered relower draws of round ROUND, to + /// measure how often it accepts, rejects or regresses what it should. Reads + /// the recorded lowering and each draw (--prompted: repeats of it, at least + /// four; --degraded: arms that should regress; --candidate: arms to rate), + /// answered through `weksa loop relower ROOT --round N --arm LABEL`, and + /// prints counts over all Validation items and over each --class + /// NAME=PREFIX[,PREFIX...] (items whose key starts with a prefix). Writes + /// no prompts; exits 1 naming the draws still unanswered. Procedure: + /// docs/calibration.md. A diagnostic, never a gate; records nothing; Test + /// and Train are never touched. Labels match [a-z0-9-]{1,32} and differ. + Calibrate { + root: PathBuf, + #[arg(long)] + round: u32, + #[arg(long)] + prompted: Vec, + #[arg(long)] + degraded: Vec, + #[arg(long)] + candidate: Vec, + #[arg(long)] + class: Vec, + #[arg(long, default_value = "http://127.0.0.1:11434")] + ollama_url: String, + }, /// Run one edit-accept cycle of the run at ROOT. The stores are read only /// from the run's record. The editor's prompt, then the candidate round's /// encode prompts, then its lowering prompts are written in turn; the @@ -660,6 +685,12 @@ impl AddArgs { } } +/// A draw or arm label: [a-z0-9-]{1,32}. +fn is_label(l: &str) -> bool { + (1..=32).contains(&l.len()) + && l.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') +} + /// The split of an item as `Train`, `Validation`, `Test` or `excluded`. fn split_text(item: &CorpusItem) -> String { SplitScheme::V1 @@ -1146,12 +1177,8 @@ fn main() -> Result<()> { floor, ollama_url, }) => { - let label = |l: &str| { - (1..=32).contains(&l.len()) - && l.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') - }; - if !label(&arm) - || floor.iter().any(|f| !label(f) || *f == arm) + if !is_label(&arm) + || floor.iter().any(|f| !is_label(f) || *f == arm) || floor.iter().enumerate().any(|(i, f)| floor[..i].contains(f)) { anyhow::bail!("arm and floor labels must match [a-z0-9-]{{1,32}} and differ"); @@ -1197,6 +1224,95 @@ fn main() -> Result<()> { } } } + Command::Loop(LoopCommand::Calibrate { + root, + round, + prompted, + degraded, + candidate, + class, + ollama_url, + }) => { + let labels: Vec<&String> = prompted.iter().chain(°raded).chain(&candidate).collect(); + if labels.iter().any(|l| !is_label(l)) + || labels.iter().enumerate().any(|(i, l)| labels[..i].contains(l)) + { + anyhow::bail!("draw labels must match [a-z0-9-]{{1,32}} and differ"); + } + if prompted.len() < FLOOR_ARMS_MIN + 1 { + anyhow::bail!("at least {} --prompted labels are needed", FLOOR_ARMS_MIN + 1); + } + let mut classes: Vec<(String, Vec)> = Vec::new(); + for c in &class { + let Some((name, prefixes)) = c.split_once('=') else { + anyhow::bail!("--class is NAME=PREFIX[,PREFIX...]"); + }; + if !is_label(name) + || name == "all" + || classes.iter().any(|(n, _)| n == name) + || prefixes.split(',').any(str::is_empty) + { + anyhow::bail!("--class is NAME=PREFIX[,PREFIX...] with a new label NAME other than all"); + } + classes.push((name.to_string(), prefixes.split(',').map(String::from).collect())); + } + let run = RunStores::open(&root)?; + let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder)); + match run.calibrate(round, &prompted, °raded, &candidate, &classes, &scorer)? { + CalibrateOutcome::Unanswered(missing) => { + for (label, n) in &missing { + println!("unanswered {label} {n}"); + } + eprintln!( + "answer it through weksa loop relower ROOT --round N --arm LABEL first" + ); + std::process::exit(1); + } + CalibrateOutcome::Read { unusable, classes } => { + println!( + "{}", + run.labelled(&format!( + "calibrate draws prompted={} degraded={} candidate={} floor-arms={FLOOR_ARMS_MIN} unusable={unusable}", + prompted.len(), + degraded.len(), + candidate.len(), + )) + ); + for c in classes { + let items: String = c + .languages + .iter() + .map(|l| format!(" {} n={} skipped={}", l.language, l.n, l.skipped)) + .collect(); + let t = &c.trials; + println!("{}", run.labelled(&format!("calibrate class={} items{items}", c.name))); + println!( + "{}", + run.labelled(&format!( + "calibrate class={} prompted trials={} nogain={} accepted={} regressed={} noise={:.4}", + c.name, t.prompted.trials, t.prompted.nogain, t.prompted.accepted, t.prompted.regressed, t.noise + )) + ); + for (label, d) in degraded.iter().zip(&t.degraded) { + println!( + "{}", + run.labelled(&format!( + "calibrate class={} degraded={label} trials={} regressed={}", + c.name, d.trials, d.regressed + )) + ); + } + println!( + "{}", + run.labelled(&format!( + "calibrate class={} candidate trials={} accepted={} nogain={} regressed={}", + c.name, t.candidate.trials, t.candidate.accepted, t.candidate.nogain, t.candidate.regressed + )) + ); + } + } + } + } Command::Loop(LoopCommand::Recall { root }) => { let run = RunStores::open(&root)?; let mut files = run.answer_files(run.recall_dir())?; diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 463db61..b15fd07 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -44,7 +44,7 @@ use crate::encode::{ EditDraft, LoweringDraft, MAX_EDIT_CHANGES, OntologyDraft, PacketDraft, encoder_id, }; use crate::form::{is_attestation, is_language_tag, is_slug}; -use crate::model::{AnswerFiles, ModelPort, PortError}; +use crate::model::{AnswerFiles, ModelPort, PortError, RecordedAnswers}; use crate::packet::{Packet, Stance}; use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate, validation_items}; use crate::split::{Split, SplitScheme, fnv1a64, fnv1a64_of, hex16}; @@ -1675,27 +1675,90 @@ impl RunStores { floors: &mut [&mut dyn ModelPort], scorer: &Scorer, ) -> Result { - let store = |_| RunError::Store; check_embedder(self, scorer)?; - let held = self.held_splits(n, &[Split::Validation])?; - let inventory = self.inventory.inventory().map_err(store)?; - let ontology = self.ontology.ontology().map_err(store)?; - - let dir = self.relower_dir(n, arm_label); - std::fs::create_dir_all(&dir).map_err(|e| RunError::Io(e.kind()))?; - for suffix in ["cc", "cc.lock"] { - let _ = std::fs::remove_file(dir.join(format!("base.{suffix}"))); - } - let mut scratch = PacketStore::open(&dir.join("base.cc")).map_err(store)?; - if (1..FLOOR_ARMS_MIN).contains(&floors.len()) { return Err(RunError::FloorArms); } - // Per language, per item: the item text, then the recorded and arm - // lowerings and one per floor arm (`None` when unusable). - type Row = (String, Option, Option, Vec>); - let mut waiting: Vec = Vec::new(); - let mut floor_waiting = false; + let dir = self.relower_dir(n, arm_label); + // Ports in the order the prompts are asked: recorded, floors, arm. + let floor_count = floors.len(); + let mut ports: Vec<&mut dyn ModelPort> = vec![recorded]; + for f in floors.iter_mut() { + let f: &mut dyn ModelPort = &mut **f; + ports.push(f); + } + ports.push(arm); + let drawn = self.read_draws(n, &dir.join("base.cc"), &mut ports, scorer)?; + let arm_at = floor_count + 1; + if drawn.waiting[1..arm_at].iter().any(|w| !w.is_empty()) { + return Err(RunError::FloorUnanswered); + } + if !drawn.waiting[arm_at].is_empty() { + return Ok(RelowerOutcome::Unanswered { prompts: drawn.waiting[arm_at].clone() }); + } + let mut languages = Vec::new(); + for l in &drawn.languages { + let of_port = |p: usize| -> Vec { l.items.iter().map(|(_, s)| s[p].clone()).collect() }; + let rec = of_port(0); + let floor_scores: Vec> = (1..arm_at).map(of_port).collect(); + languages.push(RelowerReading { + language: l.language.clone(), + n: l.items.len() as u32, + skipped: l.skipped, + unusable: l.unusable[arm_at], + floor_unusable: l.unusable[1..arm_at].iter().sum(), + recorded: reading_of(&rec)?, + arm: reading_of(&of_port(arm_at))?, + floor: if floor_count == 0 { None } else { Some(paired_floor(&rec, &floor_scores)) }, + }); + } + let verdict = if floor_count > 0 { + let panel: Vec<(Reading, Reading, Reading)> = languages + .iter() + .filter_map(|l| Some((l.recorded, l.arm, l.floor?))) + .collect(); + Some(decide_panel(&panel)) + } else { + None + }; + Ok(RelowerOutcome::Read { languages, verdict }) + } + + /// Asks every port, in the order given, each live Validation item's + /// byte-identical lowering prompt of round `n` and scores the answers against + /// the items in one [`Scorer::score`] call: the one owner of per-item + /// scoring for [`relower`](Self::relower) and + /// [`calibrate`](Self::calibrate). An answer that is not a usable lowering + /// scores 0. Port 0 is the recorded lowering, which must be answered + /// (`Unrecorded` at once); any other port that is not is listed in + /// `waiting`, and nothing is scored. `scratch` is the packet store the + /// prompts are built in, removed and reopened here; its directory is + /// created here, after the round is found. + fn read_draws( + &self, + n: u32, + scratch: &Path, + ports: &mut [&mut dyn ModelPort], + scorer: &Scorer, + ) -> Result { + let store = |_| RunError::Store; + let held = self.held_splits(n, &[Split::Validation])?; + let inventory = self.inventory.inventory().map_err(store)?; + let ontology = self.ontology.ontology().map_err(store)?; + if let Some(dir) = scratch.parent() { + std::fs::create_dir_all(dir).map_err(|e| RunError::Io(e.kind()))?; + } + for suffix in ["", ".lock"] { + let mut name = scratch.as_os_str().to_owned(); + name.push(suffix); + let _ = std::fs::remove_file(PathBuf::from(name)); + } + let mut scratch = PacketStore::open(scratch).map_err(store)?; + + // Per language, per item: the item id and text, then each port's lowering + // (`None` when unusable or unanswered). + let mut waiting: Vec> = vec![Vec::new(); ports.len()]; + type Row = (String, String, Vec>); let mut rows: Vec> = Vec::new(); for (_, _, items) in &held { let mut language_rows = Vec::new(); @@ -1707,49 +1770,33 @@ impl RunStores { else { continue; }; - let text = |a: Asked| match a { - Asked::Text(t) => Some(t), - _ => None, - }; - let recorded_text = match asked(recorded, &prompt)? { - Asked::Waiting => return Err(RunError::Unrecorded), - other => text(other), - }; - let mut floor_texts = Vec::new(); - for f in floors.iter_mut() { - floor_texts.push(match asked(&mut **f, &prompt)? { + let mut texts = Vec::new(); + for (p, port) in ports.iter_mut().enumerate() { + texts.push(match asked(&mut **port, &prompt)? { + Asked::Text(t) => Some(t), + Asked::Unusable => None, + Asked::Waiting if p == 0 => return Err(RunError::Unrecorded), Asked::Waiting => { - floor_waiting = true; + let d = prompt.digest(); + if !waiting[p].contains(&d) { + waiting[p].push(d); + } None } - other => text(other), }); } - let arm_text = match asked(arm, &prompt)? { - Asked::Waiting => { - let d = prompt.digest(); - if !waiting.contains(&d) { - waiting.push(d); - } - None - } - other => text(other), - }; - language_rows.push((item.text.clone(), recorded_text, arm_text, floor_texts)); + language_rows.push((item.id.as_str().to_string(), item.text.clone(), texts)); } rows.push(language_rows); } - if floor_waiting { - return Err(RunError::FloorUnanswered); - } - if !waiting.is_empty() { - return Ok(RelowerOutcome::Unanswered { prompts: waiting }); + if waiting.iter().any(|w| !w.is_empty()) { + return Ok(Drawn { languages: Vec::new(), waiting }); } // One score call: each usable lowering against its item, in row order. let mut inputs: Vec = Vec::new(); - for (reference, recorded, arm, floor_texts) in rows.iter().flatten() { - for output in [recorded, arm].into_iter().chain(floor_texts).flatten() { + for (_, reference, texts) in rows.iter().flatten() { + for output in texts.iter().flatten() { inputs.push(ScoreInput { reference, output }); } } @@ -1759,67 +1806,272 @@ impl RunStores { scorer.score(&inputs).map_err(|_| RunError::Embed)? } .into_iter(); - let mut next = |t: &Option| -> Result { - match t { - Some(_) => scores.next().ok_or(RunError::Store), - None => Ok(ItemScore { chrf: 0.0, meaning: Some(0.0) }), - } - }; - let reading_of = |scored: &[ItemScore]| { - Reading::of(&SplitMean { - count: scored.len(), - chrf: mean(scored.iter().map(|s| s.chrf)), - meaning: mean(scored.iter().filter_map(|s| s.meaning)), - }) - .ok_or(RunError::NoValidation) - }; - let mut languages = Vec::new(); let live = self.corpus.all().map_err(store)?; - for ((language, _, _), language_rows) in held.iter().zip(&rows) { - let (mut rec, mut arm_scores) = (Vec::new(), Vec::new()); - let mut floor_scores: Vec> = vec![Vec::new(); floors.len()]; - let (mut unusable, mut floor_unusable) = (0u32, 0u32); - for (_, r, a, f) in language_rows { - rec.push(next(r)?); - arm_scores.push(next(a)?); - if a.is_none() { - unusable += 1; - } - for (scores, t) in floor_scores.iter_mut().zip(f) { - scores.push(next(t)?); - if t.is_none() { - floor_unusable += 1; - } + let mut languages = Vec::new(); + for ((language, _, _), language_rows) in held.iter().zip(rows) { + let mut unusable = vec![0u32; ports.len()]; + let mut items = Vec::new(); + for (id, _, texts) in language_rows { + let mut item_scores = Vec::new(); + for (p, t) in texts.iter().enumerate() { + item_scores.push(match t { + Some(_) => scores.next().ok_or(RunError::Store)?, + None => { + unusable[p] += 1; + ItemScore { chrf: 0.0, meaning: Some(0.0) } + } + }); } + items.push((id, item_scores)); } - let recorded_reading = reading_of(&rec)?; - languages.push(RelowerReading { + let live_validation = live + .iter() + .filter(|i| &i.language == language && SplitScheme::V1.live(i) == Some(Split::Validation)) + .count() as u32; + languages.push(DrawnLanguage { language: language.clone(), - n: language_rows.len() as u32, - skipped: live - .iter() - .filter(|i| { - &i.language == language && SplitScheme::V1.live(i) == Some(Split::Validation) - }) - .count() as u32 - - language_rows.len() as u32, + skipped: live_validation - items.len() as u32, + items, unusable, - floor_unusable, - recorded: recorded_reading, - arm: reading_of(&arm_scores)?, - floor: if floors.is_empty() { None } else { Some(paired_floor(&rec, &floor_scores)) }, }); } - let verdict = if !floors.is_empty() { - let panel: Vec<(Reading, Reading, Reading)> = languages - .iter() - .filter_map(|l| Some((l.recorded, l.arm, l.floor?))) - .collect(); - Some(decide_panel(&panel)) - } else { - None + Ok(Drawn { languages, waiting }) + } +} + +/// One panel language of [`RunStores::read_draws`]. +struct DrawnLanguage { + language: String, + /// Live Validation items of the language with no lowering prompt. + skipped: u32, + /// Per read item in key order: its id and one score per port. + items: Vec<(String, Vec)>, + /// Per port, answers that were not a usable lowering. + unusable: Vec, +} + +struct Drawn { + /// Per panel language in panel order; empty while any port is waiting. + languages: Vec, + /// Per port, the digests of the prompts it holds no answer to. + waiting: Vec>, +} + +/// A reading over per-item scores; `NoValidation` without items. +fn reading_of(scored: &[ItemScore]) -> Result { + Reading::of(&SplitMean { + count: scored.len(), + chrf: mean(scored.iter().map(|s| s.chrf)), + meaning: mean(scored.iter().filter_map(|s| s.meaning)), + }) + .ok_or(RunError::NoValidation) +} + +/// How trials of one kind read: counts, not rates. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Counts { + pub trials: u32, + pub accepted: u32, + pub nogain: u32, + pub regressed: u32, +} + +impl Counts { + fn tally(&mut self, v: Verdict) { + self.trials += 1; + match v { + Verdict::Accepted => self.accepted += 1, + Verdict::Rejected(Why::Regressed(_)) => self.regressed += 1, + Verdict::Rejected(_) => self.nogain += 1, + } + } +} + +/// What [`calibration_trials`] counted over one set of items. +#[derive(Clone, Debug, PartialEq)] +pub struct Trials { + /// Ordered (base, candidate) pairs of prompted draws, each over every floor. + pub prompted: Counts, + /// Per degraded arm in the order given: the arm as the candidate. + pub degraded: Vec, + /// Every candidate arm as the candidate, pooled. + pub candidate: Counts, + /// The mean of [`panel_noise`] over every (base, floor) choice. + pub noise: f64, +} + +/// Every `k`-subset of `pool`, in order. +fn subsets(pool: &[usize], k: usize) -> Vec> { + if k == 0 { + return vec![Vec::new()]; + } + let mut out = Vec::new(); + for (i, &first) in pool.iter().enumerate() { + for mut rest in subsets(&pool[i + 1..], k - 1) { + rest.insert(0, first); + out.push(rest); + } + } + out +} + +/// The accept rule replayed over draws that are all lowerings of one round, as +/// the live loop would meet them: [`decide_panel`] over (base, candidate, +/// [`paired_floor`] of the base against `FLOOR_ARMS_MIN` other prompted draws), +/// one row per language. `draws` is `[port][language][item]`; ports +/// `0..prompted` are prompted draws (the recorded lowering first), then +/// `degraded` and `candidate` ports follow. A language with no items reads +/// NaN, which fails closed. +pub fn calibration_trials( + draws: &[Vec>], + prompted: usize, + degraded: usize, + candidate: usize, +) -> Trials { + let languages = draws.first().map_or(0, Vec::len); + let rows = |b: usize, c: usize, floors: &[usize]| -> Vec<(Reading, Reading, Reading)> { + let read = |scores: &[ItemScore]| { + reading_of(scores).unwrap_or(Reading { chrf: f64::NAN, meaning: f64::NAN, composite: f64::NAN }) }; - Ok(RelowerOutcome::Read { languages, verdict }) + (0..languages) + .map(|l| { + let arms: Vec> = floors.iter().map(|&f| draws[f][l].clone()).collect(); + (read(&draws[b][l]), read(&draws[c][l]), paired_floor(&draws[b][l], &arms)) + }) + .collect() + }; + let pool: Vec = (0..prompted).collect(); + let without = |out: &[usize]| -> Vec { pool.iter().copied().filter(|p| !out.contains(p)).collect() }; + let (mut prompted_counts, mut candidate_counts) = (Counts::default(), Counts::default()); + let mut degraded_counts = vec![Counts::default(); degraded]; + let (mut noise, mut noises) = (0.0, 0u32); + for &b in &pool { + for floors in subsets(&without(&[b]), FLOOR_ARMS_MIN) { + for (d, counts) in degraded_counts.iter_mut().enumerate() { + counts.tally(decide_panel(&rows(b, prompted + d, &floors))); + } + for c in prompted + degraded..prompted + degraded + candidate { + candidate_counts.tally(decide_panel(&rows(b, c, &floors))); + } + noise += panel_noise(&rows(b, b, &floors)); + noises += 1; + } + for &c in pool.iter().filter(|&&c| c != b) { + for floors in subsets(&without(&[b, c]), FLOOR_ARMS_MIN) { + prompted_counts.tally(decide_panel(&rows(b, c, &floors))); + } + } + } + Trials { + prompted: prompted_counts, + degraded: degraded_counts, + candidate: candidate_counts, + noise: noise / f64::from(noises), + } +} + +/// One panel language of a class: items read and live Validation items of the +/// class not read. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ClassLanguage { + pub language: String, + pub n: u32, + pub skipped: u32, +} + +/// The trials over one class of items. +#[derive(Clone, Debug, PartialEq)] +pub struct ClassTrials { + pub name: String, + pub languages: Vec, + pub trials: Trials, +} + +#[derive(Debug)] +pub enum CalibrateOutcome { + /// Draw labels with the number of prompts each has no answer to. + Unanswered(Vec<(String, u32)>), + /// `unusable` over every prompted, degraded and candidate draw; `all` first, + /// then the classes in the order given. + Read { unusable: u32, classes: Vec }, +} + +impl RunStores { + /// Replays the accept rule over answered relower draws: reads every draw of + /// round `n` (the recorded lowering and the `relower--