From ee3e4a2e386ecdc88d0a648bd849831b31a87345 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 01:29:57 +0000 Subject: [PATCH 01/28] withheld-candidate: candidate round stops on a withheld lowering prompt (#68) --- crates/weksa/tests/golf_loop.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 9272af3..3a33363 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -5282,3 +5282,20 @@ fn a_stance_ablation_stops_on_a_withheld_prompt() { let r = ablate(&dao, &mut Withholds, &mut Lowerer::new(blind)); assert!(matches!(r, Err(RunError::Withheld)), "{:?}", r.map(|_| ())); } + +#[test] +fn the_cycles_candidate_round_stops_on_a_withheld_lowering_prompt() { + // The edit is answered and the candidate round encodes at rev 4; every lowering + // prompt is then refused. No decision is recorded and no packets are left. + let mut dao = dao(); + two_rounds(&mut dao); + let mut model = cycle_fake(ONE_CONCEPT); + model.raw_lowering = Some(Err(PortError::Withheld)); + let r = cycle(&mut dao.run, &mut model, &Scorer::new(Bag)); + assert!(matches!(r, Err(RunError::Withheld)), "{:?}", r.map(|_| ())); + assert!(dao.run.decisions().unwrap().is_empty(), "a refused cycle decides nothing"); + assert!( + !dao.root.join("cycle-1/packets.cc").exists(), + "the refused attempt's packets are discarded" + ); +} From 97aa026498318c84ef675a69ceeef65a3bb6ba8a Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 02:05:41 +0000 Subject: [PATCH 02/28] weksa loop relower: fixed-packet re-lowering measure with a lowering-only floor (#69) --- crates/weksa/src/main.rs | 77 +++- crates/weksa/src/run.rs | 366 ++++++++++++++++--- crates/weksa/tests/cli.rs | 80 +++++ crates/weksa/tests/golf_loop.rs | 436 +++++++++++++++++++++++ docs/targets/interlingua-lowering-map.md | 27 +- 5 files changed, 932 insertions(+), 54 deletions(-) diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index b76051c..9ba2135 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -13,9 +13,9 @@ use weksa::entity::{Entity, EntityOrigin, LanguageForms, Pronunciation}; use weksa::inventory::{proof_revision, seed, speaker_scene_revision}; use weksa::packet::{FeatValue, FeatureValue, Grade}; use weksa::packet::proof_packet; -use weksa::model::RecordedAnswers; +use weksa::model::{ModelPort, RecordedAnswers}; use weksa::run::{ - AblationOutcome, CycleOutcome, LowererReach, OntologyMintOutcome, RecallOutcome, RoundOutcome, RunPaths, RunStores, cycle, round, + AblationOutcome, CycleOutcome, LowererReach, OntologyMintOutcome, RecallOutcome, RelowerOutcome, RoundOutcome, RunPaths, RunStores, cycle, round, }; use weksa::score::ScoreInput; use weksa::{ @@ -153,6 +153,26 @@ enum LoopCommand { #[arg(long, default_value = "http://127.0.0.1:11434")] ollama_url: String, }, + /// Re-lower round ROUND's Validation packets through arm ARM, score them + /// against the items, and, with --floor ARM, decide against that second arm + /// by the panel accept rule. Prompts and answers live in + /// `/relower--`; the command exits 3 until the arm is + /// answered. --lowerer names the arm's model in the printed line. A + /// diagnostic, never a gate; records nothing; Test and Train are never + /// touched. Labels match [a-z0-9-]{1,32} and differ. + Relower { + root: PathBuf, + #[arg(long)] + round: u32, + #[arg(long)] + arm: String, + #[arg(long)] + lowerer: String, + #[arg(long)] + floor: Option, + #[arg(long, default_value = "http://127.0.0.1:11434")] + ollama_url: String, + }, /// Run one edit-accept cycle of the run at ROOT. The stores are read only /// from the run's record. The editor's prompt, then the candidate round's /// encode prompts, then its lowering prompts are written in turn; the @@ -1115,6 +1135,59 @@ fn main() -> Result<()> { } } } + Command::Loop(LoopCommand::Relower { + root, + round, + arm, + lowerer, + floor, + ollama_url, + }) => { + let label = |l: &str| { + (1..=32).contains(&l.len()) + && l.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') + }; + if !label(&arm) || floor.as_deref().is_some_and(|f| !label(f) || f == arm) { + anyhow::bail!("arm and floor labels must match [a-z0-9-]{{1,32}} and differ"); + } + let run = RunStores::open(&root)?; + let mut recorded = RecordedAnswers::new( + root.join(format!("round-{round}")), + run.record().model.clone(), + ); + let mut files = run.answer_files(run.relower_dir(round, &arm))?; + let mut floor_port = floor.as_deref().map(|f| { + RecordedAnswers::new(run.relower_dir(round, f), run.record().model.clone()) + }); + let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder)); + match run.relower( + round, + &arm, + &mut recorded, + &mut files, + floor_port.as_mut().map(|p| p as &mut dyn ModelPort), + &scorer, + )? { + RelowerOutcome::Unanswered { prompts } => { + for digest in &prompts { + println!("unanswered {digest}"); + } + println!("unanswered {} prompts", prompts.len()); + std::process::exit(3); + } + RelowerOutcome::Read { languages, verdict } => { + for l in languages { + println!( + "{}", + run.labelled(&format!("{} arm={arm} lowerer={lowerer}", l.line())) + ); + } + if let Some(v) = verdict { + println!("verdict {v:?}"); + } + } + } + } Command::Loop(LoopCommand::Recall { root }) => { let run = RunStores::open(&root)?; let mut files = run.answer_files(run.recall_dir())?; diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index d4b3f9f..1dc4780 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -495,6 +495,9 @@ pub enum RunError { Ontology, /// A scene does not read against the run's inventory. SpeakerState, + /// The floor arm of a relower holds no answer to a prompt; it is a + /// reference, never written to, and must be answered whole first. + FloorUnanswered, } impl fmt::Display for RunError { @@ -523,6 +526,9 @@ impl fmt::Display for RunError { RunError::SpeakerState => { f.write_str("a scene does not read against the run's inventory") } + RunError::FloorUnanswered => { + f.write_str("the floor arm holds no answer to a lowering prompt; answer it whole first") + } RunError::Panel => f.write_str( "the panel is empty, repeats or misspells a language, or names one with no live Train or Validation item", ), @@ -1337,6 +1343,66 @@ type HeldSplit = (String, Split, Vec<(CorpusItem, Packet)>); type Lowered = (String, String, String, [Option; 2]); impl RunStores { + /// The live items of each panel language in each listed split with their + /// round-`n` packets, per language then split, sorted by item key. A round + /// that is absent or a cycle's is `Malformed("round")`. Test is never held. + fn held_splits(&self, n: u32, splits: &[Split]) -> Result, RunError> { + let store = |_| RunError::Store; + self.rounds() + .map_err(store)? + .into_iter() + .find(|r| r.n() == n && r.cycle().is_none()) + .ok_or(RunError::Malformed("round"))?; + let round_packets = PacketStore::open(&self.root.join(format!("round-{n}")).join("packets.cc")) + .map_err(store)? + .all() + .map_err(store)?; + let mut held: Vec = Vec::new(); + for tag in &self.record.panel { + for split in splits { + held.push((tag.clone(), *split, Vec::new())); + } + } + for packet in round_packets { + let item = self + .corpus + .get(packet.provenance.corpus_item.as_str()) + .map_err(store)? + .ok_or(RunError::Store)?; + for (language, split, items) in &mut held { + if *language == item.language && SplitScheme::V1.live(&item) == Some(*split) { + items.push((item.clone(), packet.clone())); + } + } + } + for (_, _, items) in &mut held { + items.sort_by(|a, b| a.0.id.as_str().cmp(b.0.id.as_str())); + } + Ok(held) + } + + /// The lowering prompt of `packet` for `item`: admitted into `store`, + /// assembled and lowered. The one builder for the round's prompt, which + /// ablation and relower must reproduce byte for byte. `None` when the + /// lowering prompt itself is refused, as it was for the round. + fn lowering_prompt( + &self, + store: &mut PacketStore, + packet: Packet, + item: &CorpusItem, + inventory: &crate::Inventory, + ontology: &crate::Ontology, + ) -> Result, RunError> { + let packet_item = packet.provenance.corpus_item.clone(); + let view = store + .admit(packet, &self.inventory, &self.corpus, &self.entities) + .map_err(|_| RunError::Store)?; + let view = self + .assembled(view, &self.inventory) + .map_err(|_| RunError::Store)?; + Ok(Prompt::lower(&view, &packet_item, &item.language, inventory, &self.entities, ontology).ok()) + } + /// `/ablation-`, where a stance ablation's prompts, answers and /// variant packet stores live. pub fn ablation_dir(&self, n: u32) -> PathBuf { @@ -1362,11 +1428,8 @@ impl RunStores { ) -> Result { let store = |_| RunError::Store; check_drift(self, model, scorer)?; - self.rounds() - .map_err(store)? - .into_iter() - .find(|r| r.n() == n && r.cycle().is_none()) - .ok_or(RunError::Malformed("round"))?; + // A refused round writes nothing: the round is read before any folder is made. + let mut splits = self.held_splits(n, &[Split::Train, Split::Validation])?; let inventory = self.inventory.inventory().map_err(store)?; let ontology = self.ontology.ontology().map_err(store)?; @@ -1383,56 +1446,17 @@ impl RunStores { PacketStore::open(&dir.join("permute.cc")).map_err(store)?, ]; - // Live Train and Validation packets, in item key order within a split. - let round_packets = PacketStore::open(&self.root.join(format!("round-{n}")).join("packets.cc")) - .map_err(store)? - .all() - .map_err(store)?; - let mut splits: Vec = Vec::new(); - for tag in &self.record.panel { - for split in [Split::Train, Split::Validation] { - splits.push((tag.clone(), split, Vec::new())); - } - } - for packet in round_packets { - let item = self - .corpus - .get(packet.provenance.corpus_item.as_str()) - .map_err(store)? - .ok_or(RunError::Store)?; - for (language, split, held) in &mut splits { - if *language == item.language && SplitScheme::V1.live(&item) == Some(*split) { - held.push((item.clone(), packet.clone())); - } - } - } - let mut waiting: Vec = Vec::new(); let mut readings = Vec::new(); // Texts per split: (item text, recorded, replay, [drop, permute]). let mut texts: Vec> = Vec::new(); let mut unusable = vec![0u32; splits.len()]; for (at, (_, _, held)) in splits.iter_mut().enumerate() { - held.sort_by(|a, b| a.0.id.as_str().cmp(b.0.id.as_str())); let mut rows = Vec::new(); for (i, (item, packet)) in held.iter().enumerate() { - let packet_item = packet.provenance.corpus_item.clone(); let mut prompt_of = |slot: usize, packet: Packet| -> Result { - let view = stores[slot] - .admit(packet, &self.inventory, &self.corpus, &self.entities) - .map_err(|_| RunError::Store)?; - let view = self - .assembled(view, &self.inventory) - .map_err(|_| RunError::Store)?; - Prompt::lower( - &view, - &packet_item, - &item.language, - &inventory, - &self.entities, - &ontology, - ) - .map_err(|_| RunError::Store) + self.lowering_prompt(&mut stores[slot], packet, item, &inventory, &ontology)? + .ok_or(RunError::Store) }; let base = prompt_of(0, packet.clone())?; let mut variants: [Option; 2] = [None, None]; @@ -1547,6 +1571,236 @@ impl RunStores { } } +/// One panel language's relower reading: Validation items only. +#[derive(Clone, Debug, PartialEq)] +pub struct RelowerReading { + pub language: String, + /// Live Validation items of the language read: those with a round packet + /// whose lowering prompt the round could build. Every reading below is over + /// these items, so recorded, arm and floor stay comparable. + pub n: u32, + /// Live Validation items of the language not read: refused by the round + /// before admission (no packet) or at the lowering prompt. They are in no + /// reading, so `recorded` can differ from the round's own Validation + /// reading, which scores them 0. + pub skipped: u32, + /// Arm answers that were not a usable lowering; each scored 0. + pub unusable: u32, + /// Floor-arm answers that were not a usable lowering; each scored 0, which + /// widens `floor`. Zero without a floor arm. + pub floor_unusable: u32, + /// The round's own lowering, read against the items. + pub recorded: Reading, + pub arm: Reading, + /// `spread(recorded, floor arm)`, when a floor arm was given. + pub floor: Option, +} + +impl RelowerReading { + pub fn line(&self) -> String { + let show = |r: &Reading| { + format!("chrf={:.2} meaning={:.3} composite={:.4}", r.chrf, r.meaning, r.composite) + }; + let floor = self + .floor + .as_ref() + .map_or(String::new(), |f| { + format!(" | floor {} unusable={}", show(f), self.floor_unusable) + }); + format!( + "relower {} n={} skipped={} unusable={} recorded {} | arm {}{floor}", + self.language, + self.n, + self.skipped, + self.unusable, + show(&self.recorded), + show(&self.arm), + ) + } +} + +#[derive(Debug)] +pub enum RelowerOutcome { + Unanswered { prompts: Vec }, + /// Per panel language in panel order; the verdict only when a floor arm was given. + Read { languages: Vec, verdict: Option }, +} + +impl RunStores { + /// `/relower--`, where an arm's prompts and answers live. + pub fn relower_dir(&self, n: u32, arm: &str) -> PathBuf { + self.root.join(format!("relower-{n}-{arm}")) + } + + /// A diagnostic like [`recall`](Self::recall) and + /// [`stance_ablation`](Self::stance_ablation), never a gate: it serves the + /// distil decision, whether a tuned lowerer is worth keeping against the + /// prompted one (calibration-pending: see the map). Asks `arm` the byte-identical lowering prompts of round `n`'s live + /// Validation items, reads the arm and the round's own lowering + /// (`recorded`) against the items, and, given a `floor` arm (a second + /// prompted lowering, so the floor is lowering-only noise), decides by + /// [`decide_panel`] over (recorded, arm, spread(recorded, floor)). Only + /// the model of `arm` is unpinned; the embedder is checked. An arm answer + /// that is not a usable lowering scores 0 and is counted in `unusable`. + /// Records nothing; `arm_label` names only where the scratch packet store + /// lives, in the arm's directory. + pub fn relower( + &self, + n: u32, + arm_label: &str, + recorded: &mut dyn ModelPort, + arm: &mut dyn ModelPort, + mut floor: Option<&mut dyn ModelPort>, + scorer: &Scorer, + ) -> Result { + let store = |_| RunError::Store; + check_embedder(self, scorer)?; + let held = self.held_splits(n, &[Split::Validation])?; + let inventory = self.inventory.inventory().map_err(store)?; + let ontology = self.ontology.ontology().map_err(store)?; + + let dir = self.relower_dir(n, arm_label); + std::fs::create_dir_all(&dir).map_err(|e| RunError::Io(e.kind()))?; + for suffix in ["cc", "cc.lock"] { + let _ = std::fs::remove_file(dir.join(format!("base.{suffix}"))); + } + let mut scratch = PacketStore::open(&dir.join("base.cc")).map_err(store)?; + + // Per language, per item: the item text, then the recorded, arm and floor + // lowerings (`None` when unusable). + type Row = (String, Option, Option, Option); + let mut waiting: Vec = Vec::new(); + let mut floor_waiting = false; + let mut rows: Vec> = Vec::new(); + for (_, _, items) in &held { + let mut language_rows = Vec::new(); + for (item, packet) in items { + // An item the round could not lower has no prompt to ask; it is skipped + // and counted, not scored. + let Some(prompt) = + self.lowering_prompt(&mut scratch, packet.clone(), item, &inventory, &ontology)? + else { + continue; + }; + let text = |a: Asked| match a { + Asked::Text(t) => Some(t), + _ => None, + }; + let recorded_text = match asked(recorded, &prompt)? { + Asked::Waiting => return Err(RunError::Unrecorded), + other => text(other), + }; + let floor_text = match floor.as_deref_mut() { + Some(f) => match asked(f, &prompt)? { + Asked::Waiting => { + floor_waiting = true; + None + } + other => text(other), + }, + None => None, + }; + let arm_text = match asked(arm, &prompt)? { + Asked::Waiting => { + let d = prompt.digest(); + if !waiting.contains(&d) { + waiting.push(d); + } + None + } + other => text(other), + }; + language_rows.push((item.text.clone(), recorded_text, arm_text, floor_text)); + } + rows.push(language_rows); + } + if floor_waiting { + return Err(RunError::FloorUnanswered); + } + if !waiting.is_empty() { + return Ok(RelowerOutcome::Unanswered { prompts: waiting }); + } + + // One score call: each usable lowering against its item, in row order. + let mut inputs: Vec = Vec::new(); + for (reference, recorded, arm, floor_text) in rows.iter().flatten() { + for output in [recorded, arm, floor_text].into_iter().flatten() { + inputs.push(ScoreInput { reference, output }); + } + } + let mut scores = if inputs.is_empty() { + Vec::new() + } else { + scorer.score(&inputs).map_err(|_| RunError::Embed)? + } + .into_iter(); + let mut next = |t: &Option| -> Result { + match t { + Some(_) => scores.next().ok_or(RunError::Store), + None => Ok(ItemScore { chrf: 0.0, meaning: Some(0.0) }), + } + }; + let reading_of = |scored: &[ItemScore]| { + Reading::of(&SplitMean { + count: scored.len(), + chrf: mean(scored.iter().map(|s| s.chrf)), + meaning: mean(scored.iter().filter_map(|s| s.meaning)), + }) + .ok_or(RunError::NoValidation) + }; + let mut languages = Vec::new(); + let live = self.corpus.all().map_err(store)?; + for ((language, _, _), language_rows) in held.iter().zip(&rows) { + let (mut rec, mut arm_scores, mut floor_scores) = (Vec::new(), Vec::new(), Vec::new()); + let (mut unusable, mut floor_unusable) = (0u32, 0u32); + for (_, r, a, f) in language_rows { + rec.push(next(r)?); + arm_scores.push(next(a)?); + if a.is_none() { + unusable += 1; + } + if floor.is_some() { + floor_scores.push(next(f)?); + if f.is_none() { + floor_unusable += 1; + } + } + } + let recorded_reading = reading_of(&rec)?; + languages.push(RelowerReading { + language: language.clone(), + n: language_rows.len() as u32, + skipped: live + .iter() + .filter(|i| { + &i.language == language && SplitScheme::V1.live(i) == Some(Split::Validation) + }) + .count() as u32 + - language_rows.len() as u32, + unusable, + floor_unusable, + recorded: recorded_reading, + arm: reading_of(&arm_scores)?, + floor: if floor.is_some() { + Some(spread(&recorded_reading, &reading_of(&floor_scores)?)) + } else { + None + }, + }); + } + let verdict = if floor.is_some() { + let panel: Vec<(Reading, Reading, Reading)> = languages + .iter() + .filter_map(|l| Some((l.recorded, l.arm, l.floor?))) + .collect(); + Some(decide_panel(&panel)) + } else { + None + }; + Ok(RelowerOutcome::Read { languages, verdict }) + } +} + /// The distinct rule codes an admission refusal broke: validator rules by /// code, a store refusal by its variant name. Nothing from the packet or the /// stores is kept; an I/O failure is `io`. @@ -1613,6 +1867,11 @@ fn check_drift( if model.model() != run.record.model { return Err(RunError::Drift(DriftOf::Model)); } + check_embedder(run, scorer) +} + +/// The embedder the run pinned is the one scoring. +fn check_embedder(run: &RunStores, scorer: &Scorer) -> Result<(), RunError> { if scorer.embedder().model() != run.record.embedder { return Err(RunError::Drift(DriftOf::Embedder)); } @@ -2223,16 +2482,21 @@ fn floor_of( let (a, b) = (replicate(1, tag)?, replicate(2, tag)?); Ok(( tag.clone(), - Reading { - chrf: (a.chrf - b.chrf).abs(), - meaning: (a.meaning - b.meaning).abs(), - composite: (a.composite - b.composite).abs(), - }, + spread(&a, &b), )) }) .collect() } +/// The absolute difference of two readings, metric by metric. +fn spread(a: &Reading, b: &Reading) -> Reading { + Reading { + chrf: (a.chrf - b.chrf).abs(), + meaning: (a.meaning - b.meaning).abs(), + composite: (a.composite - b.composite).abs(), + } +} + /// Each panel language's Validation reading in `round`, in panel order. fn readings_of( round: &RoundRecord, diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 40aa288..95a147a 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -1778,6 +1778,86 @@ fn loop_ablate_waits_then_prints_one_line_per_split() { } } +#[test] +fn cli_loop_relower_exits_3_until_answered_and_refuses_bad_labels() { + let d = tempfile::tempdir().unwrap(); + open_dao_run(d.path()); + let url = fake_embedder(); + let root = d.path().join("canaryroot"); + let mut run = weksa::run::RunStores::open(&root).unwrap(); + let scorer = weksa::Scorer::new(weksa::OllamaEmbedder::new(&url, &run.record().embedder)); + let weksa::run::RoundOutcome::Recorded(r) = weksa::run::round(&mut run, &mut DaoModel, &scorer).unwrap() + else { + panic!("answered") + }; + let said = r#"{"text":"The highest excellence is like the water."}"#; + for call in &r.calls()[3..] { + std::fs::write(root.join(format!("round-1/{}.answer", call.prompt)), said).unwrap(); + } + let relower = |extra: &[&str]| { + weksa( + d.path(), + &[&["loop", "relower", "canaryroot", "--round", "1", "--lowerer", "tuned-x", "--ollama-url", &url], extra].concat(), + ) + }; + + // Unanswered: exit 3, the digests, a prompt file per digest in the arm's directory. + let first = relower(&["--arm", "tuned"]); + assert_eq!(first.status.code(), Some(3), "{}", out(&first)); + let text = out(&first); + let digests: Vec<&str> = text + .lines() + .filter_map(|l| l.strip_prefix("unanswered ")) + .filter(|x| x.len() == 16) + .collect(); + assert_eq!(digests.len(), 2, "the two Validation prompts: {text}"); + for digest in &digests { + assert!(root.join(format!("relower-1-tuned/{digest}.prompt")).is_file()); + std::fs::write(root.join(format!("relower-1-tuned/{digest}.answer")), said).unwrap(); + } + + // Answered: one line per language, labelled, naming the lowerer; no verdict without a floor. + let second = relower(&["--arm", "tuned"]); + assert!(second.status.success(), "{}", out(&second)); + let text = out(&second); + assert!(text.starts_with("relower en-US n=2 skipped=0 unusable=0 recorded "), "{text}"); + assert!(text.contains("arm=tuned lowerer=tuned-x"), "{text}"); + assert!(text.contains(" lowering="), "{text}"); + assert!(!text.contains("verdict") && !text.contains("floor"), "{text}"); + for leaked in ["item:", "packet:", "highest excellence", "canaryroot"] { + assert!(!text.contains(leaked), "{text}"); + } + + // A floor arm that is not answered whole is refused, and never written to. + let o = relower(&["--arm", "tuned", "--floor", "second"]); + assert_eq!(o.status.code(), Some(1), "{}", out(&o)); + assert!(!root.join("relower-1-second").exists()); + + // Answered floor: a verdict line. + std::fs::create_dir(root.join("relower-1-second")).unwrap(); + for digest in &digests { + std::fs::write(root.join(format!("relower-1-second/{digest}.answer")), said).unwrap(); + } + let o = relower(&["--arm", "tuned", "--floor", "second"]); + assert!(o.status.success(), "{}", out(&o)); + let text = out(&o); + assert!(text.contains("| floor chrf=0.00") && text.trim_end().ends_with("verdict Rejected(NoGain)"), "{text}"); + + // Labels: equal, uppercase, empty, too long, a path. + let long = "a".repeat(33); + for args in [ + vec!["--arm", "same", "--floor", "same"], + vec!["--arm", "Upper"], + vec!["--arm", ""], + vec!["--arm", &long], + vec!["--arm", "../x"], + vec!["--arm", "tuned", "--floor", "Bad"], + ] { + let o = relower(&args); + assert_eq!(o.status.code(), Some(1), "{args:?}: {}", out(&o)); + } +} + #[test] fn loop_ablate_without_the_rounds_answers_says_so() { let d = tempfile::tempdir().unwrap(); diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 3a33363..179d0ea 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -3764,9 +3764,18 @@ fn grade_stance(features: &[(&str, u16)]) -> weksa::packet::Stance { /// packet. Stances: item 1 restraint 800; item 2 restraint 800; item 3 /// restraint 800 and affection 300; the written item restraint 500. fn recorded_dao(edit: impl FnOnce(&mut HashMap)) -> (Dao, RoundRecord) { + recorded_dao_in(|_, _| {}, edit) +} + +/// `recorded_dao` with `corpus_edit` run over the corpus and entities first. +fn recorded_dao_in( + corpus_edit: impl FnOnce(&mut CorpusStore, &weksa::EntityStore), + edit: impl FnOnce(&mut HashMap), +) -> (Dao, RoundRecord) { let mut dao = dao_over(|corpus, entities| { let entities = weksa::EntityStore::open(entities).unwrap(); let mut store = CorpusStore::open(corpus).unwrap(); + corpus_edit(&mut store, &entities); let mut written = store.get("item:daodejing:2").unwrap().unwrap(); written.id = WRITTEN.into(); written.text = "Heaven endures through the long years without end.".into(); @@ -4112,6 +4121,28 @@ fn a_stance_ablation_refuses_a_cycle_round_and_missing_round_answers() { )); } +#[test] +fn a_refused_stance_ablation_creates_no_folder_and_wipes_no_store() { + let (dao, _) = recorded_dao(|_| {}); + let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(reader)); + // No round 99: no folder appears. + assert!(matches!( + dao.run.stance_ablation(99, &mut rec, &mut model, &Scorer::new(Bag)), + Err(RunError::Malformed("round")) + )); + assert!(!dao.run.ablation_dir(99).exists(), "a refused ablation makes no folder"); + // An earlier ablation of a refused n keeps its stores byte for byte. + let kept = dao.run.ablation_dir(2); + std::fs::create_dir_all(&kept).unwrap(); + std::fs::write(kept.join("base.cc"), b"earlier ablation").unwrap(); + assert!(matches!( + dao.run.stance_ablation(2, &mut rec, &mut model, &Scorer::new(Bag)), + Err(RunError::Malformed("round")) + )); + assert_eq!(std::fs::read(kept.join("base.cc")).unwrap(), b"earlier ablation"); + assert_eq!(std::fs::read_dir(&kept).unwrap().count(), 1); +} + #[test] fn a_stance_ablation_refuses_a_cycle_round() { let mut dao = dao(); @@ -5299,3 +5330,408 @@ fn the_cycles_candidate_round_stops_on_a_withheld_lowering_prompt() { "the refused attempt's packets are discarded" ); } + +// ---- distil-slice: relower, a fixed-packet re-lowering measure ---- + +use weksa::run::{RelowerOutcome, RelowerReading}; + +/// `relower` of round 1 with the Bag scorer; `floor` is an optional second arm. +fn relower( + dao: &Dao, + recorded: &mut dyn ModelPort, + arm: &mut dyn ModelPort, + floor: Option<&mut dyn ModelPort>, +) -> Result { + dao.run.relower(1, "arm", recorded, arm, floor, &Scorer::new(Bag)) +} + +fn relowered(outcome: Result) -> (Vec, Option) { + match outcome.unwrap() { + RelowerOutcome::Read { languages, verdict } => (languages, verdict), + RelowerOutcome::Unanswered { prompts } => panic!("unanswered {prompts:?}"), + } +} + +#[test] +fn relower_asks_only_validation_lowering_prompts_of_the_round() { + let (dao, r) = recorded_dao(|_| {}); + let round_digests: Vec = r.calls()[4..].iter().map(|c| c.prompt.clone()).collect(); + // The reference pass: a lowerer shown every packet of the round, to learn + // which digest is which item's. + let (mut every, mut rec, mut arm) = (Lowerer::new(reader), Lowerer::new(reader), Lowerer::new(reader)); + read(ablate(&dao, &mut every, &mut Lowerer::new(reader)).unwrap()); + relowered(relower(&dao, &mut rec, &mut arm, None)); + let of = |keys: &[&str]| -> Vec { + every + .seen + .iter() + .filter(|s| keys.contains(&s.1.as_str())) + .map(|s| s.0.clone()) + .fold(Vec::new(), |mut v, d| { + if !v.contains(&d) { + v.push(d); + } + v + }) + }; + let mut want = of(&["item:daodejing:2", "item:daodejing:3", WRITTEN]); + want.sort(); + let mut got = arm.digests(); + got.sort(); + assert_eq!(got, want, "exactly the Validation lowering prompts"); + assert_eq!(want.len(), 3); + for d in &want { + assert!(round_digests.contains(d), "byte-identical to the round's prompt {d}"); + } + // Neither the lone Train item, a Test item nor an encode prompt. + for (_, key, text) in &arm.seen { + assert!(!["item:daodejing:1", "item:daodejing:4", "item:daodejing:5"].contains(&key.as_str())); + assert!(text.starts_with(LOWER_INSTRUCTIONS)); + } + assert_eq!(rec.digests().len(), 3, "the recorded port is asked the same prompts"); +} + +#[test] +fn relower_reads_recorded_and_arm_against_the_item() { + let (dao, _) = recorded_dao(|_| {}); + // An arm that says the item's own sentence beats a recorded lowering that + // adds the stance to it. + let (langs, verdict) = relowered(relower( + &dao, + &mut Lowerer::new(reader), + &mut Lowerer::new(blind), + None, + )); + let l = &langs[0]; + assert_eq!((l.language.as_str(), l.n, l.unusable), ("en-US", 3, 0)); + assert!(l.recorded.chrf < l.arm.chrf - 5.0, "{}", l.line()); + assert!(l.arm.composite > l.recorded.composite); + assert!(l.floor.is_none() && verdict.is_none(), "no floor arm, no verdict"); + // An arm equal to the recorded answers reads equal. + let (langs, _) = relowered(relower(&dao, &mut Lowerer::new(reader), &mut Lowerer::new(reader), None)); + assert_eq!(langs[0].recorded, langs[0].arm); +} + +/// Answers `text` for every item but `bad`, which gets something that is no lowering. +struct Breaks(&'static str); + +impl ModelPort for Breaks { + fn model(&self) -> &str { + "fake-model" + } + fn answer(&mut self, prompt: &Prompt) -> Result { + let key = prompt.item_keys()[0].clone(); + if key == self.0 { + return Ok("this is not a lowering draft".into()); + } + Ok(serde_json::json!({ "text": english(&key) }).to_string()) + } +} + +#[test] +fn relower_counts_an_unusable_arm_answer_as_zero() { + let (dao, _) = recorded_dao(|_| {}); + let keys = ["item:daodejing:2", "item:daodejing:3", WRITTEN]; + let (whole, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), None)); + let whole = whole[0].arm; + let (mut chrf, mut meaning) = (0.0, 0.0); + for key in keys { + let (langs, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Breaks(key), None)); + let l = &langs[0]; + assert_eq!((l.n, l.unusable), (3, 1), "{}", l.line()); + assert!(l.arm.chrf < whole.chrf && l.arm.meaning < whole.meaning, "{}", l.line()); + chrf += l.arm.chrf; + meaning += l.arm.meaning; + } + // Each item is refused once: the three means hold every item twice at its + // score and once at 0, so the refused item entered its mean as 0 and n stayed 3. + assert!((chrf - 2.0 * whole.chrf).abs() < 1e-6, "{chrf} {}", whole.chrf); + assert!((meaning - 2.0 * whole.meaning).abs() < 1e-6, "{meaning} {}", whole.meaning); +} + +#[test] +fn relower_scores_an_unusable_recorded_answer_as_zero_without_counting_it_against_the_arm() { + let (dao, _) = recorded_dao(|_| {}); + let (whole, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), None)); + let (langs, _) = relowered(relower(&dao, &mut Breaks("item:daodejing:3"), &mut Lowerer::new(blind), None)); + let l = &langs[0]; + assert_eq!(l.unusable, 0, "{}", l.line()); + assert_eq!(l.arm, whole[0].arm); + assert!(l.recorded.chrf < whole[0].recorded.chrf && l.recorded.meaning < whole[0].recorded.meaning); +} + +#[test] +fn relower_decides_by_decide_panel_against_the_floor_arm() { + use weksa::run::decide_panel; + let (dao, _) = recorded_dao(|_| {}); + let run = |recorded: fn(&str, &str) -> String, arm: fn(&str, &str) -> String| { + let mut floor = Lowerer::new(recorded); + let (langs, verdict) = relowered(relower( + &dao, + &mut Lowerer::new(recorded), + &mut Lowerer::new(arm), + Some(&mut floor), + )); + let l = &langs[0]; + let floor_reading = l.floor.expect("a floor arm gives a floor"); + // A floor arm that equals the recorded answers has no spread. + assert_eq!((floor_reading.chrf, floor_reading.meaning, floor_reading.composite), (0.0, 0.0, 0.0)); + assert_eq!(verdict, Some(decide_panel(&[(l.recorded, l.arm, floor_reading)]))); + verdict.unwrap() + }; + assert_eq!(run(reader, reader), Verdict::Rejected(Why::NoGain)); + assert_eq!(run(reader, blind), Verdict::Accepted); + assert!(matches!(run(blind, reader), Verdict::Rejected(Why::Regressed(_)))); +} + +#[test] +fn relower_floor_is_the_spread_of_recorded_and_the_floor_arm() { + let (dao, _) = recorded_dao(|_| {}); + let (langs, _) = relowered(relower( + &dao, + &mut Lowerer::new(blind), + &mut Lowerer::new(blind), + Some(&mut Breaks("item:daodejing:3")), + )); + let l = &langs[0]; + let f = l.floor.unwrap(); + // The floor arm read as an arm against the same recorded answers. + let (as_arm, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Breaks("item:daodejing:3"), None)); + let a = as_arm[0].arm; + assert!(f.chrf > 0.0 && f.meaning > 0.0 && f.composite > 0.0); + assert!((f.chrf - (l.recorded.chrf - a.chrf).abs()).abs() < 1e-9, "{}", l.line()); + assert!((f.composite - (l.recorded.composite - a.composite).abs()).abs() < 1e-9); + assert!(l.line().contains(&format!("| floor chrf={:.2}", f.chrf)), "{}", l.line()); + assert_eq!(l.unusable, 0, "unusable counts the arm, not the floor"); + assert_eq!(l.floor_unusable, 1, "the floor arm's refusal is counted apart"); + assert!(l.line().contains(" unusable=0 recorded "), "{}", l.line()); + assert!(l.line().ends_with(" unusable=1"), "the floor's refusal is on the line: {}", l.line()); + // Without a floor arm there is none to count. + assert_eq!(as_arm[0].floor_unusable, 0); +} + +#[test] +fn relower_skips_and_counts_an_item_the_round_refused_before_admission() { + // Validation item 3 is not an encodable draft in the round: it has no packet. + let (dao, _) = recorded_dao(|answers| { + answers.insert("item:daodejing:3".into(), "not a draft".into()); + }); + let (mut rec, mut arm) = (Lowerer::new(blind), Lowerer::new(blind)); + let (langs, _) = relowered(relower(&dao, &mut rec, &mut arm, None)); + let l = &langs[0]; + assert_eq!((l.n, l.skipped, l.unusable), (2, 1, 0), "{}", l.line()); + assert!(l.line().starts_with("relower en-US n=2 skipped=1 unusable=0 "), "{}", l.line()); + assert_eq!(l.recorded, l.arm, "both readings are over the same two items"); + assert!(arm.seen.iter().all(|(_, key, _)| key != "item:daodejing:3"), "nothing is asked for it"); +} + +#[test] +fn relower_skips_and_counts_an_item_whose_lowering_prompt_the_round_refused() { + // ja-JP item 12 cites an entity with no ja form: the round admitted it and refused it + // at Lower. Item 13 cites none and lowers. + let mut dao = bilingual(None); + let mut model = bilingual_fake(); + model.answers.insert("item:daodejing:12".into(), citing_laozi(2)); + model.answers.insert("item:daodejing:13".into(), json(&draft_of(3))); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + assert_eq!(r.language("ja-JP").unwrap().validation_refused.lower, 1); + let (mut rec, mut arm) = (Lowerer::new(bi_blind), Lowerer::new(bi_blind)); + let (langs, _) = relowered(relower(&dao, &mut rec, &mut arm, None)); + let ja = langs.iter().find(|l| l.language == "ja-JP").unwrap(); + assert_eq!((ja.n, ja.skipped), (1, 1), "{}", ja.line()); + assert_eq!(ja.recorded, ja.arm); + let en = langs.iter().find(|l| l.language == "en-US").unwrap(); + assert_eq!((en.n, en.skipped), (2, 0), "{}", en.line()); + assert!(arm.seen.iter().all(|(_, key, _)| key != "item:daodejing:12")); +} + +/// The ja-JP item's own text, else the English one: `f` for en-US. +fn bi(key: &str, text: &str, f: fn(&str, &str) -> String) -> String { + match JA_TEXTS.iter().find(|(k, ..)| *k == key) { + Some((_, ja, _)) => ja.to_string(), + None => f(key, text), + } +} + +fn bi_blind(key: &str, text: &str) -> String { + bi(key, text, blind) +} + +fn bi_reader(key: &str, text: &str) -> String { + bi(key, text, reader) +} + +/// Fine in English, nonsense in ja-JP. +fn bi_ja_lost(key: &str, text: &str) -> String { + match JA_TEXTS.iter().any(|(k, ..)| *k == key) { + true => "zzzz".into(), + false => blind(key, text), + } +} + +#[test] +fn relower_decides_over_the_whole_panel_not_its_first_language() { + use weksa::run::decide_panel; + let mut dao = bilingual(LAOZI_JA); + recorded(round(&mut dao.run, &mut bilingual_fake(), &Scorer::new(Bag)).unwrap()); + // en-US gains (the arm drops the stance the recorded lowering repeats); ja-JP is lost. + let (langs, verdict) = relowered(relower( + &dao, + &mut Lowerer::new(bi_reader), + &mut Lowerer::new(bi_ja_lost), + Some(&mut Lowerer::new(bi_reader)), + )); + assert_eq!( + langs.iter().map(|l| (l.language.as_str(), l.n)).collect::>(), + [("en-US", 2), ("ja-JP", 2)], + "panel order, Validation only" + ); + assert!(langs[0].arm.composite > langs[0].recorded.composite, "en-US gains"); + assert!(langs[1].arm.composite < langs[1].recorded.composite, "ja-JP is lost"); + let panel: Vec<_> = langs.iter().map(|l| (l.recorded, l.arm, l.floor.unwrap())).collect(); + assert_eq!(verdict, Some(decide_panel(&panel))); + assert!(matches!(verdict, Some(Verdict::Rejected(Why::Regressed(_)))), "{verdict:?}"); + // The first language alone would have accepted. + assert_eq!(decide_panel(&panel[..1]), Verdict::Accepted); +} + +#[test] +fn relower_refuses_a_cycle_round_and_a_missing_recorded_answer() { + use weksa::model::RecordedAnswers; + let mut dao = dao(); + two_rounds(&mut dao); + run_cycle(&mut dao, &mut cycle_fake(ONE_CONCEPT)); + let candidate = dao.run.rounds().unwrap().into_iter().find(|r| r.cycle().is_some()).unwrap().n(); + let mut arm = Lowerer::new(reader); + let r = dao.run.relower(candidate, "arm", &mut Lowerer::new(reader), &mut arm, None, &Scorer::new(Bag)); + assert!(matches!(r, Err(RunError::Malformed("round")))); + let r = dao.run.relower(99, "arm", &mut Lowerer::new(reader), &mut arm, None, &Scorer::new(Bag)); + assert!(matches!(r, Err(RunError::Malformed("round")))); + assert!(arm.seen.is_empty()); + + let (dao, _) = recorded_dao(|_| {}); + let mut empty = RecordedAnswers::new(dao.root.join("round-1/nowhere"), "fake-model".into()); + let mut arm = Lowerer::new(reader); + let r = relower(&dao, &mut empty, &mut arm, None); + assert!(matches!(r, Err(RunError::Unrecorded))); + assert!(arm.seen.is_empty(), "nothing is asked of the arm before the round's answer exists"); +} + +#[test] +fn relower_floor_arm_with_an_unanswered_prompt_is_refused() { + use weksa::model::RecordedAnswers; + let (dao, _) = recorded_dao(|_| {}); + let mut floor = RecordedAnswers::new(dao.root.join("relower-1-floor"), "fake-model".into()); + let r = relower(&dao, &mut Lowerer::new(reader), &mut Lowerer::new(reader), Some(&mut floor)); + assert!(matches!(r, Err(RunError::FloorUnanswered))); + assert!(!dao.root.join("relower-1-floor").exists(), "a floor arm is never written to"); +} + +#[test] +fn relower_checks_the_embedder_not_the_model() { + struct Named(Lowerer); + impl ModelPort for Named { + fn model(&self) -> &str { + "another-model" + } + fn answer(&mut self, p: &Prompt) -> Result { + self.0.answer(p) + } + } + let (dao, _) = recorded_dao(|_| {}); + let mut other = Named(Lowerer::new(blind)); + relowered(relower(&dao, &mut Lowerer::new(reader), &mut other, None)); + let mut arm = Lowerer::new(reader); + let r = dao.run.relower(1, "arm", &mut Lowerer::new(reader), &mut arm, None, &Scorer::new(OtherEmbedder)); + assert!(matches!(r, Err(RunError::Drift(DriftOf::Embedder)))); + assert!(arm.seen.is_empty()); +} + +/// Every file under `dir` with its bytes, by relative path. +fn tree(dir: &Path) -> std::collections::BTreeMap> { + let mut out = std::collections::BTreeMap::new(); + let mut stack = vec![dir.to_path_buf()]; + while let Some(d) = stack.pop() { + for e in std::fs::read_dir(d).unwrap() { + let p = e.unwrap().path(); + if p.is_dir() { + stack.push(p); + } else { + out.insert( + p.strip_prefix(dir).unwrap().to_string_lossy().into_owned(), + std::fs::read(&p).unwrap(), + ); + } + } + } + out +} + +#[test] +fn relower_records_nothing() { + use weksa::model::RecordedAnswers; + let (dao, _) = recorded_dao(|_| {}); + let before = tree(&dao.root); + let mut recorded = RecordedAnswers::new(dao.root.join("round-1"), "fake-model".into()); + // An unanswered arm writes its prompts in its own directory only. + let mut files = dao.run.answer_files(dao.run.relower_dir(1, "arm")).unwrap(); + // The recorded answers are files here, as the relay leaves them. + let r = relower(&dao, &mut recorded, &mut files, None); + assert!(matches!(r, Err(RunError::Unrecorded)), "no round answer files yet"); + let after = tree(&dao.root); + for (path, bytes) in &before { + assert_eq!(after.get(path), Some(bytes), "{path} changed"); + } + assert!( + after.keys().all(|p| before.contains_key(p) || p.starts_with("relower-1-arm/")), + "only relower-1-arm gains files" + ); + + // With the round's answers present, the arm is asked and its prompts land in relower-1-arm. + let (dao, r) = recorded_dao(|_| {}); + for call in &r.calls()[4..] { + std::fs::write( + dao.root.join(format!("round-1/{}.answer", call.prompt)), + r#"{"text":"The highest excellence is like the water."}"#, + ) + .unwrap(); + } + let before = tree(&dao.root); + let mut recorded = RecordedAnswers::new(dao.root.join("round-1"), "fake-model".into()); + let mut files = dao.run.answer_files(dao.run.relower_dir(1, "arm")).unwrap(); + let RelowerOutcome::Unanswered { prompts } = relower(&dao, &mut recorded, &mut files, None).unwrap() else { + panic!("the arm is unanswered") + }; + assert_eq!(prompts.len(), 3); + let after = tree(&dao.root); + for (path, bytes) in &before { + assert_eq!(after.get(path), Some(bytes), "{path} changed"); + } + let added: Vec<&String> = after.keys().filter(|p| !before.contains_key(*p)).collect(); + assert!(added.iter().all(|p| p.starts_with("relower-1-arm/")), "{added:?}"); + assert_eq!(added.iter().filter(|p| p.ends_with(".prompt")).count(), 3); +} + +#[test] +fn relower_arm_prompt_holding_test_text_is_withheld() { + // A Test sentence that a Validation lowering prompt holds: a line of the + // lowering instructions, which every lowering prompt carries. + let line = LOWER_INSTRUCTIONS.lines().find(|l| l.trim().chars().count() > 30).unwrap().trim().to_string(); + let (dao, _) = recorded_dao_in( + |corpus, entities| { + let mut held = corpus.get("item:daodejing:4").unwrap().unwrap(); + held.id = "item:held:9".into(); + held.text = line.clone(); + corpus.add(held, entities).unwrap(); + }, + |_| {}, + ); + let mut files = dao.run.answer_files(dao.run.relower_dir(1, "arm")).unwrap(); + let r = relower(&dao, &mut Lowerer::new(reader), &mut files, None); + assert!(matches!(r, Err(RunError::Withheld)), "{:?}", r.map(|_| ())); + let dir = dao.run.relower_dir(1, "arm"); + let prompts = std::fs::read_dir(&dir) + .map(|d| d.filter(|e| e.as_ref().unwrap().file_name().to_string_lossy().ends_with(".prompt")).count()) + .unwrap_or(0); + assert_eq!(prompts, 0, "no prompt file is written"); +} diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 1aa7299..4a78abe 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -355,6 +355,31 @@ checkpoint the test-split composite is within the noise floor of validation. The test split is read only at checkpoints. Self then records the freeze, and the training set is the whole corpus re-encoded under the frozen revision. +## Distil measure (`distil-slice`) + +A lowerer is compared on round n's Validation packets with the prompts +byte-identical to the round's, by `weksa loop relower ROOT --round N --arm ARM +--lowerer LABEL [--floor ARM2]`. Every arm is answered through the one prompt +door (`RunStores::answer_files`, the Test-text guard); a tuned model is a script +that reads `.prompt` and writes `.answer` in +`/relower--`, not a new adapter. The recorded reading is the +round's own lowering, read over the same items as the arm. An item the round +refused before admission (no packet) or at the lowering prompt has no prompt to +ask: it is skipped, so the readings exclude it, and it is counted as `skipped=` +on the line; `n` is the items read, so `recorded` can differ from the round's +own Validation reading, which scores those items 0. An arm answer that is not a +usable lowering scores 0 and counts in `unusable=`; the same for the floor arm, +counted after `floor ... unusable=`. The floor is the spread of the recorded +lowering and a second prompted arm (lowering-only noise, no encoder noise). The +verdict is `decide_panel` over (recorded, arm, floor): Accepted is the rule's +name for a gain past the floor, Rejected(Regressed) for a loss past it, +Rejected(NoGain) for neither. None is yet a finding about the arm. The rule is calibration-pending: a one-pair floor is noisy +(a simulated prompted repeat reads NoGain only about 7-15% of the time, Regressed +44-60%, Accepted 33-41%), so NoGain is not yet evidence of non-inferiority and +is not yet the bar for cut 13; the calibration arms (a prompted repeat, a +control) decide what the floor must be before the verdict is read as a ruling. +Training pairs are Train only, at a frozen revision; the export is 12b. + ## Lowerer first cut (`golf-loop-lowerer-cut`) Authority map for the smallest lowerer, written before its code (Imagination @@ -436,5 +461,5 @@ One line per cut; specs exist for the cuts about to run (2d and 4, siblings on ` 9. `ja-panel`: ja-JP into the loop's panel, in two cuts on `e9cbd9b`. 9a `ja-panel-loop` (spec r1): language is the item's and the panel the run's: `Prompt::lower` takes the item's language and prints it on a language line (no prompt names English), entity forms are read in that language, `RunRecord` pins the panel (`weksa loop open --panel`), offers, mint and recall read only panel items, round means, floors and readings are per panel language, the accept rule runs `decide` per language, the recall split works on unspaced scripts, and `weksa corpus add` has no default language. 9b `ja-panel-corpus` (spec r1, after 9a): the ja-JP slice of `data/original`, model-written Japanese Train and Validation passages and eight Aozora Bunko public-domain Test sentences with full provenance, and their entities. Cut 5 is not a prerequisite: neither language lowers with ontology entries until cut 5 gives both theirs. 10. `conlang-lexicon`: the first conlang's lexicon placed at concept embeddings, its morphology data, its ontology entries and the gloss reader. Waits on 3, 5 and question `first-conlang`. 11. `convergence-freeze`: the convergence test, `Checkpoint`'s one door (built from the convergence window over the persisted round records), the freeze, the re-encode into training pairs. Waits on 7, 9 and 10. -12. `distil`: fine-tune a lowerer on the frozen training set. Waits on 11 and a host question asked then. +12. `distil`, three cuts. 12a `distil-slice` (relower and the prompted baseline; spec r1). 12b `distil-export`: Train-only prompt/completion JSONL at a frozen revision, Test-text guarded (follow-up `distil-export-at-freeze`); waits on 11. 12c `distil-train`: waits on 12b and the host question (follow-up `distil-host-question-owed`). 13. `live-lowering`: agents emit packets with stance; Weksa reads speaker state from persona state (follow-up `persona-state-speech-community`) and lowers for the reader. Waits on 11. From 353dd2138f33ee21a8a396072ecfbfc04e09d4de Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 03:01:52 +0000 Subject: [PATCH 03/28] relower: paired per-item noise floor over >=3 prompted repeats; panel gain clears the noise of the sum (#70) --- crates/weksa/src/main.rs | 23 ++- crates/weksa/src/run.rs | 200 +++++++++++++++++------ crates/weksa/tests/cli.rs | 18 +- crates/weksa/tests/golf_loop.rs | 82 ++++++---- docs/targets/interlingua-lowering-map.md | 34 ++-- 5 files changed, 251 insertions(+), 106 deletions(-) diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 9ba2135..fc57236 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -154,8 +154,9 @@ enum LoopCommand { ollama_url: String, }, /// Re-lower round ROUND's Validation packets through arm ARM, score them - /// against the items, and, with --floor ARM, decide against that second arm - /// by the panel accept rule. Prompts and answers live in + /// against the items, and, with --floor ARM given two or more times (prompted + /// repeats of the lowering), decide against the paired noise floor of those + /// repeats by the panel accept rule. Prompts and answers live in /// `/relower--`; the command exits 3 until the arm is /// answered. --lowerer names the arm's model in the printed line. A /// diagnostic, never a gate; records nothing; Test and Train are never @@ -169,7 +170,7 @@ enum LoopCommand { #[arg(long)] lowerer: String, #[arg(long)] - floor: Option, + floor: Vec, #[arg(long, default_value = "http://127.0.0.1:11434")] ollama_url: String, }, @@ -1147,7 +1148,10 @@ fn main() -> Result<()> { (1..=32).contains(&l.len()) && l.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') }; - if !label(&arm) || floor.as_deref().is_some_and(|f| !label(f) || f == arm) { + if !label(&arm) + || floor.iter().any(|f| !label(f) || *f == arm) + || floor.iter().enumerate().any(|(i, f)| floor[..i].contains(f)) + { anyhow::bail!("arm and floor labels must match [a-z0-9-]{{1,32}} and differ"); } let run = RunStores::open(&root)?; @@ -1156,16 +1160,19 @@ fn main() -> Result<()> { run.record().model.clone(), ); let mut files = run.answer_files(run.relower_dir(round, &arm))?; - let mut floor_port = floor.as_deref().map(|f| { - RecordedAnswers::new(run.relower_dir(round, f), run.record().model.clone()) - }); + let mut floor_ports: Vec = floor + .iter() + .map(|f| RecordedAnswers::new(run.relower_dir(round, f), run.record().model.clone())) + .collect(); + let mut floor_dyn: Vec<&mut dyn ModelPort> = + floor_ports.iter_mut().map(|p| p as &mut dyn ModelPort).collect(); let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder)); match run.relower( round, &arm, &mut recorded, &mut files, - floor_port.as_mut().map(|p| p as &mut dyn ModelPort), + &mut floor_dyn, &scorer, )? { RelowerOutcome::Unanswered { prompts } => { diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 1dc4780..dd6bc4a 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -498,6 +498,8 @@ pub enum RunError { /// The floor arm of a relower holds no answer to a prompt; it is a /// reference, never written to, and must be answered whole first. FloorUnanswered, + /// A relower was given fewer floor arms than [`FLOOR_ARMS_MIN`]. + FloorArms, } impl fmt::Display for RunError { @@ -526,6 +528,7 @@ impl fmt::Display for RunError { RunError::SpeakerState => { f.write_str("a scene does not read against the run's inventory") } + RunError::FloorArms => f.write_str("a floor needs three or more floor arms; fewer measure no noise"), RunError::FloorUnanswered => { f.write_str("the floor arm holds no answer to a lowering prompt; answer it whole first") } @@ -1586,13 +1589,14 @@ pub struct RelowerReading { pub skipped: u32, /// Arm answers that were not a usable lowering; each scored 0. pub unusable: u32, - /// Floor-arm answers that were not a usable lowering; each scored 0, which - /// widens `floor`. Zero without a floor arm. + /// Floor-arm answers that were not a usable lowering, over every floor arm; + /// each scored 0, which widens `floor`. Zero without floor arms. pub floor_unusable: u32, /// The round's own lowering, read against the items. pub recorded: Reading, pub arm: Reading, - /// `spread(recorded, floor arm)`, when a floor arm was given. + /// [`paired_floor`] of the recorded lowering against the floor arms, when + /// floor arms were given. pub floor: Option, } @@ -1637,9 +1641,10 @@ impl RunStores { /// distil decision, whether a tuned lowerer is worth keeping against the /// prompted one (calibration-pending: see the map). Asks `arm` the byte-identical lowering prompts of round `n`'s live /// Validation items, reads the arm and the round's own lowering - /// (`recorded`) against the items, and, given a `floor` arm (a second - /// prompted lowering, so the floor is lowering-only noise), decides by - /// [`decide_panel`] over (recorded, arm, spread(recorded, floor)). Only + /// (`recorded`) against the items, and, given `floors` (three or more + /// prompted repeats of the recorded lowering, so the floor is lowering-only + /// noise), decides by [`decide_panel`] over (recorded, arm, + /// [`paired_floor`]). Only /// the model of `arm` is unpinned; the embedder is checked. An arm answer /// that is not a usable lowering scores 0 and is counted in `unusable`. /// Records nothing; `arm_label` names only where the scratch packet store @@ -1650,7 +1655,7 @@ impl RunStores { arm_label: &str, recorded: &mut dyn ModelPort, arm: &mut dyn ModelPort, - mut floor: Option<&mut dyn ModelPort>, + floors: &mut [&mut dyn ModelPort], scorer: &Scorer, ) -> Result { let store = |_| RunError::Store; @@ -1666,9 +1671,12 @@ impl RunStores { } let mut scratch = PacketStore::open(&dir.join("base.cc")).map_err(store)?; - // Per language, per item: the item text, then the recorded, arm and floor - // lowerings (`None` when unusable). - type Row = (String, Option, Option, Option); + if (1..FLOOR_ARMS_MIN).contains(&floors.len()) { + return Err(RunError::FloorArms); + } + // Per language, per item: the item text, then the recorded and arm + // lowerings and one per floor arm (`None` when unusable). + type Row = (String, Option, Option, Vec>); let mut waiting: Vec = Vec::new(); let mut floor_waiting = false; let mut rows: Vec> = Vec::new(); @@ -1690,16 +1698,16 @@ impl RunStores { Asked::Waiting => return Err(RunError::Unrecorded), other => text(other), }; - let floor_text = match floor.as_deref_mut() { - Some(f) => match asked(f, &prompt)? { + let mut floor_texts = Vec::new(); + for f in floors.iter_mut() { + floor_texts.push(match asked(&mut **f, &prompt)? { Asked::Waiting => { floor_waiting = true; None } other => text(other), - }, - None => None, - }; + }); + } let arm_text = match asked(arm, &prompt)? { Asked::Waiting => { let d = prompt.digest(); @@ -1710,7 +1718,7 @@ impl RunStores { } other => text(other), }; - language_rows.push((item.text.clone(), recorded_text, arm_text, floor_text)); + language_rows.push((item.text.clone(), recorded_text, arm_text, floor_texts)); } rows.push(language_rows); } @@ -1723,8 +1731,8 @@ impl RunStores { // One score call: each usable lowering against its item, in row order. let mut inputs: Vec = Vec::new(); - for (reference, recorded, arm, floor_text) in rows.iter().flatten() { - for output in [recorded, arm, floor_text].into_iter().flatten() { + for (reference, recorded, arm, floor_texts) in rows.iter().flatten() { + for output in [recorded, arm].into_iter().chain(floor_texts).flatten() { inputs.push(ScoreInput { reference, output }); } } @@ -1751,7 +1759,8 @@ impl RunStores { let mut languages = Vec::new(); let live = self.corpus.all().map_err(store)?; for ((language, _, _), language_rows) in held.iter().zip(&rows) { - let (mut rec, mut arm_scores, mut floor_scores) = (Vec::new(), Vec::new(), Vec::new()); + let (mut rec, mut arm_scores) = (Vec::new(), Vec::new()); + let mut floor_scores: Vec> = vec![Vec::new(); floors.len()]; let (mut unusable, mut floor_unusable) = (0u32, 0u32); for (_, r, a, f) in language_rows { rec.push(next(r)?); @@ -1759,9 +1768,9 @@ impl RunStores { if a.is_none() { unusable += 1; } - if floor.is_some() { - floor_scores.push(next(f)?); - if f.is_none() { + for (scores, t) in floor_scores.iter_mut().zip(f) { + scores.push(next(t)?); + if t.is_none() { floor_unusable += 1; } } @@ -1781,14 +1790,10 @@ impl RunStores { floor_unusable, recorded: recorded_reading, arm: reading_of(&arm_scores)?, - floor: if floor.is_some() { - Some(spread(&recorded_reading, &reading_of(&floor_scores)?)) - } else { - None - }, + floor: if floors.is_empty() { None } else { Some(paired_floor(&rec, &floor_scores)) }, }); } - let verdict = if floor.is_some() { + let verdict = if !floors.is_empty() { let panel: Vec<(Reading, Reading, Reading)> = languages .iter() .filter_map(|l| Some((l.recorded, l.arm, l.floor?))) @@ -2261,7 +2266,11 @@ pub fn decide(base: &Reading, candidate: &Reading, floor: &Reading) -> Verdict { /// decides whether a language regressed (any regression rejects; the first in /// panel order names the metric), and the gain test is [`decide`]'s for one /// language and, for two or more, the panel's: the sum over languages of the -/// composite deltas (billionths) must exceed one [`LADDER_STEP`]. A language +/// composite deltas (billionths) must exceed the larger of one [`LADDER_STEP`] +/// and the noise of that sum, the root of the languages' composite floors +/// squared (independent languages; a one-language panel's noise is its floor, +/// as in [`decide`]). Without that noise term a prompted repeat of the base +/// lowering read Accepted in 40% of the calibration trials on `distil-base` at `FLOOR_Z` 3. A language /// may lose up to its own noise floor if the panel nets more than a step; the /// rule gives up strict Pareto non-regression, which on noisy per-language /// readings ratchets every base to a favourable draw and strangles the loop @@ -2289,7 +2298,10 @@ pub fn decide_panel(rows: &[(Reading, Reading, Reading)]) -> Verdict { .iter() .map(|(base, candidate, _)| nano(candidate.composite - base.composite)) .sum(); - if net > nano(LADDER_STEP) { + // The noise of a sum of independent languages' differences: the root of the + // sum of their floors squared. + let noise = rows.iter().map(|(_, _, floor)| floor.composite.powi(2)).sum::().sqrt(); + if net > nano(noise.max(LADDER_STEP)) { Verdict::Accepted } else { Verdict::Rejected(Why::NoGain) @@ -2488,6 +2500,50 @@ fn floor_of( .collect() } +/// How many standard errors of a prompted-vs-prompted mean difference the +/// relower floor is wide, and the fewest floor arms (prompted repeats of the +/// recorded lowering) it is measured over. Calibrated on `runs/distil-base` +/// (eight prompted draws of 14 en-US and 14 ja-JP Validation items, 28 +/// unordered pairs): with three floor arms a prompted repeat reads NoGain in +/// 92% of the trials (bootstrap 95% over draws 80 to 100%), Regressed 8% and +/// Accepted 0%; a "Yes." arm and a mild degradation (one detail blurred per +/// utterance) read Regressed in 100%, and a subtler one in 75% (50 to 100%): at +/// 14 items a degradation under about three standard errors cannot be told from +/// noise. `FLOOR_Z` was chosen on that data, so the rates are in-sample. +pub const FLOOR_Z: f64 = 3.0; +pub const FLOOR_ARMS_MIN: usize = 3; + +/// The lowering-only noise floor of a relower: `FLOOR_Z` standard errors of the +/// mean per-item difference between the recorded lowering and a repeat of it, +/// metric by metric. Each floor arm gives one paired sample of per-item +/// differences; their variances are pooled over the arms, so the floor rests on +/// `arms * (n - 1)` degrees of freedom instead of the one coincidental +/// difference of means a two-run spread is (which reads 0 whenever the two +/// means happen to meet, and then a one-step dip regresses). Paired per item, +/// after Koehn (2004). NaN (fail closed in [`decide`]) when there are fewer +/// than two items or no arm (0/0), and when an arm is ragged. +pub fn paired_floor(recorded: &[ItemScore], arms: &[Vec]) -> Reading { + let n = recorded.len(); + if arms.iter().any(|a| a.len() != n) { + return Reading { chrf: f64::NAN, meaning: f64::NAN, composite: f64::NAN }; + } + let pooled = |of: &dyn Fn(&ItemScore) -> f64| { + let variance = |arm: &Vec| { + let d: Vec = recorded.iter().zip(arm).map(|(r, a)| of(r) - of(a)).collect(); + let mean = d.iter().sum::() / n as f64; + d.iter().map(|x| (x - mean).powi(2)).sum::() / (n - 1) as f64 + }; + let v = arms.iter().map(variance).sum::() / arms.len() as f64; + FLOOR_Z * (v / n as f64).sqrt() + }; + let meaning = |s: &ItemScore| s.meaning.unwrap_or(0.0); + Reading { + chrf: pooled(&|s| s.chrf), + meaning: pooled(&meaning), + composite: pooled(&|s| (s.chrf / 100.0 + meaning(s)) / 2.0), + } +} + /// The absolute difference of two readings, metric by metric. fn spread(a: &Reading, b: &Reading) -> Reading { Reading { @@ -2869,6 +2925,52 @@ mod tests { assert_eq!(decide(&base, &gain(0.506), &floor), rejected(Why::NoGain)); } + #[test] + fn paired_floor_is_z_standard_errors_of_the_paired_difference() { + let item = |chrf: f64, meaning: f64| ItemScore { chrf, meaning: Some(meaning) }; + let recorded = [item(60.0, 0.8), item(40.0, 0.6), item(50.0, 0.7), item(30.0, 0.5)]; + // An arm that differs by a constant has no spread of differences: a + // shifted mean is the thing the floor is read against, not noise. + let shifted: Vec = recorded.iter().map(|s| item(s.chrf - 5.0, s.meaning.unwrap() - 0.1)).collect(); + let f = paired_floor(&recorded, &[shifted.clone(), shifted.clone(), shifted]); + assert!(f.chrf.abs() < 1e-9 && f.meaning.abs() < 1e-9 && f.composite.abs() < 1e-9, "{f:?}"); + // Differences 2, -2, 2, -2 on chrF: sample variance 16/3, standard error + // sqrt(16/3/4), FLOOR_Z of it; arms pool, so a second equal arm changes nothing. + let wobble_arm = |recorded: &[ItemScore]| -> Vec { + recorded + .iter() + .enumerate() + .map(|(i, s)| item(s.chrf - if i % 2 == 0 { 2.0 } else { -2.0 }, s.meaning.unwrap())) + .collect() + }; + let wobble = wobble_arm(&recorded); + let want = FLOOR_Z * (16.0_f64 / 3.0 / 4.0).sqrt(); + let one = paired_floor(&recorded, std::slice::from_ref(&wobble)); + let two = paired_floor(&recorded, &[wobble.clone(), wobble.clone()]); + assert!((one.chrf - want).abs() < 1e-9 && (two.chrf - want).abs() < 1e-9, "{one:?} {two:?}"); + assert!(one.meaning.abs() < 1e-9); + // Arms pool by the mean of their variances: one wobbling arm and one + // exact repeat give the root of half the variance. + let exact = recorded.to_vec(); + let mixed = paired_floor(&recorded, &[wobble.clone(), exact]); + assert!((mixed.chrf - FLOOR_Z * (16.0_f64 / 3.0 / 2.0 / 4.0).sqrt()).abs() < 1e-9, "{mixed:?}"); + // The calibrated constants, pinned apart from the formula. + assert_eq!((FLOOR_Z, FLOOR_ARMS_MIN), (3.0, 3)); + // Under two items, no arms or a ragged arm there is no floor: NaN fails + // closed in `decide`. + for f in [ + paired_floor(&recorded[..1], &[vec![item(1.0, 1.0)]]), + paired_floor(&recorded, &[]), + paired_floor(&recorded, &[vec![item(1.0, 1.0)]]), + ] { + assert!(f.composite.is_nan()); + assert_eq!( + decide(&reading(50.0, 0.5, 0.5), &reading(50.0, 0.5, 0.9), &f), + Verdict::Rejected(Why::Regressed(Metric::Composite)) + ); + } + } + #[test] fn decide_panel_table() { let zero = reading(0.0, 0.0, 0.0); @@ -2922,9 +3024,8 @@ mod tests { assert_eq!(net(LADDER_STEP + 6e-10), Verdict::Accepted); assert_eq!(net(LADDER_STEP + 2e-9), Verdict::Accepted); // Liveness: no language need pass its own threshold. Two languages each - // gaining 0.6 of a step at zero floor accept; so does one gaining two - // steps under a 0.02 floor (its own threshold) with the others flat; so - // does a gain with another language falling inside its own floor. + // gaining 0.6 of a step at zero floor accept; so does a gain past the + // noise of the sum with another language falling inside its own floor. let each = 0.6 * LADDER_STEP; assert_eq!( decide_panel(&[ @@ -2933,27 +3034,32 @@ mod tests { ]), Verdict::Accepted ); + // The sum must also clear its own noise: the root of the floors squared + // (0.02 and 0.02 give 0.0283), not just a step. A repeat of the base + // lowering gains that much by chance. let big_floor = reading(0.0, 0.0, 0.02); - assert_eq!( - decide_panel(&[(base, reading(50.0, 0.5, 0.5 + 2.0 * LADDER_STEP), big_floor), (base, flat, big_floor)]), - Verdict::Accepted - ); + let sum_of = |gain: f64| { + decide_panel(&[(base, reading(50.0, 0.5, 0.5 + gain), big_floor), (base, flat, big_floor)]) + }; + assert_eq!(sum_of(0.02), rejected(Why::NoGain), "past a step, under the noise of the sum"); + assert_eq!(sum_of(0.028), rejected(Why::NoGain)); + assert_eq!(sum_of(0.029), Verdict::Accepted); let own_floor = reading(0.0, 0.0, 0.004); + assert_eq!( + decide_panel(&[ + (base, reading(50.0, 0.5, 0.55), reading(0.0, 0.0, 0.03)), + (base, reading(50.0, 0.5, 0.497), own_floor) + ]), + Verdict::Accepted, + "a fall inside the language's floor, net 0.047 over a noise of 0.0303" + ); assert_eq!( decide_panel(&[ (base, reading(50.0, 0.5, 0.51), reading(0.0, 0.0, 0.03)), (base, reading(50.0, 0.5, 0.497), own_floor) ]), - Verdict::Accepted, - "a fall inside the language's floor, net 0.007" - ); - // The same fall with the net exactly a step rejects. - assert_eq!( - decide_panel(&[ - (base, reading(50.0, 0.5, 0.508), reading(0.0, 0.0, 0.03)), - (base, reading(50.0, 0.5, 0.497), own_floor) - ]), - rejected(Why::NoGain) + rejected(Why::NoGain), + "net 0.007 is under the noise" ); // The sum is over the deltas, with each language's fall counted. assert_eq!( diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 95a147a..e3abfc7 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -1833,12 +1833,19 @@ fn cli_loop_relower_exits_3_until_answered_and_refuses_bad_labels() { assert_eq!(o.status.code(), Some(1), "{}", out(&o)); assert!(!root.join("relower-1-second").exists()); - // Answered floor: a verdict line. - std::fs::create_dir(root.join("relower-1-second")).unwrap(); - for digest in &digests { - std::fs::write(root.join(format!("relower-1-second/{digest}.answer")), said).unwrap(); - } + // Fewer than three floor arms is refused: one pair measures no noise. let o = relower(&["--arm", "tuned", "--floor", "second"]); + assert_eq!(o.status.code(), Some(1), "{}", out(&o)); + assert!(out(&o).contains("three or more floor arms"), "{}", out(&o)); + + // Answered floors: a verdict line. + for label in ["second", "third", "fourth"] { + std::fs::create_dir_all(root.join(format!("relower-1-{label}"))).unwrap(); + for digest in &digests { + std::fs::write(root.join(format!("relower-1-{label}/{digest}.answer")), said).unwrap(); + } + } + let o = relower(&["--arm", "tuned", "--floor", "second", "--floor", "third", "--floor", "fourth"]); assert!(o.status.success(), "{}", out(&o)); let text = out(&o); assert!(text.contains("| floor chrf=0.00") && text.trim_end().ends_with("verdict Rejected(NoGain)"), "{text}"); @@ -1852,6 +1859,7 @@ fn cli_loop_relower_exits_3_until_answered_and_refuses_bad_labels() { vec!["--arm", &long], vec!["--arm", "../x"], vec!["--arm", "tuned", "--floor", "Bad"], + vec!["--arm", "tuned", "--floor", "b", "--floor", "b", "--floor", "c"], ] { let o = relower(&args); assert_eq!(o.status.code(), Some(1), "{args:?}: {}", out(&o)); diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 179d0ea..2d0320a 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -5335,14 +5335,14 @@ fn the_cycles_candidate_round_stops_on_a_withheld_lowering_prompt() { use weksa::run::{RelowerOutcome, RelowerReading}; -/// `relower` of round 1 with the Bag scorer; `floor` is an optional second arm. +/// `relower` of round 1 with the Bag scorer; `floors` are the floor arms (none, or three or more). fn relower( dao: &Dao, recorded: &mut dyn ModelPort, arm: &mut dyn ModelPort, - floor: Option<&mut dyn ModelPort>, + floors: &mut [&mut dyn ModelPort], ) -> Result { - dao.run.relower(1, "arm", recorded, arm, floor, &Scorer::new(Bag)) + dao.run.relower(1, "arm", recorded, arm, floors, &Scorer::new(Bag)) } fn relowered(outcome: Result) -> (Vec, Option) { @@ -5360,7 +5360,7 @@ fn relower_asks_only_validation_lowering_prompts_of_the_round() { // which digest is which item's. let (mut every, mut rec, mut arm) = (Lowerer::new(reader), Lowerer::new(reader), Lowerer::new(reader)); read(ablate(&dao, &mut every, &mut Lowerer::new(reader)).unwrap()); - relowered(relower(&dao, &mut rec, &mut arm, None)); + relowered(relower(&dao, &mut rec, &mut arm, &mut [])); let of = |keys: &[&str]| -> Vec { every .seen @@ -5400,7 +5400,7 @@ fn relower_reads_recorded_and_arm_against_the_item() { &dao, &mut Lowerer::new(reader), &mut Lowerer::new(blind), - None, + &mut [], )); let l = &langs[0]; assert_eq!((l.language.as_str(), l.n, l.unusable), ("en-US", 3, 0)); @@ -5408,7 +5408,7 @@ fn relower_reads_recorded_and_arm_against_the_item() { assert!(l.arm.composite > l.recorded.composite); assert!(l.floor.is_none() && verdict.is_none(), "no floor arm, no verdict"); // An arm equal to the recorded answers reads equal. - let (langs, _) = relowered(relower(&dao, &mut Lowerer::new(reader), &mut Lowerer::new(reader), None)); + let (langs, _) = relowered(relower(&dao, &mut Lowerer::new(reader), &mut Lowerer::new(reader), &mut [])); assert_eq!(langs[0].recorded, langs[0].arm); } @@ -5432,11 +5432,11 @@ impl ModelPort for Breaks { fn relower_counts_an_unusable_arm_answer_as_zero() { let (dao, _) = recorded_dao(|_| {}); let keys = ["item:daodejing:2", "item:daodejing:3", WRITTEN]; - let (whole, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), None)); + let (whole, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), &mut [])); let whole = whole[0].arm; let (mut chrf, mut meaning) = (0.0, 0.0); for key in keys { - let (langs, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Breaks(key), None)); + let (langs, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Breaks(key), &mut [])); let l = &langs[0]; assert_eq!((l.n, l.unusable), (3, 1), "{}", l.line()); assert!(l.arm.chrf < whole.chrf && l.arm.meaning < whole.meaning, "{}", l.line()); @@ -5452,8 +5452,8 @@ fn relower_counts_an_unusable_arm_answer_as_zero() { #[test] fn relower_scores_an_unusable_recorded_answer_as_zero_without_counting_it_against_the_arm() { let (dao, _) = recorded_dao(|_| {}); - let (whole, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), None)); - let (langs, _) = relowered(relower(&dao, &mut Breaks("item:daodejing:3"), &mut Lowerer::new(blind), None)); + let (whole, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), &mut [])); + let (langs, _) = relowered(relower(&dao, &mut Breaks("item:daodejing:3"), &mut Lowerer::new(blind), &mut [])); let l = &langs[0]; assert_eq!(l.unusable, 0, "{}", l.line()); assert_eq!(l.arm, whole[0].arm); @@ -5465,16 +5465,16 @@ fn relower_decides_by_decide_panel_against_the_floor_arm() { use weksa::run::decide_panel; let (dao, _) = recorded_dao(|_| {}); let run = |recorded: fn(&str, &str) -> String, arm: fn(&str, &str) -> String| { - let mut floor = Lowerer::new(recorded); + let (mut f1, mut f2, mut f3) = (Lowerer::new(recorded), Lowerer::new(recorded), Lowerer::new(recorded)); let (langs, verdict) = relowered(relower( &dao, &mut Lowerer::new(recorded), &mut Lowerer::new(arm), - Some(&mut floor), + &mut [&mut f1, &mut f2, &mut f3], )); let l = &langs[0]; let floor_reading = l.floor.expect("a floor arm gives a floor"); - // A floor arm that equals the recorded answers has no spread. + // Floor arms that equal the recorded answers have no spread. assert_eq!((floor_reading.chrf, floor_reading.meaning, floor_reading.composite), (0.0, 0.0, 0.0)); assert_eq!(verdict, Some(decide_panel(&[(l.recorded, l.arm, floor_reading)]))); verdict.unwrap() @@ -5485,29 +5485,34 @@ fn relower_decides_by_decide_panel_against_the_floor_arm() { } #[test] -fn relower_floor_is_the_spread_of_recorded_and_the_floor_arm() { +fn relower_floor_is_the_paired_noise_of_recorded_against_the_floor_arms() { let (dao, _) = recorded_dao(|_| {}); let (langs, _) = relowered(relower( &dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), - Some(&mut Breaks("item:daodejing:3")), + &mut [&mut Lowerer::new(blind), &mut Lowerer::new(blind), &mut Breaks("item:daodejing:3")], )); let l = &langs[0]; let f = l.floor.unwrap(); - // The floor arm read as an arm against the same recorded answers. - let (as_arm, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Breaks("item:daodejing:3"), None)); - let a = as_arm[0].arm; - assert!(f.chrf > 0.0 && f.meaning > 0.0 && f.composite > 0.0); - assert!((f.chrf - (l.recorded.chrf - a.chrf).abs()).abs() < 1e-9, "{}", l.line()); - assert!((f.composite - (l.recorded.composite - a.composite).abs()).abs() < 1e-9); + assert!(f.chrf > 0.0 && f.meaning > 0.0 && f.composite > 0.0, "{}", l.line()); assert!(l.line().contains(&format!("| floor chrf={:.2}", f.chrf)), "{}", l.line()); assert_eq!(l.unusable, 0, "unusable counts the arm, not the floor"); - assert_eq!(l.floor_unusable, 1, "the floor arm's refusal is counted apart"); + assert_eq!(l.floor_unusable, 1, "a floor arm's refusal is counted apart"); assert!(l.line().contains(" unusable=0 recorded "), "{}", l.line()); assert!(l.line().ends_with(" unusable=1"), "the floor's refusal is on the line: {}", l.line()); - // Without a floor arm there is none to count. - assert_eq!(as_arm[0].floor_unusable, 0); + // A floor needs three arms: fewer measure no noise, and nothing is asked of the arm. + for arms in [1usize, 2] { + let mut fs: Vec = (0..arms).map(|_| Lowerer::new(blind)).collect(); + let mut dyns: Vec<&mut dyn ModelPort> = fs.iter_mut().map(|f| f as &mut dyn ModelPort).collect(); + let mut arm = Lowerer::new(blind); + let r = relower(&dao, &mut Lowerer::new(blind), &mut arm, &mut dyns); + assert!(matches!(r, Err(RunError::FloorArms))); + assert!(arm.seen.is_empty()); + } + // Without floor arms there is none to count. + let (none, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), &mut [])); + assert_eq!(none[0].floor_unusable, 0); } #[test] @@ -5517,7 +5522,7 @@ fn relower_skips_and_counts_an_item_the_round_refused_before_admission() { answers.insert("item:daodejing:3".into(), "not a draft".into()); }); let (mut rec, mut arm) = (Lowerer::new(blind), Lowerer::new(blind)); - let (langs, _) = relowered(relower(&dao, &mut rec, &mut arm, None)); + let (langs, _) = relowered(relower(&dao, &mut rec, &mut arm, &mut [])); let l = &langs[0]; assert_eq!((l.n, l.skipped, l.unusable), (2, 1, 0), "{}", l.line()); assert!(l.line().starts_with("relower en-US n=2 skipped=1 unusable=0 "), "{}", l.line()); @@ -5536,7 +5541,7 @@ fn relower_skips_and_counts_an_item_whose_lowering_prompt_the_round_refused() { let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); assert_eq!(r.language("ja-JP").unwrap().validation_refused.lower, 1); let (mut rec, mut arm) = (Lowerer::new(bi_blind), Lowerer::new(bi_blind)); - let (langs, _) = relowered(relower(&dao, &mut rec, &mut arm, None)); + let (langs, _) = relowered(relower(&dao, &mut rec, &mut arm, &mut [])); let ja = langs.iter().find(|l| l.language == "ja-JP").unwrap(); assert_eq!((ja.n, ja.skipped), (1, 1), "{}", ja.line()); assert_eq!(ja.recorded, ja.arm); @@ -5579,7 +5584,7 @@ fn relower_decides_over_the_whole_panel_not_its_first_language() { &dao, &mut Lowerer::new(bi_reader), &mut Lowerer::new(bi_ja_lost), - Some(&mut Lowerer::new(bi_reader)), + &mut [&mut Lowerer::new(bi_reader), &mut Lowerer::new(bi_reader), &mut Lowerer::new(bi_reader)], )); assert_eq!( langs.iter().map(|l| (l.language.as_str(), l.n)).collect::>(), @@ -5603,16 +5608,16 @@ fn relower_refuses_a_cycle_round_and_a_missing_recorded_answer() { run_cycle(&mut dao, &mut cycle_fake(ONE_CONCEPT)); let candidate = dao.run.rounds().unwrap().into_iter().find(|r| r.cycle().is_some()).unwrap().n(); let mut arm = Lowerer::new(reader); - let r = dao.run.relower(candidate, "arm", &mut Lowerer::new(reader), &mut arm, None, &Scorer::new(Bag)); + let r = dao.run.relower(candidate, "arm", &mut Lowerer::new(reader), &mut arm, &mut [], &Scorer::new(Bag)); assert!(matches!(r, Err(RunError::Malformed("round")))); - let r = dao.run.relower(99, "arm", &mut Lowerer::new(reader), &mut arm, None, &Scorer::new(Bag)); + let r = dao.run.relower(99, "arm", &mut Lowerer::new(reader), &mut arm, &mut [], &Scorer::new(Bag)); assert!(matches!(r, Err(RunError::Malformed("round")))); assert!(arm.seen.is_empty()); let (dao, _) = recorded_dao(|_| {}); let mut empty = RecordedAnswers::new(dao.root.join("round-1/nowhere"), "fake-model".into()); let mut arm = Lowerer::new(reader); - let r = relower(&dao, &mut empty, &mut arm, None); + let r = relower(&dao, &mut empty, &mut arm, &mut []); assert!(matches!(r, Err(RunError::Unrecorded))); assert!(arm.seen.is_empty(), "nothing is asked of the arm before the round's answer exists"); } @@ -5622,7 +5627,12 @@ fn relower_floor_arm_with_an_unanswered_prompt_is_refused() { use weksa::model::RecordedAnswers; let (dao, _) = recorded_dao(|_| {}); let mut floor = RecordedAnswers::new(dao.root.join("relower-1-floor"), "fake-model".into()); - let r = relower(&dao, &mut Lowerer::new(reader), &mut Lowerer::new(reader), Some(&mut floor)); + let r = relower( + &dao, + &mut Lowerer::new(reader), + &mut Lowerer::new(reader), + &mut [&mut floor, &mut Lowerer::new(reader), &mut Lowerer::new(reader)], + ); assert!(matches!(r, Err(RunError::FloorUnanswered))); assert!(!dao.root.join("relower-1-floor").exists(), "a floor arm is never written to"); } @@ -5640,9 +5650,9 @@ fn relower_checks_the_embedder_not_the_model() { } let (dao, _) = recorded_dao(|_| {}); let mut other = Named(Lowerer::new(blind)); - relowered(relower(&dao, &mut Lowerer::new(reader), &mut other, None)); + relowered(relower(&dao, &mut Lowerer::new(reader), &mut other, &mut [])); let mut arm = Lowerer::new(reader); - let r = dao.run.relower(1, "arm", &mut Lowerer::new(reader), &mut arm, None, &Scorer::new(OtherEmbedder)); + let r = dao.run.relower(1, "arm", &mut Lowerer::new(reader), &mut arm, &mut [], &Scorer::new(OtherEmbedder)); assert!(matches!(r, Err(RunError::Drift(DriftOf::Embedder)))); assert!(arm.seen.is_empty()); } @@ -5676,7 +5686,7 @@ fn relower_records_nothing() { // An unanswered arm writes its prompts in its own directory only. let mut files = dao.run.answer_files(dao.run.relower_dir(1, "arm")).unwrap(); // The recorded answers are files here, as the relay leaves them. - let r = relower(&dao, &mut recorded, &mut files, None); + let r = relower(&dao, &mut recorded, &mut files, &mut []); assert!(matches!(r, Err(RunError::Unrecorded)), "no round answer files yet"); let after = tree(&dao.root); for (path, bytes) in &before { @@ -5699,7 +5709,7 @@ fn relower_records_nothing() { let before = tree(&dao.root); let mut recorded = RecordedAnswers::new(dao.root.join("round-1"), "fake-model".into()); let mut files = dao.run.answer_files(dao.run.relower_dir(1, "arm")).unwrap(); - let RelowerOutcome::Unanswered { prompts } = relower(&dao, &mut recorded, &mut files, None).unwrap() else { + let RelowerOutcome::Unanswered { prompts } = relower(&dao, &mut recorded, &mut files, &mut []).unwrap() else { panic!("the arm is unanswered") }; assert_eq!(prompts.len(), 3); @@ -5727,7 +5737,7 @@ fn relower_arm_prompt_holding_test_text_is_withheld() { |_| {}, ); let mut files = dao.run.answer_files(dao.run.relower_dir(1, "arm")).unwrap(); - let r = relower(&dao, &mut Lowerer::new(reader), &mut files, None); + let r = relower(&dao, &mut Lowerer::new(reader), &mut files, &mut []); assert!(matches!(r, Err(RunError::Withheld)), "{:?}", r.map(|_| ())); let dir = dao.run.relower_dir(1, "arm"); let prompts = std::fs::read_dir(&dir) diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 4a78abe..31018f6 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -359,7 +359,7 @@ the training set is the whole corpus re-encoded under the frozen revision. A lowerer is compared on round n's Validation packets with the prompts byte-identical to the round's, by `weksa loop relower ROOT --round N --arm ARM ---lowerer LABEL [--floor ARM2]`. Every arm is answered through the one prompt +--lowerer LABEL [--floor ARM2 --floor ARM3 --floor ARM4]`. Every arm is answered through the one prompt door (`RunStores::answer_files`, the Test-text guard); a tuned model is a script that reads `.prompt` and writes `.answer` in `/relower--`, not a new adapter. The recorded reading is the @@ -369,15 +369,29 @@ ask: it is skipped, so the readings exclude it, and it is counted as `skipped=` on the line; `n` is the items read, so `recorded` can differ from the round's own Validation reading, which scores those items 0. An arm answer that is not a usable lowering scores 0 and counts in `unusable=`; the same for the floor arm, -counted after `floor ... unusable=`. The floor is the spread of the recorded -lowering and a second prompted arm (lowering-only noise, no encoder noise). The -verdict is `decide_panel` over (recorded, arm, floor): Accepted is the rule's -name for a gain past the floor, Rejected(Regressed) for a loss past it, -Rejected(NoGain) for neither. None is yet a finding about the arm. The rule is calibration-pending: a one-pair floor is noisy -(a simulated prompted repeat reads NoGain only about 7-15% of the time, Regressed -44-60%, Accepted 33-41%), so NoGain is not yet evidence of non-inferiority and -is not yet the bar for cut 13; the calibration arms (a prompted repeat, a -control) decide what the floor must be before the verdict is read as a ruling. +counted (over all floor arms) after `floor ... unusable=`. The floor needs three +or more floor arms, each a prompted repeat of the recorded lowering on the same +prompts: it is `FLOOR_Z` (3) standard errors of the mean per-item difference +between the recorded lowering and a repeat, per metric, with the variances +pooled over the arms (`paired_floor`; lowering-only noise, no encoder noise). The +verdict is `decide_panel` over (recorded, arm, floor): Accepted is a composite +gain past the noise of the panel's sum (the root of the languages' composite +floors squared, at least one step), Rejected(Regressed) a loss past a language's +floor, Rejected(NoGain) neither. The cycle's own floor is still the spread of two +round replicates (`floor_of`); it has no per-item scores to pair. +Calibration (`runs/distil-base`, 8 prompted draws, each its own subagent, 14 +en-US and 14 ja-JP Validation items; every base, candidate and 3-arm floor +combination, 1120 trials from 28 unordered pairs; intervals bootstrap over +draws, each degraded arm one fixed draw): a prompted repeat reads NoGain 92% +(80 to 100), Regressed 8% (0 to 20), Accepted 0%, which with 28 pairs bounds +false-accept only to about 10%; an all-"Yes." arm reads Regressed 100%; a mild +degradation (one detail blurred per utterance) 100%; a subtler one 75% (50 to +100). At 14 items a degradation under about three standard errors (a composite +near 0.06) cannot be told from noise, and no arm known to be better exists, so +the rule's power to accept a real gain is unmeasured. `FLOOR_Z` was chosen on +this data (2.5 gives NoGain 84% and 93% on the subtler arm), so the rates are +in-sample. Without the noise term in the gain test a prompted repeat reads +Accepted in 40% of the trials. Training pairs are Train only, at a frozen revision; the export is 12b. ## Lowerer first cut (`golf-loop-lowerer-cut`) From 98ab798ca539e0577a79b7c30cf645a533eaa087 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 03:39:17 +0000 Subject: [PATCH 04/28] paired floor: one composite formula and its pin; docs state the power measure (#72) --- crates/weksa/src/run.rs | 20 +++++++++++++++----- docs/targets/interlingua-lowering-map.md | 24 +++++++++++++++++------- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index dd6bc4a..a9062a1 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -2157,6 +2157,12 @@ fn score_round( Ok(RoundOutcome::Recorded(record)) } +/// The composite of one chrF (0 to 100) and one meaning: their mean, chrF as a +/// fraction. The one formula for a split mean's [`Reading`] and a per-item score. +fn composite_of(chrf: f64, meaning: f64) -> f64 { + (chrf / 100.0 + meaning) / 2.0 +} + /// One validation reading: chrF 0 to 100, meaning 0 to 1, and their composite. #[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct Reading { @@ -2177,7 +2183,7 @@ impl Reading { Some(Self { chrf, meaning, - composite: (chrf / 100.0 + meaning) / 2.0, + composite: composite_of(chrf, meaning), }) } } @@ -2507,9 +2513,10 @@ fn floor_of( /// unordered pairs): with three floor arms a prompted repeat reads NoGain in /// 92% of the trials (bootstrap 95% over draws 80 to 100%), Regressed 8% and /// Accepted 0%; a "Yes." arm and a mild degradation (one detail blurred per -/// utterance) read Regressed in 100%, and a subtler one in 75% (50 to 100%): at -/// 14 items a degradation under about three standard errors cannot be told from -/// noise. `FLOOR_Z` was chosen on that data, so the rates are in-sample. +/// utterance) read Regressed in 100% (each a single fixed draw), and a subtler +/// one in 75% (50 to 100%): at 14 items a mean composite drop near 0.04 against +/// a floor near 0.06 is flagged only some of the time. `FLOOR_Z` was chosen on +/// that data, so the rates are in-sample. pub const FLOOR_Z: f64 = 3.0; pub const FLOOR_ARMS_MIN: usize = 3; @@ -2540,7 +2547,7 @@ pub fn paired_floor(recorded: &[ItemScore], arms: &[Vec]) -> Reading Reading { chrf: pooled(&|s| s.chrf), meaning: pooled(&meaning), - composite: pooled(&|s| (s.chrf / 100.0 + meaning(s)) / 2.0), + composite: pooled(&|s| composite_of(s.chrf, meaning(s))), } } @@ -2949,6 +2956,9 @@ mod tests { let two = paired_floor(&recorded, &[wobble.clone(), wobble.clone()]); assert!((one.chrf - want).abs() < 1e-9 && (two.chrf - want).abs() < 1e-9, "{one:?} {two:?}"); assert!(one.meaning.abs() < 1e-9); + // Composite per item is (chrF / 100 + meaning) / 2: the wobble moves chrF + // by 2 and meaning by 0, so the composite wobbles by 0.01. + assert!((one.composite - FLOOR_Z * (0.01_f64 * 0.01 * 4.0 / 3.0 / 4.0).sqrt()).abs() < 1e-9, "{one:?}"); // Arms pool by the mean of their variances: one wobbling arm and one // exact repeat give the root of half the variance. let exact = recorded.to_vec(); diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 31018f6..fbf2a50 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -379,19 +379,29 @@ gain past the noise of the panel's sum (the root of the languages' composite floors squared, at least one step), Rejected(Regressed) a loss past a language's floor, Rejected(NoGain) neither. The cycle's own floor is still the spread of two round replicates (`floor_of`); it has no per-item scores to pair. -Calibration (`runs/distil-base`, 8 prompted draws, each its own subagent, 14 +Calibration (`runs/distil-base`, 8 prompted draws by one model family on +identical prompts, no draw sharing more than 7 of 28 answers with another; 14 en-US and 14 ja-JP Validation items; every base, candidate and 3-arm floor combination, 1120 trials from 28 unordered pairs; intervals bootstrap over draws, each degraded arm one fixed draw): a prompted repeat reads NoGain 92% (80 to 100), Regressed 8% (0 to 20), Accepted 0%, which with 28 pairs bounds false-accept only to about 10%; an all-"Yes." arm reads Regressed 100%; a mild degradation (one detail blurred per utterance) 100%; a subtler one 75% (50 to -100). At 14 items a degradation under about three standard errors (a composite -near 0.06) cannot be told from noise, and no arm known to be better exists, so -the rule's power to accept a real gain is unmeasured. `FLOOR_Z` was chosen on -this data (2.5 gives NoGain 84% and 93% on the subtler arm), so the rates are -in-sample. Without the noise term in the gain test a prompted repeat reads -Accepted in 40% of the trials. +100), whose mean composite drop (0.044) sits under the mean floor (0.060). +`FLOOR_Z` was chosen on this data (2.5 gives NoGain 84% and 93% on the subtler +arm), so the rates are in-sample. Without the noise term in the gain test a +prompted repeat reads Accepted in 40% of the trials. +Power (can the rule Accept a real gain?): the only better arm that could be +built without Test or Validation answers is a lowerer shown the Train sentences +of the corpus (no packets, no Validation or Test text) as a register guide. +Four such draws beat the prompted draws' mean composite in both languages (en-US +0.742 to 0.759 to 0.795; ja-JP 0.559 to 0.578 to 0.620), so the premise that it is +better is measured on the very Validation items the rule reads, not independent. +Against the prompted bases with 3-arm floors the rule reads it Accepted in 29% +of 1120 trials, NoGain 71%, Regressed 0%: a net gain of 0.079 against a mean +noise term of 0.079. At 14 items per language the rule is therefore a low-power +filter for a gain of this size; by 1/sqrt(n), about 35 or more items per +language would bring the noise term near 0.05 (an extrapolation, not measured). Training pairs are Train only, at a frozen revision; the export is 12b. ## Lowerer first cut (`golf-loop-lowerer-cut`) From 516eb4b885c9a1ccda5412cd0c3a0ed1a734dcc5 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 03:47:53 +0000 Subject: [PATCH 05/28] round: per-item Validation scores in the round record (cycle-floor-scores) (#71) --- crates/weksa/src/run.rs | 30 ++- crates/weksa/src/score/mod.rs | 46 +++- crates/weksa/tests/golf_loop.rs | 231 +++++++++++++++++++ docs/targets/interlingua-lowering-map.md | 2 +- schemas/cultnet/weksa.loop_round.schema.json | 50 +++- 5 files changed, 349 insertions(+), 10 deletions(-) diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index a9062a1..62a8815 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -46,7 +46,7 @@ use crate::encode::{ use crate::form::{is_attestation, is_language_tag, is_slug}; use crate::model::{AnswerFiles, ModelPort, PortError}; use crate::packet::{Packet, Stance}; -use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate}; +use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate, validation_items}; use crate::split::{Split, SplitScheme, fnv1a64, fnv1a64_of, hex16}; use crate::store::{ CorpusStore, EntityStore, InventoryStore, OntologyStore, PacketStore, SpeakerStateStore, @@ -246,9 +246,18 @@ pub struct LanguageRound { pub validation_refused: RefusalCounts, } +/// One panel language's Validation scores in a round, in corpus order: numbers +/// only, no item key and no text. Only `run.rs` reads them (the paired floor). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct LanguageScores { + pub language: String, + pub scores: Vec, +} + /// One scored round. It holds no text and no Test reading: per-item rows for -/// Train only, Validation only as means and refusal counts by stage. Only -/// `run::round` builds one. +/// Train only; Validation as means and refusal counts by stage, and as +/// per-item scores (numbers only, private to this module). Only `run::round` +/// builds one. /// /// ``` /// fn f(r: &weksa::run::RoundRecord) -> u32 { r.n() } @@ -277,6 +286,7 @@ pub struct LanguageRound { /// parse_reasons: vec![], /// own_family: 0, /// languages: vec![], +/// validation_scores: vec![], /// } /// } /// ``` @@ -336,6 +346,12 @@ pub struct RoundRecord { // Key 20 is `own_family`; the per-language readings take 21. #[cultcache(key = 21)] languages: Vec, + /// Each panel language's live Validation scores in corpus order, a refused + /// item as 0. Private with no getter: only `run.rs` (the floor) reads them, + /// so a leak outside it is a compile error. A run from before has none. + #[cultcache(key = 22, default)] + #[schemars(default)] + validation_scores: Vec, } impl RoundRecord { @@ -2069,6 +2085,7 @@ fn score_round( let mut train_rows = Vec::new(); let mut languages: Vec = Vec::new(); + let mut validation_scores: Vec = Vec::new(); for tag in &run.record.panel { let of: Vec<(&CorpusItem, ItemScore)> = scored .iter() @@ -2076,6 +2093,10 @@ fn score_round( .cloned() .collect(); let means = aggregate(&of); + validation_scores.push(LanguageScores { + language: tag.clone(), + scores: validation_items(&of), + }); languages.push(LanguageRound { language: tag.clone(), train_mean: means.train, @@ -2153,6 +2174,7 @@ fn score_round( .collect(), own_family, languages, + validation_scores, }; Ok(RoundOutcome::Recorded(record)) } @@ -2771,6 +2793,7 @@ mod tests { parse_reasons: vec![], own_family: 0, languages: vec![], + validation_scores: vec![], }; assert_eq!(r.n(), 1); } @@ -3286,6 +3309,7 @@ mod tests { validation_scored: 2, validation_refused: RefusalCounts::default(), }], + validation_scores: vec![], } } diff --git a/crates/weksa/src/score/mod.rs b/crates/weksa/src/score/mod.rs index c1033a4..e411d7c 100644 --- a/crates/weksa/src/score/mod.rs +++ b/crates/weksa/src/score/mod.rs @@ -28,7 +28,7 @@ pub struct ScoreInput<'a> { pub output: &'a str, } -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct ItemScore { /// chrF of the output against the reference. pub chrf: f64, @@ -131,13 +131,18 @@ pub struct Aggregate { pub validation: SplitMean, } -/// The mean of the scores of the items read in split `want`. -fn split_mean(scored: &[(&CorpusItem, ItemScore)], want: Split) -> SplitMean { - let of: Vec<&ItemScore> = scored +/// The scores of the items read in split `want`, in list order. +fn scores_in<'a>(scored: &'a [(&CorpusItem, ItemScore)], want: Split) -> Vec<&'a ItemScore> { + scored .iter() .filter(|(item, _)| SplitScheme::V1.live(item) == Some(want)) .map(|(_, score)| score) - .collect(); + .collect() +} + +/// The mean of the scores of the items read in split `want`. +fn split_mean(scored: &[(&CorpusItem, ItemScore)], want: Split) -> SplitMean { + let of = scores_in(scored, want); let average = |values: Vec| { (!values.is_empty()).then(|| values.iter().sum::() / values.len() as f64) }; @@ -157,6 +162,12 @@ pub fn aggregate(scored: &[(&CorpusItem, ItemScore)]) -> Aggregate { } } +/// The Validation items' scores in list order, numbers only: no key, no text. +/// A refused item arrives scored 0 and keeps its position. +pub fn validation_items(scored: &[(&CorpusItem, ItemScore)]) -> Vec { + scores_in(scored, Split::Validation).into_iter().cloned().collect() +} + /// The Test split's readings, scalars only: no item key, no minted string, no /// pair. The checkpoint decision (convergence-test) needs numbers; a key beside /// a minted string is the handle an edit step would act on. The only producer @@ -249,6 +260,31 @@ mod tests { item(key, text, &group_in(split), model) } + #[test] + fn validation_items_keeps_validation_only_in_order() { + let train = in_split("item:t:1", UNRELATED, Split::Train, false); + let first = in_split("item:v:1", "first words", Split::Validation, false); + let refused = in_split("item:v:2", "second words", Split::Validation, false); + let last = in_split("item:v:3", "third words", Split::Validation, false); + let mut gone = in_split("item:v:4", "withdrawn words", Split::Validation, false); + gone.withdrawn = true; + let test = in_split("item:c:1", HELD_OUT, Split::Test, false); + let scored = [ + (&train, score(10.0, Some(0.1))), + (&first, score(40.0, Some(0.4))), + (&test, score(60.0, Some(0.6))), + (&refused, score(0.0, Some(0.0))), + (&gone, score(90.0, Some(0.9))), + (&last, score(80.0, None)), + ]; + let got = validation_items(&scored); + assert_eq!( + got, + [score(40.0, Some(0.4)), score(0.0, Some(0.0)), score(80.0, None)] + ); + assert_eq!(got.len(), aggregate(&scored).validation.count); + } + #[test] fn test_readings_at_checkpoint() { let train_item = in_split("item:t:1", UNRELATED, Split::Train, false); diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 2d0320a..635065e 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -2645,6 +2645,9 @@ fn source_tripwires() { "Aggregate", "validation_mean", "validation_refused", + "ItemScore", + "LanguageScores", + "validation_scores", ] { assert!(!context.contains(banned), "context.rs names {banned}"); } @@ -3364,6 +3367,234 @@ fn a_run_from_before_does_not_open() { assert!(reopened.rounds().is_err()); } +/// A round as the file holds it, with key 22 the lib keeps private. +#[derive(Clone, Debug, schemars::JsonSchema, cultcache_rs::DatabaseEntry)] +#[cultcache(type = "weksa.loop_round")] +struct ScoredRound { + #[cultcache(key = 0)] + n: u32, + #[cultcache(key = 1)] + inventory_rev: u32, + #[cultcache(key = 2)] + prompt_digest: String, + #[cultcache(key = 4)] + corpus_digest: String, + #[cultcache(key = 5)] + entity_digest: String, + #[cultcache(key = 6)] + model: String, + #[cultcache(key = 7)] + embedder: String, + #[cultcache(key = 8)] + chrf_version: String, + #[cultcache(key = 9)] + calls: Vec, + #[cultcache(key = 10)] + train: Vec, + #[cultcache(key = 16)] + inventory_digest: String, + #[cultcache(key = 17)] + cycle: Option, + #[cultcache(key = 18)] + admit_rules: Vec, + #[cultcache(key = 19)] + parse_reasons: Vec, + #[cultcache(key = 20)] + own_family: u32, + #[cultcache(key = 21)] + languages: Vec, + #[cultcache(key = 22)] + validation_scores: Vec, +} + +/// A round as written before key 22 existed. +#[derive(Clone, Debug, schemars::JsonSchema, cultcache_rs::DatabaseEntry)] +#[cultcache(type = "weksa.loop_round")] +struct PreScoresRound { + #[cultcache(key = 0)] + n: u32, + #[cultcache(key = 1)] + inventory_rev: u32, + #[cultcache(key = 2)] + prompt_digest: String, + #[cultcache(key = 4)] + corpus_digest: String, + #[cultcache(key = 5)] + entity_digest: String, + #[cultcache(key = 6)] + model: String, + #[cultcache(key = 7)] + embedder: String, + #[cultcache(key = 8)] + chrf_version: String, + #[cultcache(key = 9)] + calls: Vec, + #[cultcache(key = 10)] + train: Vec, + #[cultcache(key = 16)] + inventory_digest: String, + #[cultcache(key = 17)] + cycle: Option, + #[cultcache(key = 18)] + admit_rules: Vec, + #[cultcache(key = 19)] + parse_reasons: Vec, + #[cultcache(key = 20)] + own_family: u32, + #[cultcache(key = 21)] + languages: Vec, +} + +/// The scores a round put in `run.cc`, read through the mirror (the lib's field +/// is private). The run must be dropped first. +fn stored_scores(root: &Path) -> Vec { + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache + .add_generic_backing_store(SingleFileMessagePackBackingStore::new(root.join("run.cc"))) + .unwrap(); + cache.pull_all_backing_stores().unwrap(); + cache.get_all::().unwrap().remove(0).validation_scores +} + +/// The stored scores agree with the round's readings: one entry per language +/// in the record's order, as many scores as Validation items (scored plus +/// refused), the same means, and at least the refused items as exact zeros. +fn assert_scores_match_readings(r: &RoundRecord, stored: &[weksa::run::LanguageScores]) { + let tags: Vec<&str> = r.languages().iter().map(|l| l.language.as_str()).collect(); + let got: Vec<&str> = stored.iter().map(|l| l.language.as_str()).collect(); + assert_eq!(got, tags, "one entry per panel language, in panel order"); + for (reading, entry) in r.languages().iter().zip(stored) { + let m = &reading.validation_mean; + let v = reading.validation_refused; + let refused = (v.parse + v.admit + v.lower) as usize; + assert_eq!(entry.scores.len(), m.count, "{}", reading.language); + assert_eq!(entry.scores.len(), reading.validation_scored as usize + refused); + let n = entry.scores.len() as f64; + let chrf = entry.scores.iter().map(|s| s.chrf).sum::() / n; + assert!((chrf - m.chrf.unwrap()).abs() < 1e-9, "{}", reading.language); + let zeros = entry + .scores + .iter() + .filter(|s| s.chrf == 0.0 && s.meaning == Some(0.0)) + .count(); + assert!(zeros >= refused, "{}: refused items are stored as zeros", reading.language); + } +} + +#[test] +fn a_round_stores_one_score_per_validation_item() { + let mut dao = dao(); + let mut model = fake(&good_answers()); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + let root = dao.root.clone(); + drop(dao.run); + let stored = stored_scores(&root); + assert_eq!(stored.len(), 1); + assert_eq!(stored[0].language, "en-US"); + assert_scores_match_readings(&r, &stored); +} + +#[test] +fn a_two_language_round_stores_each_languages_scores_in_panel_order() { + let mut dao = bilingual(LAOZI_JA); + let mut model = bilingual_fake(); + // The ja-JP Validation item 13 is not an encodable draft: it scores 0. + model.answers.insert("item:daodejing:13".into(), "not a draft".into()); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + let root = dao.root.clone(); + drop(dao.run); + let stored = stored_scores(&root); + assert_scores_match_readings(&r, &stored); + let tags: Vec<&str> = stored.iter().map(|l| l.language.as_str()).collect(); + assert_eq!(tags, ["en-US", "ja-JP"]); + // ja-JP: item 12 reads perfectly, then item 13 (the refused one), in + // stored item order; none of en-US's items is among them. + let ja: Vec<(f64, Option)> = stored[1].scores.iter().map(|s| (s.chrf, s.meaning)).collect(); + assert_eq!(ja, [(100.0, Some(1.0)), (0.0, Some(0.0))]); + assert_eq!(stored[0].scores.len(), 2); + assert!(stored[0].scores.iter().all(|s| s.chrf < 100.0), "en-US holds its own items"); +} + +#[test] +fn refused_validation_items_are_stored_as_zeros_at_every_stage() { + // Lower: the ja-JP items cite an entity with no ja-JP form. + let mut dao = bilingual(None); + let mut model = bilingual_fake(); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + let root = dao.root.clone(); + drop(dao.run); + let stored = stored_scores(&root); + assert_eq!(r.language("ja-JP").unwrap().validation_refused.lower, 2); + assert_scores_match_readings(&r, &stored); + assert!(stored[1].scores.iter().all(|s| s.chrf == 0.0 && s.meaning == Some(0.0))); + + // Parse (item 3 is Validation) and Admit (item 2), one run each. + for (parse, admit) in [(true, false), (false, true)] { + let mut dao = self::dao(); + let mut answers = good_answers(); + if parse { + answers[2].1 = "not json at all".into(); + } + if admit { + answers[1].1 = edited(2, unknown_concept); + } + let mut model = fake(&answers); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + let root = dao.root.clone(); + drop(dao.run); + let stored = stored_scores(&root); + let v = r.language("en-US").unwrap().validation_refused; + assert_eq!((v.parse, v.admit), (u32::from(parse), u32::from(admit))); + assert_scores_match_readings(&r, &stored); + } +} + +#[test] +fn an_old_run_without_scores_still_opens() { + let mut dao = dao(); + let mut model = fake(&good_answers()); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + let rec = dao.run.record().clone(); + let root = dao.root.clone(); + drop(dao.run); + let old = PreScoresRound { + n: r.n(), + inventory_rev: r.inventory_rev(), + prompt_digest: r.prompt_digest().into(), + corpus_digest: r.corpus_digest().into(), + entity_digest: r.entity_digest().into(), + model: r.model().into(), + embedder: r.embedder().into(), + chrf_version: r.chrf_version().into(), + calls: r.calls().to_vec(), + train: r.train().to_vec(), + inventory_digest: r.inventory_digest().into(), + cycle: r.cycle(), + admit_rules: vec![], + parse_reasons: vec![], + own_family: 0, + languages: vec![r.language("en-US").unwrap().clone()], + }; + let _ = std::fs::remove_file(root.join("run.cc")); + { + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache + .add_generic_backing_store(SingleFileMessagePackBackingStore::new(root.join("run.cc"))) + .unwrap(); + cache.pull_all_backing_stores().unwrap(); + cache.put("run:dao".to_string(), &rec).unwrap(); + cache.put("run:dao:round:1".to_string(), &old).unwrap(); + } + let reopened = RunStores::open(&root).unwrap(); + let rounds = reopened.rounds().unwrap(); + assert_eq!(rounds.len(), 1); + assert_eq!(rounds[0].language("en-US"), r.language("en-US")); +} + #[test] fn the_published_run_and_round_schemas_require_none_of_the_new_fields() { let published = weksa::schema::published(); diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index fbf2a50..9766d35 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -40,7 +40,7 @@ means no store holds it and nothing writes it. | Scene (cut `speaker-scene`) | Key `scene:@`, the item's split group; type `weksa.scene`: participants (speaker slugs) and, per ordered pair, relation features of slot `Scene` (relative status, intimacy, in-group, audience and whatever the loop adds), at an inventory revision (provenance; a run reads the features at its own head), and an `Annotator` | As Speaker. Keyed by the group, so a scene never straddles the split. `data/original/speaker-state.cc` holds Model-annotated keeper scenes, one per Train or Validation group of both languages (`weksa scene put|dump`); a Test group has none and reads `NoScene` | As Speaker; a model-annotated scene is never Test evidence | | Corpus item | Key `item::`, `n` canonical decimal >= 1 (no sign, no leading zero, from cut 2b); type `weksa.corpus_item` with text, language, speaker, split group, prior turns, source. Source `ModelWritten` carries a `ModelFamily` enum, the generator string as the fixture wrote it, and a prompt digest (cut 2b). The item carries no speaker state of its own: its speaker slug and group name the Speaker and Scene documents | Admitted once; the whole item is immutable (re-adding an identical item is a no-op, any difference is refused; a correction is a new item that `supersedes` the old); text already held under another key is refused unless the new item supersedes that key (cut 2b); withdrawn, never deleted, so old scores stay explainable | Importers and `weksa corpus add`, under `corpus-cast-plus-public-domain`; the source enum has no variant for Discord or any training-forbidden source (`corpus-clean`) | | Split assignment | Derived: `split(scheme, item.group)` | Fixed for a scheme; a new scheme is a new constant, never a reassignment. A model-written item whose group derives test is in no split; a superseding item keeps its predecessor's group | The hash function. Nobody assigns by hand, so there is no writer to forbid | -| Loop round | Key `run: