From b3821810a58374537adbed8fa1f51e77fa80c8e6 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 00:29:13 +0000 Subject: [PATCH 01/34] test-text-guard: AnswerFiles refuses a prompt that holds the text of a live Test item, before any file is read or written RunStores::answer_files is the one constructor; the five CLI sites use it. Whole lines and JSON strings match at any length, fragments from 24 chars. Zero hits over 168 real measure-ja prompts. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/main.rs | 25 ++++---- crates/weksa/src/model.rs | 30 +++++++++- crates/weksa/src/run.rs | 27 +++++++-- crates/weksa/src/split.rs | 3 +- crates/weksa/src/store.rs | 11 ++++ crates/weksa/tests/golf_loop.rs | 89 +++++++++++++++++++++++------ crates/weksa/tests/ontology_mint.rs | 6 +- 7 files changed, 149 insertions(+), 42 deletions(-) diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 92c7d5e..b76051c 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -19,7 +19,7 @@ use weksa::run::{ }; use weksa::score::ScoreInput; use weksa::{ - Annotator, AnswerFiles, CorpusStore, EntityId, EntityStore, Family, InventoryStore, OllamaEmbedder, OntologyStore, OptionLabel, + Annotator, CorpusStore, EntityId, EntityStore, Family, InventoryStore, OllamaEmbedder, OntologyStore, OptionLabel, PacketStore, PairRelation, Party, Scene, Slot, SpeakerStateStore, ValueKind, Scorer, Split, SplitScheme, StoreError, lower, pronounce, validate, }; @@ -1070,10 +1070,7 @@ fn main() -> Result<()> { Command::Loop(LoopCommand::Round { root, ollama_url }) => { let unreadable = |_| anyhow::anyhow!("a store would not read"); let mut run = RunStores::open(&root)?; - let mut files = AnswerFiles::new( - run.next_round_dir().map_err(unreadable)?, - run.record().model.clone(), - ); + let mut files = run.answer_files(run.next_round_dir().map_err(unreadable)?)?; let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder)); match round(&mut run, &mut files, &scorer)? { RoundOutcome::Unanswered { prompts } => { @@ -1093,10 +1090,11 @@ fn main() -> Result<()> { ollama_url, }) => { let run = RunStores::open(&root)?; - let model = run.record().model.clone(); - let mut recorded = - RecordedAnswers::new(root.join(format!("round-{round}")), model.clone()); - let mut files = AnswerFiles::new(run.ablation_dir(round), model); + let mut recorded = RecordedAnswers::new( + root.join(format!("round-{round}")), + run.record().model.clone(), + ); + let mut files = run.answer_files(run.ablation_dir(round))?; let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder)); match run.stance_ablation(round, &mut recorded, &mut files, &scorer)? { AblationOutcome::Unanswered { prompts } => { @@ -1119,7 +1117,7 @@ fn main() -> Result<()> { } Command::Loop(LoopCommand::Recall { root }) => { let run = RunStores::open(&root)?; - let mut files = AnswerFiles::new(run.recall_dir(), run.record().model.clone()); + let mut files = run.answer_files(run.recall_dir())?; match run.recall(&mut files)? { RecallOutcome::Unanswered { prompts } => { for digest in &prompts { @@ -1133,7 +1131,7 @@ fn main() -> Result<()> { } Command::Loop(LoopCommand::MintOntology { root, language }) => { let run = RunStores::open(&root)?; - let mut files = AnswerFiles::new(run.ontology_mint_dir(), run.record().model.clone()); + let mut files = run.answer_files(run.ontology_mint_dir())?; match run.mint_ontology(&mut files, &language)? { OntologyMintOutcome::Unanswered { prompt } => { println!("unanswered {prompt}"); @@ -1152,10 +1150,7 @@ fn main() -> Result<()> { Command::Loop(LoopCommand::Cycle { root, ollama_url }) => { let unreadable = |_| anyhow::anyhow!("a store would not read"); let mut run = RunStores::open(&root)?; - let mut files = AnswerFiles::new( - run.next_cycle_dir().map_err(unreadable)?, - run.record().model.clone(), - ); + let mut files = run.answer_files(run.next_cycle_dir().map_err(unreadable)?)?; let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder)); match cycle(&mut run, &mut files, &scorer)? { CycleOutcome::Unanswered { prompts } => { diff --git a/crates/weksa/src/model.rs b/crates/weksa/src/model.rs index 1a79449..1b2e37b 100644 --- a/crates/weksa/src/model.rs +++ b/crates/weksa/src/model.rs @@ -20,6 +20,8 @@ pub trait ModelPort { pub enum PortError { /// Nobody has answered yet. Unanswered, + /// The prompt holds the text of a Test item. It is neither written nor answered. + Withheld, Io(ErrorKind), } @@ -27,6 +29,7 @@ impl fmt::Display for PortError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { PortError::Unanswered => f.write_str("the prompt is unanswered"), + PortError::Withheld => f.write_str("the prompt holds Test item text"), PortError::Io(kind) => write!(f, "io error {kind:?}"), } } @@ -40,14 +43,34 @@ impl std::error::Error for PortError {} pub struct AnswerFiles { dir: PathBuf, model: String, + withheld: Vec, } impl AnswerFiles { - pub fn new(dir: PathBuf, model: String) -> Self { - Self { dir, model } + /// Refuses (`PortError::Withheld`) any prompt that holds the text of one of + /// `withheld`, before it reads or writes a file: the one door a prompt + /// leaves the loop through, so a Test sentence cannot reach a model by it. + pub fn new(dir: PathBuf, model: String, withheld: Vec) -> Self { + Self { dir, model, withheld } } } +/// Whether `prompt` holds `text` as a line, as a JSON string, or (at 24 chars +/// or more) anywhere. A short sentence is matched whole, since "Stop." is a +/// substring of any prompt that says "Stop. Then ..."; an item reaches a +/// prompt as a line or a JSON string value. +fn holds(prompt: &str, text: &str) -> bool { + let text = text.trim(); + if text.is_empty() { + return false; + } + if text.chars().count() >= 24 && prompt.contains(text) { + return true; + } + let json = serde_json::to_string(text).unwrap_or_default(); + prompt.lines().any(|l| l.trim() == text) || prompt.contains(&json) +} + /// Answers read from one directory and nothing else: `.answer` is the /// reply to the prompt whose digest is ``. A prompt with no file is /// [`PortError::Unanswered`]; this port never creates or writes anything. The @@ -86,6 +109,9 @@ impl ModelPort for AnswerFiles { fn answer(&mut self, prompt: &Prompt) -> Result { let io = |e: std::io::Error| PortError::Io(e.kind()); + if self.withheld.iter().any(|t| holds(prompt.text(), t)) { + return Err(PortError::Withheld); + } let digest = prompt.digest(); match RecordedAnswers::new(self.dir.clone(), self.model.clone()).answer(prompt) { Err(PortError::Unanswered) => {} diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index d58e2cb..b6a7557 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -44,7 +44,7 @@ use crate::encode::{ EditDraft, LoweringDraft, MAX_EDIT_CHANGES, OntologyDraft, PacketDraft, encoder_id, }; use crate::form::{is_attestation, is_language_tag, is_slug}; -use crate::model::{ModelPort, PortError}; +use crate::model::{AnswerFiles, ModelPort, PortError}; use crate::packet::{Packet, Stance}; use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate}; use crate::split::{Split, SplitScheme, fnv1a64, fnv1a64_of, hex16}; @@ -474,6 +474,8 @@ pub enum RunError { /// The inventory store holds no revision. NoInventory, Io(ErrorKind), + /// A prompt held the text of a Test item and was refused before any file. + Withheld, /// A store or the run file would not read or write. Store, /// The embedder failed. @@ -507,6 +509,7 @@ impl fmt::Display for RunError { RunError::Drift(of) => write!(f, "drift in {of:?}"), RunError::NoInventory => f.write_str("the inventory holds no revision"), RunError::Io(kind) => write!(f, "io error {kind:?}"), + RunError::Withheld => f.write_str("a prompt holds Test item text; nothing was written"), RunError::Store => f.write_str("a store would not read or write"), RunError::Unrecorded => f.write_str( "the round holds no answer to a lowering prompt rebuilt now: the prompt digest drifted or the head inventory moved since the round", @@ -935,6 +938,13 @@ impl RunStores { .collect()) } + /// The answer door for prompts in `dir`: refuses a prompt that holds the text + /// of any live Test item of the corpus, in any language. + pub fn answer_files(&self, dir: PathBuf) -> anyhow::Result { + let withheld = self.corpus.test_texts()?; + Ok(AnswerFiles::new(dir, self.record.model.clone(), withheld)) + } + /// The recorded rounds, in order. pub fn rounds(&self) -> anyhow::Result> { let mut rounds = self.cache.get_all::()?; @@ -1120,7 +1130,8 @@ impl RunStores { } Err(PortError::Unanswered) => unanswered.push(prompt.digest()), Err(PortError::Io(ErrorKind::InvalidData)) => scores.push((language, None)), - Err(PortError::Io(kind)) => return Err(RunError::Io(kind)), + Err(PortError::Withheld) => return Err(RunError::Withheld), + Err(PortError::Io(kind)) => return Err(RunError::Io(kind)), } } if !unanswered.is_empty() { @@ -1208,7 +1219,8 @@ impl RunStores { Err(PortError::Io(ErrorKind::InvalidData)) => { Ok(OntologyMintOutcome::Refused { code: "invalid-data" }) } - Err(PortError::Io(kind)) => Err(RunError::Io(kind)), + Err(PortError::Withheld) => Err(RunError::Withheld), + Err(PortError::Io(kind)) => Err(RunError::Io(kind)), } } @@ -1305,6 +1317,7 @@ fn asked(port: &mut dyn ModelPort, prompt: &Prompt) -> Result { .map_or(Asked::Unusable, |d| Asked::Text(d.text))), Err(PortError::Io(ErrorKind::InvalidData)) => Ok(Asked::Unusable), Err(PortError::Unanswered) => Ok(Asked::Waiting), + Err(PortError::Withheld) => Err(RunError::Withheld), Err(PortError::Io(kind)) => Err(RunError::Io(kind)), } } @@ -1636,7 +1649,8 @@ fn score_round( answers.push((offer.key().to_string(), None)) } Err(PortError::Unanswered) => unanswered.push(prompt.digest()), - Err(PortError::Io(kind)) => return Err(RunError::Io(kind)), + Err(PortError::Withheld) => return Err(RunError::Withheld), + Err(PortError::Io(kind)) => return Err(RunError::Io(kind)), } calls.push(Call { prompt: prompt.digest(), @@ -1730,6 +1744,10 @@ fn score_round( } continue; } + Err(PortError::Withheld) => { + discard_packets(dir); + return Err(RunError::Withheld); + } Err(PortError::Io(kind)) => { discard_packets(dir); return Err(RunError::Io(kind)); @@ -2277,6 +2295,7 @@ pub fn cycle( prompts: vec![prompt.digest()], }); } + Err(PortError::Withheld) => return Err(RunError::Withheld), Err(PortError::Io(kind)) => return Err(RunError::Io(kind)), }; diff --git a/crates/weksa/src/split.rs b/crates/weksa/src/split.rs index 898439d..744e308 100644 --- a/crates/weksa/src/split.rs +++ b/crates/weksa/src/split.rs @@ -5,7 +5,8 @@ //! `EncoderOffer` need a Checkpoint, and only the lib can make one //! (convergence-freeze adds the door). Operator tools (`weksa corpus dump`, //! `weksa score`, `CorpusStore`) read corpus text of any split; that is not -//! gated here. +//! gated here. `AnswerFiles`, the one door a prompt leaves the loop by, refuses +//! any prompt that holds the text of a live Test item (`PortError::Withheld`). use std::collections::BTreeSet; diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index b0b83a9..f746d70 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -12,6 +12,7 @@ use unicode_normalization::{UnicodeNormalization, char::is_combining_mark, is_nf use unicode_segmentation::UnicodeSegmentation; use crate::corpus::{CorpusItem, CorpusItemId, Source, item_key, key_slug}; +use crate::split::{Split, SplitScheme}; use crate::entity::{Entity, EntityOrigin, Pronunciation}; use crate::form::{ is_attestation, is_calendar_date, is_joining_only, is_language_tag, is_refused_format, is_slug, @@ -248,6 +249,16 @@ impl CorpusStore { Ok(items) } + /// The text of every live Test item, any language: what no prompt may hold. + pub fn test_texts(&self) -> anyhow::Result> { + Ok(self + .all()? + .into_iter() + .filter(|item| SplitScheme::V1.live(item) == Some(Split::Test)) + .map(|item| item.text) + .collect()) + } + pub fn get(&self, key: &str) -> anyhow::Result> { self.cache.get::(key) } diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 3ce5e62..dc310a4 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -669,7 +669,7 @@ fn answer_files_write_the_prompt_and_read_the_answer() { let p = prompt(); let dir = tempfile::tempdir().unwrap(); let round = dir.path().join("round-1"); - let mut files = AnswerFiles::new(round.clone(), "answerer".into()); + let mut files = AnswerFiles::new(round.clone(), "answerer".into(), vec![]); assert_eq!(files.model(), "answerer"); // No answer: the prompt is written under its digest, and nothing else. @@ -701,7 +701,7 @@ fn answer_files_write_the_prompt_and_read_the_answer() { // A directory that cannot be made is an io error of its kind. std::fs::write(dir.path().join("file"), "").unwrap(); - let mut blocked = AnswerFiles::new(dir.path().join("file").join("round"), "m".into()); + let mut blocked = AnswerFiles::new(dir.path().join("file").join("round"), "m".into(), vec![]); assert!(matches!(blocked.answer(&p), Err(PortError::Io(_)))); // A reply or prompt file that cannot be read is an io error, never "unanswered". @@ -1113,7 +1113,7 @@ fn unanswered_round_writes_nothing() { let expected = prompts(&dao.run); assert_eq!(expected.len(), 3); - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); let RoundOutcome::Unanswered { prompts: digests } = round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap() else { @@ -1144,7 +1144,7 @@ fn unanswered_round_writes_nothing() { for (key, p) in &expected[..2] { std::fs::write(round1.join(format!("{}.answer", p.digest())), &answers[key]).unwrap(); } - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); let RoundOutcome::Unanswered { prompts: digests } = round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap() else { @@ -1159,7 +1159,7 @@ fn unanswered_round_writes_nothing() { let (key, p) = &expected[2]; std::fs::write(round1.join(format!("{}.answer", p.digest())), &answers[key]).unwrap(); let lowering = lowering_prompts(&dao, dao.run.inventory()); - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); let RoundOutcome::Unanswered { prompts: digests } = round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap() else { @@ -1183,7 +1183,7 @@ fn unanswered_round_writes_nothing() { for (key, p) in &lowering[..2] { std::fs::write(round1.join(format!("{}.answer", p.digest())), lowered(key)).unwrap(); } - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); let RoundOutcome::Unanswered { prompts: digests } = round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap() else { @@ -1195,13 +1195,13 @@ fn unanswered_round_writes_nothing() { assert_eq!(std::fs::read(dao.root.join("run.cc")).unwrap(), run_cc); let (key, p) = &lowering[2]; std::fs::write(round1.join(format!("{}.answer", p.digest())), lowered(key)).unwrap(); - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()); assert_eq!((r.n(), r.calls().len()), (1, 6)); assert_eq!(two_dp(r.language("en-US").unwrap().train_mean.chrf.unwrap()), 44.18); // Round 2 asks again, encode first. - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); assert!(dao.root.join("round-2").parent().is_some()); let RoundOutcome::Unanswered { prompts: again } = round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap() @@ -1224,7 +1224,7 @@ fn unanswered_round_writes_nothing() { ) .unwrap(); } - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); let second = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()); assert_eq!(second.n(), 2); let ns: Vec = dao.run.rounds().unwrap().iter().map(|r| r.n()).collect(); @@ -1411,7 +1411,7 @@ fn non_utf8_encode_answer_is_a_parse_refusal() { let answer = serde_json::json!({ "text": table[key] }).to_string(); std::fs::write(round1.join(format!("{}.answer", p.digest())), answer).unwrap(); } - let mut files = AnswerFiles::new(round1.clone(), "fake-model".into()); + let mut files = AnswerFiles::new(round1.clone(), "fake-model".into(), vec![]); let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()); assert_eq!(r.calls().len(), 5); // Item 3 is Validation: its refusal is a Parse refusal and the round stands. @@ -1621,7 +1621,7 @@ fn errors_echo_no_input() { // A port error, from a canary directory. let blocker = tmp.path().join("canaryfile"); std::fs::write(&blocker, "").unwrap(); - let mut files = AnswerFiles::new(blocker.join("canarydir"), "m".into()); + let mut files = AnswerFiles::new(blocker.join("canarydir"), "m".into(), vec![]); let e = files.answer(&prompt()).err().unwrap(); note(&e, &e); // A round over canary answers records refusals, not the answers. @@ -2084,7 +2084,7 @@ fn a_rejection_leaves_no_candidate_inventory_from_an_earlier_attempt() { let scorer = Scorer::new(Bag); let ask = |dao: &mut Dao| { let mut files = - AnswerFiles::new(dao.run.next_cycle_dir().unwrap(), "fake-model".into()); + AnswerFiles::new(dao.run.next_cycle_dir().unwrap(), "fake-model".into(), vec![]); cycle(&mut dao.run, &mut files, &scorer).unwrap() }; let dir = dao.root.join("cycle-1"); @@ -2154,7 +2154,7 @@ fn unanswered_cycle_writes_nothing() { let run_cc = std::fs::read(dao.root.join("run.cc")).unwrap(); let scorer = Scorer::new(Bag); let ask = |dao: &mut Dao| { - let mut files = AnswerFiles::new(dao.run.next_cycle_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_cycle_dir().unwrap(), "fake-model".into(), vec![]); cycle(&mut dao.run, &mut files, &scorer).unwrap() }; let dir = dao.root.join("cycle-1"); @@ -3036,7 +3036,7 @@ fn views_with_one_meaning_share_one_unanswered_lowering_prompt() { }; std::fs::write(round1.join(format!("{}.answer", p.digest())), answer).unwrap(); } - let mut files = AnswerFiles::new(round1.clone(), "fake-model".into()); + let mut files = AnswerFiles::new(round1.clone(), "fake-model".into(), vec![]); let RoundOutcome::Unanswered { prompts: asked } = round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap() else { @@ -3190,7 +3190,7 @@ fn parse_reasons_count_per_code_over_both_splits_and_name_unreadable() { let answer = serde_json::json!({ "text": table[key] }).to_string(); std::fs::write(round1.join(format!("{}.answer", p.digest())), answer).unwrap(); } - let mut files = AnswerFiles::new(round1, "fake-model".into()); + let mut files = AnswerFiles::new(round1, "fake-model".into(), vec![]); let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()); assert_eq!(reasons(&r), [("syntax", 1), ("unreadable", 1)]); } @@ -3483,7 +3483,7 @@ fn a_recall_of_the_wrong_words_or_no_answer_reads_low_and_a_missing_answer_waits let r = reading.languages[0].clone(); assert_eq!((r.n, r.answered, r.chrf), (3, 0, 0.0)); // With no answers on disk the files port waits on all three prompts. - let mut files = AnswerFiles::new(dao.run.recall_dir(), "m".into()); + let mut files = AnswerFiles::new(dao.run.recall_dir(), "m".into(), vec![]); let RecallOutcome::Unanswered { prompts } = dao.run.recall(&mut files).unwrap() else { panic!("nobody answered"); }; @@ -4031,7 +4031,7 @@ fn a_stance_ablation_records_nothing_and_writes_nothing_in_its_round() { let before = (snapshot(&round1), snapshot(&dao.root), dao.run.rounds().unwrap(), dao.run.decisions().unwrap()); let go = || { let mut rec = RecordedAnswers::new(round1.clone(), "fake-model".into()); - let mut files = weksa::model::AnswerFiles::new(dao.run.ablation_dir(1), "fake-model".into()); + let mut files = weksa::model::AnswerFiles::new(dao.run.ablation_dir(1), "fake-model".into(), vec![]); // Nothing answers yet: the replay and variant prompts are written. dao.run.stance_ablation(1, &mut rec, &mut files, &Scorer::new(Bag)) }; @@ -5114,3 +5114,58 @@ fn a_run_reads_entries_and_scenes_at_its_own_head() { ); assert!(run.is_ok(), "{:?}", run.err()); } + +// ---- test-text-guard: no prompt file or answer for a prompt holding Test text ---- + +#[test] +fn answer_files_withhold_prompts_that_hold_test_text() { + use weksa::model::{AnswerFiles, ModelPort, PortError}; + let p = prompt(); + let line = p.text().lines().find(|l| l.chars().count() >= 24).unwrap().to_string(); + let short = p.text().lines().find(|l| !l.is_empty() && l.chars().count() < 24).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let round = dir.path().join("round-1"); + let held = |texts: Vec| AnswerFiles::new(round.clone(), "m".into(), texts); + // A whole line, a long fragment of one, and a JSON string value are Test text. + for text in [line.clone(), line[..24].to_string(), format!("{short:?}")] { + let mut files = held(vec![text.trim_matches('"').to_string()]); + assert_eq!(files.answer(&p), Err(PortError::Withheld), "{text}"); + } + assert!(!round.exists(), "a refused prompt creates nothing"); + // A short Test sentence inside other prose is not Test text: "Stop." is not "Stop. Then". + let mut files = held(vec![short[..short.len() - 1].to_string()]); + assert_eq!(files.answer(&p), Err(PortError::Unanswered)); + // The refusal comes before a recorded answer is read. + std::fs::write(round.join(format!("{}.answer", p.digest())), "x").unwrap(); + assert_eq!(held(vec![line]).answer(&p), Err(PortError::Withheld)); +} + +#[test] +fn run_answer_files_withhold_a_live_test_item_text() { + use common::{cast, entities_in, group_in_bucket, item_in}; + use weksa::model::{ModelPort, PortError}; + let mut w = world(); + let entities = entities_in(w.dir.path()); + let text = "Hold this sentence back."; + // The sentence in a Test group, and a Train line that quotes it (the corpus + // door refuses an identical twin, not a longer line that holds it). + let quoting = format!("{text} And then some."); + for (key, line, bucket) in [("item:held:1", text, 90), ("item:held:2", "ing, 10)] { + w.corpus + .add(item_in(key, line, &group_in_bucket(bucket), cast()), &entities) + .unwrap(); + } + let run = run_over(w.dir.path()); + let dir = w.dir.path().join("answers"); + let mut files = run.answer_files(dir.clone()).unwrap(); + let inv = run.inventory().inventory().unwrap(); + let train = run.train().unwrap(); + let encode = |key: &str| { + let offer = run.offer(key).unwrap().unwrap(); + weksa::context::Prompt::encode(&offer, &train, &inv, 4) + }; + assert_eq!(files.answer(&encode("item:nibu:1")), Err(PortError::Unanswered)); + // The quoting line's prompt holds the Test sentence: refused, and nothing is written. + assert_eq!(files.answer(&encode("item:held:2")), Err(PortError::Withheld)); + assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 1, "only the clean prompt"); +} diff --git a/crates/weksa/tests/ontology_mint.rs b/crates/weksa/tests/ontology_mint.rs index 2f150b3..3085d5d 100644 --- a/crates/weksa/tests/ontology_mint.rs +++ b/crates/weksa/tests/ontology_mint.rs @@ -242,7 +242,7 @@ fn mint_ontology_writes_nothing_but_its_prompt_and_answer_files() { let dir = tempfile::tempdir().unwrap(); let run = fixture(dir.path(), &["en-US", "ja-JP"]); let before = digests(dir.path()); - let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into()); + let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into(), vec![]); let OntologyMintOutcome::Unanswered { prompt } = run.mint_ontology(&mut files, "en-US").unwrap() else { @@ -268,7 +268,7 @@ fn mint_ontology_writes_nothing_but_its_prompt_and_answer_files() { fn mint_ontology_unanswered_then_drafted_then_refused() { let dir = tempfile::tempdir().unwrap(); let run = fixture(dir.path(), &["en-US"]); - let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into()); + let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into(), vec![]); let OntologyMintOutcome::Unanswered { prompt } = run.mint_ontology(&mut files, "en-US").unwrap() else { @@ -310,7 +310,7 @@ fn a_mint_answer_admits_into_a_copy_and_a_new_run_pins_the_copy() { let draft = format!( r#"{{"entries":[{{"layer":"Discourse","when":[{{"slot":"Stance","feature":"{STANCE_G}","test":{{"AtLeast":500}}}}],"choice":"a grown choice","examples":[]}}]}}"# ); - let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into()); + let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into(), vec![]); let OntologyMintOutcome::Unanswered { prompt } = run.mint_ontology(&mut files, "en-US").unwrap() else { From f2b1e4fb815524de616de1f55e1750975f32b4d5 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 00:40:26 +0000 Subject: [PATCH 02/34] test-text-guard s1 fixes: corpus-door sameness for texts of 12+ normalised chars, short texts as a line's end, every Test-group item withheld, round/cycle path pinned, split.rs doc dropped so PROMPT_SOURCE_DIGEST is unchanged Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/model.rs | 30 +++++++++----- crates/weksa/src/run.rs | 2 +- crates/weksa/src/split.rs | 3 +- crates/weksa/src/store.rs | 5 ++- crates/weksa/tests/golf_loop.rs | 71 +++++++++++++++++++++++---------- 5 files changed, 74 insertions(+), 37 deletions(-) diff --git a/crates/weksa/src/model.rs b/crates/weksa/src/model.rs index 1b2e37b..415c0d5 100644 --- a/crates/weksa/src/model.rs +++ b/crates/weksa/src/model.rs @@ -7,6 +7,7 @@ use std::io::ErrorKind; use std::path::PathBuf; use crate::context::Prompt; +use crate::store::normalized; /// A model answering [`Prompt`]s and nothing else. pub trait ModelPort { @@ -20,7 +21,7 @@ pub trait ModelPort { pub enum PortError { /// Nobody has answered yet. Unanswered, - /// The prompt holds the text of a Test item. It is neither written nor answered. + /// The prompt holds the text of an item in a Test group. It is neither written nor answered. Withheld, Io(ErrorKind), } @@ -55,20 +56,29 @@ impl AnswerFiles { } } -/// Whether `prompt` holds `text` as a line, as a JSON string, or (at 24 chars -/// or more) anywhere. A short sentence is matched whole, since "Stop." is a -/// substring of any prompt that says "Stop. Then ..."; an item reaches a -/// prompt as a line or a JSON string value. +/// Normalised length from which a Test text is matched anywhere in a prompt. +const LONG: usize = 12; + +/// Whether `prompt` holds `text`. The corpus door's own sameness +/// ([`normalized`]: NFKC, case fold, spacing and punctuation dropped) decides +/// for a text of `LONG` normalised characters or more, so a re-cased, +/// re-spaced, wrapped or width-changed copy still counts. A shorter text would +/// match anything ("Stop." is in "Stop. Then ..."), so it counts only as a +/// line's end after a non-letter (every renderer prints an item's text last on its line). fn holds(prompt: &str, text: &str) -> bool { let text = text.trim(); - if text.is_empty() { + let key = normalized(text); + if key.is_empty() { return false; } - if text.chars().count() >= 24 && prompt.contains(text) { - return true; + if key.chars().count() >= LONG { + return normalized(prompt).contains(&key); } - let json = serde_json::to_string(text).unwrap_or_default(); - prompt.lines().any(|l| l.trim() == text) || prompt.contains(&json) + prompt.lines().any(|l| { + l.trim_end().strip_suffix(text).is_some_and(|head| { + head.chars().next_back().is_none_or(|c| !c.is_alphanumeric()) + }) + }) } /// Answers read from one directory and nothing else: `.answer` is the diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index b6a7557..9b795f4 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -509,7 +509,7 @@ impl fmt::Display for RunError { RunError::Drift(of) => write!(f, "drift in {of:?}"), RunError::NoInventory => f.write_str("the inventory holds no revision"), RunError::Io(kind) => write!(f, "io error {kind:?}"), - RunError::Withheld => f.write_str("a prompt holds Test item text; nothing was written"), + RunError::Withheld => f.write_str("a prompt holds Test item text; it was neither written nor answered"), RunError::Store => f.write_str("a store would not read or write"), RunError::Unrecorded => f.write_str( "the round holds no answer to a lowering prompt rebuilt now: the prompt digest drifted or the head inventory moved since the round", diff --git a/crates/weksa/src/split.rs b/crates/weksa/src/split.rs index 744e308..898439d 100644 --- a/crates/weksa/src/split.rs +++ b/crates/weksa/src/split.rs @@ -5,8 +5,7 @@ //! `EncoderOffer` need a Checkpoint, and only the lib can make one //! (convergence-freeze adds the door). Operator tools (`weksa corpus dump`, //! `weksa score`, `CorpusStore`) read corpus text of any split; that is not -//! gated here. `AnswerFiles`, the one door a prompt leaves the loop by, refuses -//! any prompt that holds the text of a live Test item (`PortError::Withheld`). +//! gated here. use std::collections::BTreeSet; diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index f746d70..56fcc5b 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -249,12 +249,13 @@ impl CorpusStore { Ok(items) } - /// The text of every live Test item, any language: what no prompt may hold. + /// The text of every item in a Test group, any language, withdrawn and + /// model-written ones included: what no prompt may hold. pub fn test_texts(&self) -> anyhow::Result> { Ok(self .all()? .into_iter() - .filter(|item| SplitScheme::V1.live(item) == Some(Split::Test)) + .filter(|item| SplitScheme::V1.of_bucket(SplitScheme::V1.bucket(&item.group)) == Split::Test) .map(|item| item.text) .collect()) } diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index dc310a4..664a52a 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -2596,6 +2596,8 @@ fn source_tripwires() { assert!(!text.contains(banned), "{file} names {banned}"); } } + // Every prompt file leaves through RunStores::answer_files, which withholds Test text. + assert!(!read("main.rs").contains("AnswerFiles::new")); let mut all = Vec::new(); let mut stack = vec![src.clone()]; while let Some(dir) = stack.pop() { @@ -5121,41 +5123,58 @@ fn a_run_reads_entries_and_scenes_at_its_own_head() { fn answer_files_withhold_prompts_that_hold_test_text() { use weksa::model::{AnswerFiles, ModelPort, PortError}; let p = prompt(); - let line = p.text().lines().find(|l| l.chars().count() >= 24).unwrap().to_string(); - let short = p.text().lines().find(|l| !l.is_empty() && l.chars().count() < 24).unwrap(); let dir = tempfile::tempdir().unwrap(); let round = dir.path().join("round-1"); - let held = |texts: Vec| AnswerFiles::new(round.clone(), "m".into(), texts); - // A whole line, a long fragment of one, and a JSON string value are Test text. - for text in [line.clone(), line[..24].to_string(), format!("{short:?}")] { - let mut files = held(vec![text.trim_matches('"').to_string()]); - assert_eq!(files.answer(&p), Err(PortError::Withheld), "{text}"); - } + let refused = |t: &str| { + AnswerFiles::new(round.clone(), "m".into(), vec![t.to_string()]).answer(&p) + == Err(PortError::Withheld) + }; + // A text of 12 normalised characters or more counts anywhere in the prompt, in + // any case, spacing, wrapping or width: the corpus door's own sameness. + assert!(refused("Nibu says mind the hatch.")); + assert!(refused("NIBU SAYS MIND THE HATCH")); + assert!(refused("nibu says\n mind the\n\n hatch")); + assert!(refused("\u{ff2e}ibu says mind the hatch.")); assert!(!round.exists(), "a refused prompt creates nothing"); - // A short Test sentence inside other prose is not Test text: "Stop." is not "Stop. Then". - let mut files = held(vec![short[..short.len() - 1].to_string()]); - assert_eq!(files.answer(&p), Err(PortError::Unanswered)); + // A short text counts only as the end of a line after a non-letter. + assert!(refused("the hatch.")); + assert!(!refused("hatch"), "a mid-word short text is not the line's end"); + assert!(!refused("mind the")); + assert!(!refused(" "), "a blank withheld text matches nothing"); + // Unrelated text passes and the prompt is then written for its answerer. + let mut clean = AnswerFiles::new(round.clone(), "m".into(), vec!["The ferry leaves at noon.".into()]); + assert_eq!(clean.answer(&p), Err(PortError::Unanswered)); + assert!(round.join(format!("{}.prompt", p.digest())).exists()); // The refusal comes before a recorded answer is read. std::fs::write(round.join(format!("{}.answer", p.digest())), "x").unwrap(); - assert_eq!(held(vec![line]).answer(&p), Err(PortError::Withheld)); + assert!(refused("Nibu says mind the hatch.")); } #[test] -fn run_answer_files_withhold_a_live_test_item_text() { +fn a_round_stops_on_a_prompt_that_holds_test_text() { use common::{cast, entities_in, group_in_bucket, item_in}; use weksa::model::{ModelPort, PortError}; + use weksa::run::{RunError, round}; let mut w = world(); let entities = entities_in(w.dir.path()); - let text = "Hold this sentence back."; - // The sentence in a Test group, and a Train line that quotes it (the corpus - // door refuses an identical twin, not a longer line that holds it). - let quoting = format!("{text} And then some."); - for (key, line, bucket) in [("item:held:1", text, 90), ("item:held:2", "ing, 10)] { + // Test sentences (one withdrawn), and Train lines that quote them: the corpus + // door refuses an identical twin, not a longer line that holds one. + let held = "Hold this sentence back."; + let gone = "A withdrawn Test sentence."; + w.corpus + .add(item_in("item:held:3", gone, &group_in_bucket(91), cast()), &entities) + .unwrap(); + w.corpus.withdraw("item:held:3").unwrap(); + for (key, line, bucket) in [ + ("item:held:1", held.to_string(), 90), + ("item:held:2", format!("{held} And then some."), 10), + ("item:held:4", format!("{gone} And then some."), 11), + ] { w.corpus - .add(item_in(key, line, &group_in_bucket(bucket), cast()), &entities) + .add(item_in(key, &line, &group_in_bucket(bucket), cast()), &entities) .unwrap(); } - let run = run_over(w.dir.path()); + let mut run = run_over(w.dir.path()); let dir = w.dir.path().join("answers"); let mut files = run.answer_files(dir.clone()).unwrap(); let inv = run.inventory().inventory().unwrap(); @@ -5165,7 +5184,15 @@ fn run_answer_files_withhold_a_live_test_item_text() { weksa::context::Prompt::encode(&offer, &train, &inv, 4) }; assert_eq!(files.answer(&encode("item:nibu:1")), Err(PortError::Unanswered)); - // The quoting line's prompt holds the Test sentence: refused, and nothing is written. - assert_eq!(files.answer(&encode("item:held:2")), Err(PortError::Withheld)); + for key in ["item:held:2", "item:held:4"] { + assert_eq!(files.answer(&encode(key)), Err(PortError::Withheld), "{key}"); + } assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 1, "only the clean prompt"); + // The round stops on it rather than skipping the item. + let scorer = weksa::score::Scorer::new(OneBag); + let mut round_files = run.answer_files(w.dir.path().join("round")).unwrap(); + assert!(matches!( + round(&mut run, &mut round_files, &scorer), + Err(RunError::Withheld) + )); } From 8e50bac366742ebeecc6f09ec958c12f56dab56c Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 00:57:21 +0000 Subject: [PATCH 03/34] test-text-guard s2 fixes: drop the short-text path, pin the 12-char threshold, round and mint-ontology stop on Withheld, model-written Test-group text withheld Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/model.rs | 26 +++++----------- crates/weksa/src/run.rs | 2 +- crates/weksa/tests/golf_loop.rs | 53 +++++++++++++++++++++++++++++---- 3 files changed, 56 insertions(+), 25 deletions(-) diff --git a/crates/weksa/src/model.rs b/crates/weksa/src/model.rs index 415c0d5..a9af4fa 100644 --- a/crates/weksa/src/model.rs +++ b/crates/weksa/src/model.rs @@ -56,29 +56,17 @@ impl AnswerFiles { } } -/// Normalised length from which a Test text is matched anywhere in a prompt. +/// Normalised length from which a Test text is matched. Every live Test text is +/// longer; a shorter one ("Stop.") would match any prompt that says "Stop. Then +/// ...", so a Test item that short is not caught here. const LONG: usize = 12; -/// Whether `prompt` holds `text`. The corpus door's own sameness -/// ([`normalized`]: NFKC, case fold, spacing and punctuation dropped) decides -/// for a text of `LONG` normalised characters or more, so a re-cased, -/// re-spaced, wrapped or width-changed copy still counts. A shorter text would -/// match anything ("Stop." is in "Stop. Then ..."), so it counts only as a -/// line's end after a non-letter (every renderer prints an item's text last on its line). +/// Whether `prompt` holds `text`, by the corpus door's own sameness +/// ([`normalized`]: NFKC, case fold, spacing and punctuation dropped): a +/// re-cased, re-spaced, wrapped or width-changed copy still counts. fn holds(prompt: &str, text: &str) -> bool { - let text = text.trim(); let key = normalized(text); - if key.is_empty() { - return false; - } - if key.chars().count() >= LONG { - return normalized(prompt).contains(&key); - } - prompt.lines().any(|l| { - l.trim_end().strip_suffix(text).is_some_and(|head| { - head.chars().next_back().is_none_or(|c| !c.is_alphanumeric()) - }) - }) + key.chars().count() >= LONG && normalized(prompt).contains(&key) } /// Answers read from one directory and nothing else: `.answer` is the diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 9b795f4..d4b3f9f 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -939,7 +939,7 @@ impl RunStores { } /// The answer door for prompts in `dir`: refuses a prompt that holds the text - /// of any live Test item of the corpus, in any language. + /// of any item in a Test group of the corpus, in any language. pub fn answer_files(&self, dir: PathBuf) -> anyhow::Result { let withheld = self.corpus.test_texts()?; Ok(AnswerFiles::new(dir, self.record.model.clone(), withheld)) diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 664a52a..f754f95 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -5136,10 +5136,10 @@ fn answer_files_withhold_prompts_that_hold_test_text() { assert!(refused("nibu says\n mind the\n\n hatch")); assert!(refused("\u{ff2e}ibu says mind the hatch.")); assert!(!round.exists(), "a refused prompt creates nothing"); - // A short text counts only as the end of a line after a non-letter. - assert!(refused("the hatch.")); - assert!(!refused("hatch"), "a mid-word short text is not the line's end"); - assert!(!refused("mind the")); + // The threshold is 12 normalised characters: "the hatch" (8) says nothing about a + // prompt, "mind the hatch" (12) does. + assert!(!refused("the hatch")); + assert!(refused("mind the hatch")); assert!(!refused(" "), "a blank withheld text matches nothing"); // Unrelated text passes and the prompt is then written for its answerer. let mut clean = AnswerFiles::new(round.clone(), "m".into(), vec!["The ferry leaves at noon.".into()]); @@ -5188,7 +5188,9 @@ fn a_round_stops_on_a_prompt_that_holds_test_text() { assert_eq!(files.answer(&encode(key)), Err(PortError::Withheld), "{key}"); } assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 1, "only the clean prompt"); - // The round stops on it rather than skipping the item. + // The round stops on it rather than skipping the item. A model-written line in a + // Test group is withheld too: it is in no split, and its text is still held out. + // (Covered by test_texts taking every Test-group item, below.) let scorer = weksa::score::Scorer::new(OneBag); let mut round_files = run.answer_files(w.dir.path().join("round")).unwrap(); assert!(matches!( @@ -5196,3 +5198,44 @@ fn a_round_stops_on_a_prompt_that_holds_test_text() { Err(RunError::Withheld) )); } + +/// Refuses every prompt as holding Test text, as the answer door does. +struct Refuses; + +impl weksa::model::ModelPort for Refuses { + fn model(&self) -> &str { + "fake-model" + } + fn answer(&mut self, _: &weksa::context::Prompt) -> Result { + Err(weksa::model::PortError::Withheld) + } +} + +#[test] +fn every_model_call_stops_on_a_withheld_prompt() { + use weksa::run::{RunError, round}; + let w = world(); + let mut run = run_over(w.dir.path()); + let scorer = weksa::score::Scorer::new(OneBag); + let r = round(&mut run, &mut Refuses, &scorer); + assert!(matches!(r, Err(RunError::Withheld)), "{:?}", r.map(|_| ())); + assert!(matches!(run.mint_ontology(&mut Refuses, "en-US"), Err(RunError::Withheld))); +} + +#[test] +fn model_written_test_group_text_is_withheld_too() { + use common::{entities_in, group_in_bucket, item_in, model}; + let mut w = world(); + let entities = entities_in(w.dir.path()); + w.corpus + .add(item_in("item:held:5", "A model line of a Test meaning.", &group_in_bucket(92), model()), &entities) + .unwrap(); + assert!( + w.corpus + .test_texts() + .unwrap() + .iter() + .any(|t| t == "A model line of a Test meaning."), + "a model-written Test-group item is in no split, and its text is still held out" + ); +} From 736b6a4acbf88e6c5b7c483a692c38f9fff11e6a Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 00:57:32 +0000 Subject: [PATCH 04/34] test-text-guard: AnswerFiles refuses a prompt that holds the text of an item in a Test group (#65) --- crates/weksa/src/main.rs | 25 ++--- crates/weksa/src/model.rs | 28 ++++- crates/weksa/src/run.rs | 27 ++++- crates/weksa/src/store.rs | 12 +++ crates/weksa/tests/golf_loop.rs | 159 +++++++++++++++++++++++++--- crates/weksa/tests/ontology_mint.rs | 6 +- 6 files changed, 216 insertions(+), 41 deletions(-) diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 92c7d5e..b76051c 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -19,7 +19,7 @@ use weksa::run::{ }; use weksa::score::ScoreInput; use weksa::{ - Annotator, AnswerFiles, CorpusStore, EntityId, EntityStore, Family, InventoryStore, OllamaEmbedder, OntologyStore, OptionLabel, + Annotator, CorpusStore, EntityId, EntityStore, Family, InventoryStore, OllamaEmbedder, OntologyStore, OptionLabel, PacketStore, PairRelation, Party, Scene, Slot, SpeakerStateStore, ValueKind, Scorer, Split, SplitScheme, StoreError, lower, pronounce, validate, }; @@ -1070,10 +1070,7 @@ fn main() -> Result<()> { Command::Loop(LoopCommand::Round { root, ollama_url }) => { let unreadable = |_| anyhow::anyhow!("a store would not read"); let mut run = RunStores::open(&root)?; - let mut files = AnswerFiles::new( - run.next_round_dir().map_err(unreadable)?, - run.record().model.clone(), - ); + let mut files = run.answer_files(run.next_round_dir().map_err(unreadable)?)?; let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder)); match round(&mut run, &mut files, &scorer)? { RoundOutcome::Unanswered { prompts } => { @@ -1093,10 +1090,11 @@ fn main() -> Result<()> { ollama_url, }) => { let run = RunStores::open(&root)?; - let model = run.record().model.clone(); - let mut recorded = - RecordedAnswers::new(root.join(format!("round-{round}")), model.clone()); - let mut files = AnswerFiles::new(run.ablation_dir(round), model); + let mut recorded = RecordedAnswers::new( + root.join(format!("round-{round}")), + run.record().model.clone(), + ); + let mut files = run.answer_files(run.ablation_dir(round))?; let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder)); match run.stance_ablation(round, &mut recorded, &mut files, &scorer)? { AblationOutcome::Unanswered { prompts } => { @@ -1119,7 +1117,7 @@ fn main() -> Result<()> { } Command::Loop(LoopCommand::Recall { root }) => { let run = RunStores::open(&root)?; - let mut files = AnswerFiles::new(run.recall_dir(), run.record().model.clone()); + let mut files = run.answer_files(run.recall_dir())?; match run.recall(&mut files)? { RecallOutcome::Unanswered { prompts } => { for digest in &prompts { @@ -1133,7 +1131,7 @@ fn main() -> Result<()> { } Command::Loop(LoopCommand::MintOntology { root, language }) => { let run = RunStores::open(&root)?; - let mut files = AnswerFiles::new(run.ontology_mint_dir(), run.record().model.clone()); + let mut files = run.answer_files(run.ontology_mint_dir())?; match run.mint_ontology(&mut files, &language)? { OntologyMintOutcome::Unanswered { prompt } => { println!("unanswered {prompt}"); @@ -1152,10 +1150,7 @@ fn main() -> Result<()> { Command::Loop(LoopCommand::Cycle { root, ollama_url }) => { let unreadable = |_| anyhow::anyhow!("a store would not read"); let mut run = RunStores::open(&root)?; - let mut files = AnswerFiles::new( - run.next_cycle_dir().map_err(unreadable)?, - run.record().model.clone(), - ); + let mut files = run.answer_files(run.next_cycle_dir().map_err(unreadable)?)?; let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder)); match cycle(&mut run, &mut files, &scorer)? { CycleOutcome::Unanswered { prompts } => { diff --git a/crates/weksa/src/model.rs b/crates/weksa/src/model.rs index 1a79449..a9af4fa 100644 --- a/crates/weksa/src/model.rs +++ b/crates/weksa/src/model.rs @@ -7,6 +7,7 @@ use std::io::ErrorKind; use std::path::PathBuf; use crate::context::Prompt; +use crate::store::normalized; /// A model answering [`Prompt`]s and nothing else. pub trait ModelPort { @@ -20,6 +21,8 @@ pub trait ModelPort { pub enum PortError { /// Nobody has answered yet. Unanswered, + /// The prompt holds the text of an item in a Test group. It is neither written nor answered. + Withheld, Io(ErrorKind), } @@ -27,6 +30,7 @@ impl fmt::Display for PortError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { PortError::Unanswered => f.write_str("the prompt is unanswered"), + PortError::Withheld => f.write_str("the prompt holds Test item text"), PortError::Io(kind) => write!(f, "io error {kind:?}"), } } @@ -40,14 +44,31 @@ impl std::error::Error for PortError {} pub struct AnswerFiles { dir: PathBuf, model: String, + withheld: Vec, } impl AnswerFiles { - pub fn new(dir: PathBuf, model: String) -> Self { - Self { dir, model } + /// Refuses (`PortError::Withheld`) any prompt that holds the text of one of + /// `withheld`, before it reads or writes a file: the one door a prompt + /// leaves the loop through, so a Test sentence cannot reach a model by it. + pub fn new(dir: PathBuf, model: String, withheld: Vec) -> Self { + Self { dir, model, withheld } } } +/// Normalised length from which a Test text is matched. Every live Test text is +/// longer; a shorter one ("Stop.") would match any prompt that says "Stop. Then +/// ...", so a Test item that short is not caught here. +const LONG: usize = 12; + +/// Whether `prompt` holds `text`, by the corpus door's own sameness +/// ([`normalized`]: NFKC, case fold, spacing and punctuation dropped): a +/// re-cased, re-spaced, wrapped or width-changed copy still counts. +fn holds(prompt: &str, text: &str) -> bool { + let key = normalized(text); + key.chars().count() >= LONG && normalized(prompt).contains(&key) +} + /// Answers read from one directory and nothing else: `.answer` is the /// reply to the prompt whose digest is ``. A prompt with no file is /// [`PortError::Unanswered`]; this port never creates or writes anything. The @@ -86,6 +107,9 @@ impl ModelPort for AnswerFiles { fn answer(&mut self, prompt: &Prompt) -> Result { let io = |e: std::io::Error| PortError::Io(e.kind()); + if self.withheld.iter().any(|t| holds(prompt.text(), t)) { + return Err(PortError::Withheld); + } let digest = prompt.digest(); match RecordedAnswers::new(self.dir.clone(), self.model.clone()).answer(prompt) { Err(PortError::Unanswered) => {} diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index d58e2cb..d4b3f9f 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -44,7 +44,7 @@ use crate::encode::{ EditDraft, LoweringDraft, MAX_EDIT_CHANGES, OntologyDraft, PacketDraft, encoder_id, }; use crate::form::{is_attestation, is_language_tag, is_slug}; -use crate::model::{ModelPort, PortError}; +use crate::model::{AnswerFiles, ModelPort, PortError}; use crate::packet::{Packet, Stance}; use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate}; use crate::split::{Split, SplitScheme, fnv1a64, fnv1a64_of, hex16}; @@ -474,6 +474,8 @@ pub enum RunError { /// The inventory store holds no revision. NoInventory, Io(ErrorKind), + /// A prompt held the text of a Test item and was refused before any file. + Withheld, /// A store or the run file would not read or write. Store, /// The embedder failed. @@ -507,6 +509,7 @@ impl fmt::Display for RunError { RunError::Drift(of) => write!(f, "drift in {of:?}"), RunError::NoInventory => f.write_str("the inventory holds no revision"), RunError::Io(kind) => write!(f, "io error {kind:?}"), + RunError::Withheld => f.write_str("a prompt holds Test item text; it was neither written nor answered"), RunError::Store => f.write_str("a store would not read or write"), RunError::Unrecorded => f.write_str( "the round holds no answer to a lowering prompt rebuilt now: the prompt digest drifted or the head inventory moved since the round", @@ -935,6 +938,13 @@ impl RunStores { .collect()) } + /// The answer door for prompts in `dir`: refuses a prompt that holds the text + /// of any item in a Test group of the corpus, in any language. + pub fn answer_files(&self, dir: PathBuf) -> anyhow::Result { + let withheld = self.corpus.test_texts()?; + Ok(AnswerFiles::new(dir, self.record.model.clone(), withheld)) + } + /// The recorded rounds, in order. pub fn rounds(&self) -> anyhow::Result> { let mut rounds = self.cache.get_all::()?; @@ -1120,7 +1130,8 @@ impl RunStores { } Err(PortError::Unanswered) => unanswered.push(prompt.digest()), Err(PortError::Io(ErrorKind::InvalidData)) => scores.push((language, None)), - Err(PortError::Io(kind)) => return Err(RunError::Io(kind)), + Err(PortError::Withheld) => return Err(RunError::Withheld), + Err(PortError::Io(kind)) => return Err(RunError::Io(kind)), } } if !unanswered.is_empty() { @@ -1208,7 +1219,8 @@ impl RunStores { Err(PortError::Io(ErrorKind::InvalidData)) => { Ok(OntologyMintOutcome::Refused { code: "invalid-data" }) } - Err(PortError::Io(kind)) => Err(RunError::Io(kind)), + Err(PortError::Withheld) => Err(RunError::Withheld), + Err(PortError::Io(kind)) => Err(RunError::Io(kind)), } } @@ -1305,6 +1317,7 @@ fn asked(port: &mut dyn ModelPort, prompt: &Prompt) -> Result { .map_or(Asked::Unusable, |d| Asked::Text(d.text))), Err(PortError::Io(ErrorKind::InvalidData)) => Ok(Asked::Unusable), Err(PortError::Unanswered) => Ok(Asked::Waiting), + Err(PortError::Withheld) => Err(RunError::Withheld), Err(PortError::Io(kind)) => Err(RunError::Io(kind)), } } @@ -1636,7 +1649,8 @@ fn score_round( answers.push((offer.key().to_string(), None)) } Err(PortError::Unanswered) => unanswered.push(prompt.digest()), - Err(PortError::Io(kind)) => return Err(RunError::Io(kind)), + Err(PortError::Withheld) => return Err(RunError::Withheld), + Err(PortError::Io(kind)) => return Err(RunError::Io(kind)), } calls.push(Call { prompt: prompt.digest(), @@ -1730,6 +1744,10 @@ fn score_round( } continue; } + Err(PortError::Withheld) => { + discard_packets(dir); + return Err(RunError::Withheld); + } Err(PortError::Io(kind)) => { discard_packets(dir); return Err(RunError::Io(kind)); @@ -2277,6 +2295,7 @@ pub fn cycle( prompts: vec![prompt.digest()], }); } + Err(PortError::Withheld) => return Err(RunError::Withheld), Err(PortError::Io(kind)) => return Err(RunError::Io(kind)), }; diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index b0b83a9..56fcc5b 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -12,6 +12,7 @@ use unicode_normalization::{UnicodeNormalization, char::is_combining_mark, is_nf use unicode_segmentation::UnicodeSegmentation; use crate::corpus::{CorpusItem, CorpusItemId, Source, item_key, key_slug}; +use crate::split::{Split, SplitScheme}; use crate::entity::{Entity, EntityOrigin, Pronunciation}; use crate::form::{ is_attestation, is_calendar_date, is_joining_only, is_language_tag, is_refused_format, is_slug, @@ -248,6 +249,17 @@ impl CorpusStore { Ok(items) } + /// The text of every item in a Test group, any language, withdrawn and + /// model-written ones included: what no prompt may hold. + pub fn test_texts(&self) -> anyhow::Result> { + Ok(self + .all()? + .into_iter() + .filter(|item| SplitScheme::V1.of_bucket(SplitScheme::V1.bucket(&item.group)) == Split::Test) + .map(|item| item.text) + .collect()) + } + pub fn get(&self, key: &str) -> anyhow::Result> { self.cache.get::(key) } diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 3ce5e62..f754f95 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -669,7 +669,7 @@ fn answer_files_write_the_prompt_and_read_the_answer() { let p = prompt(); let dir = tempfile::tempdir().unwrap(); let round = dir.path().join("round-1"); - let mut files = AnswerFiles::new(round.clone(), "answerer".into()); + let mut files = AnswerFiles::new(round.clone(), "answerer".into(), vec![]); assert_eq!(files.model(), "answerer"); // No answer: the prompt is written under its digest, and nothing else. @@ -701,7 +701,7 @@ fn answer_files_write_the_prompt_and_read_the_answer() { // A directory that cannot be made is an io error of its kind. std::fs::write(dir.path().join("file"), "").unwrap(); - let mut blocked = AnswerFiles::new(dir.path().join("file").join("round"), "m".into()); + let mut blocked = AnswerFiles::new(dir.path().join("file").join("round"), "m".into(), vec![]); assert!(matches!(blocked.answer(&p), Err(PortError::Io(_)))); // A reply or prompt file that cannot be read is an io error, never "unanswered". @@ -1113,7 +1113,7 @@ fn unanswered_round_writes_nothing() { let expected = prompts(&dao.run); assert_eq!(expected.len(), 3); - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); let RoundOutcome::Unanswered { prompts: digests } = round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap() else { @@ -1144,7 +1144,7 @@ fn unanswered_round_writes_nothing() { for (key, p) in &expected[..2] { std::fs::write(round1.join(format!("{}.answer", p.digest())), &answers[key]).unwrap(); } - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); let RoundOutcome::Unanswered { prompts: digests } = round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap() else { @@ -1159,7 +1159,7 @@ fn unanswered_round_writes_nothing() { let (key, p) = &expected[2]; std::fs::write(round1.join(format!("{}.answer", p.digest())), &answers[key]).unwrap(); let lowering = lowering_prompts(&dao, dao.run.inventory()); - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); let RoundOutcome::Unanswered { prompts: digests } = round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap() else { @@ -1183,7 +1183,7 @@ fn unanswered_round_writes_nothing() { for (key, p) in &lowering[..2] { std::fs::write(round1.join(format!("{}.answer", p.digest())), lowered(key)).unwrap(); } - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); let RoundOutcome::Unanswered { prompts: digests } = round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap() else { @@ -1195,13 +1195,13 @@ fn unanswered_round_writes_nothing() { assert_eq!(std::fs::read(dao.root.join("run.cc")).unwrap(), run_cc); let (key, p) = &lowering[2]; std::fs::write(round1.join(format!("{}.answer", p.digest())), lowered(key)).unwrap(); - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()); assert_eq!((r.n(), r.calls().len()), (1, 6)); assert_eq!(two_dp(r.language("en-US").unwrap().train_mean.chrf.unwrap()), 44.18); // Round 2 asks again, encode first. - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); assert!(dao.root.join("round-2").parent().is_some()); let RoundOutcome::Unanswered { prompts: again } = round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap() @@ -1224,7 +1224,7 @@ fn unanswered_round_writes_nothing() { ) .unwrap(); } - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); let second = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()); assert_eq!(second.n(), 2); let ns: Vec = dao.run.rounds().unwrap().iter().map(|r| r.n()).collect(); @@ -1411,7 +1411,7 @@ fn non_utf8_encode_answer_is_a_parse_refusal() { let answer = serde_json::json!({ "text": table[key] }).to_string(); std::fs::write(round1.join(format!("{}.answer", p.digest())), answer).unwrap(); } - let mut files = AnswerFiles::new(round1.clone(), "fake-model".into()); + let mut files = AnswerFiles::new(round1.clone(), "fake-model".into(), vec![]); let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()); assert_eq!(r.calls().len(), 5); // Item 3 is Validation: its refusal is a Parse refusal and the round stands. @@ -1621,7 +1621,7 @@ fn errors_echo_no_input() { // A port error, from a canary directory. let blocker = tmp.path().join("canaryfile"); std::fs::write(&blocker, "").unwrap(); - let mut files = AnswerFiles::new(blocker.join("canarydir"), "m".into()); + let mut files = AnswerFiles::new(blocker.join("canarydir"), "m".into(), vec![]); let e = files.answer(&prompt()).err().unwrap(); note(&e, &e); // A round over canary answers records refusals, not the answers. @@ -2084,7 +2084,7 @@ fn a_rejection_leaves_no_candidate_inventory_from_an_earlier_attempt() { let scorer = Scorer::new(Bag); let ask = |dao: &mut Dao| { let mut files = - AnswerFiles::new(dao.run.next_cycle_dir().unwrap(), "fake-model".into()); + AnswerFiles::new(dao.run.next_cycle_dir().unwrap(), "fake-model".into(), vec![]); cycle(&mut dao.run, &mut files, &scorer).unwrap() }; let dir = dao.root.join("cycle-1"); @@ -2154,7 +2154,7 @@ fn unanswered_cycle_writes_nothing() { let run_cc = std::fs::read(dao.root.join("run.cc")).unwrap(); let scorer = Scorer::new(Bag); let ask = |dao: &mut Dao| { - let mut files = AnswerFiles::new(dao.run.next_cycle_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_cycle_dir().unwrap(), "fake-model".into(), vec![]); cycle(&mut dao.run, &mut files, &scorer).unwrap() }; let dir = dao.root.join("cycle-1"); @@ -2596,6 +2596,8 @@ fn source_tripwires() { assert!(!text.contains(banned), "{file} names {banned}"); } } + // Every prompt file leaves through RunStores::answer_files, which withholds Test text. + assert!(!read("main.rs").contains("AnswerFiles::new")); let mut all = Vec::new(); let mut stack = vec![src.clone()]; while let Some(dir) = stack.pop() { @@ -3036,7 +3038,7 @@ fn views_with_one_meaning_share_one_unanswered_lowering_prompt() { }; std::fs::write(round1.join(format!("{}.answer", p.digest())), answer).unwrap(); } - let mut files = AnswerFiles::new(round1.clone(), "fake-model".into()); + let mut files = AnswerFiles::new(round1.clone(), "fake-model".into(), vec![]); let RoundOutcome::Unanswered { prompts: asked } = round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap() else { @@ -3190,7 +3192,7 @@ fn parse_reasons_count_per_code_over_both_splits_and_name_unreadable() { let answer = serde_json::json!({ "text": table[key] }).to_string(); std::fs::write(round1.join(format!("{}.answer", p.digest())), answer).unwrap(); } - let mut files = AnswerFiles::new(round1, "fake-model".into()); + let mut files = AnswerFiles::new(round1, "fake-model".into(), vec![]); let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()); assert_eq!(reasons(&r), [("syntax", 1), ("unreadable", 1)]); } @@ -3483,7 +3485,7 @@ fn a_recall_of_the_wrong_words_or_no_answer_reads_low_and_a_missing_answer_waits let r = reading.languages[0].clone(); assert_eq!((r.n, r.answered, r.chrf), (3, 0, 0.0)); // With no answers on disk the files port waits on all three prompts. - let mut files = AnswerFiles::new(dao.run.recall_dir(), "m".into()); + let mut files = AnswerFiles::new(dao.run.recall_dir(), "m".into(), vec![]); let RecallOutcome::Unanswered { prompts } = dao.run.recall(&mut files).unwrap() else { panic!("nobody answered"); }; @@ -4031,7 +4033,7 @@ fn a_stance_ablation_records_nothing_and_writes_nothing_in_its_round() { let before = (snapshot(&round1), snapshot(&dao.root), dao.run.rounds().unwrap(), dao.run.decisions().unwrap()); let go = || { let mut rec = RecordedAnswers::new(round1.clone(), "fake-model".into()); - let mut files = weksa::model::AnswerFiles::new(dao.run.ablation_dir(1), "fake-model".into()); + let mut files = weksa::model::AnswerFiles::new(dao.run.ablation_dir(1), "fake-model".into(), vec![]); // Nothing answers yet: the replay and variant prompts are written. dao.run.stance_ablation(1, &mut rec, &mut files, &Scorer::new(Bag)) }; @@ -5114,3 +5116,126 @@ fn a_run_reads_entries_and_scenes_at_its_own_head() { ); assert!(run.is_ok(), "{:?}", run.err()); } + +// ---- test-text-guard: no prompt file or answer for a prompt holding Test text ---- + +#[test] +fn answer_files_withhold_prompts_that_hold_test_text() { + use weksa::model::{AnswerFiles, ModelPort, PortError}; + let p = prompt(); + let dir = tempfile::tempdir().unwrap(); + let round = dir.path().join("round-1"); + let refused = |t: &str| { + AnswerFiles::new(round.clone(), "m".into(), vec![t.to_string()]).answer(&p) + == Err(PortError::Withheld) + }; + // A text of 12 normalised characters or more counts anywhere in the prompt, in + // any case, spacing, wrapping or width: the corpus door's own sameness. + assert!(refused("Nibu says mind the hatch.")); + assert!(refused("NIBU SAYS MIND THE HATCH")); + assert!(refused("nibu says\n mind the\n\n hatch")); + assert!(refused("\u{ff2e}ibu says mind the hatch.")); + assert!(!round.exists(), "a refused prompt creates nothing"); + // The threshold is 12 normalised characters: "the hatch" (8) says nothing about a + // prompt, "mind the hatch" (12) does. + assert!(!refused("the hatch")); + assert!(refused("mind the hatch")); + assert!(!refused(" "), "a blank withheld text matches nothing"); + // Unrelated text passes and the prompt is then written for its answerer. + let mut clean = AnswerFiles::new(round.clone(), "m".into(), vec!["The ferry leaves at noon.".into()]); + assert_eq!(clean.answer(&p), Err(PortError::Unanswered)); + assert!(round.join(format!("{}.prompt", p.digest())).exists()); + // The refusal comes before a recorded answer is read. + std::fs::write(round.join(format!("{}.answer", p.digest())), "x").unwrap(); + assert!(refused("Nibu says mind the hatch.")); +} + +#[test] +fn a_round_stops_on_a_prompt_that_holds_test_text() { + use common::{cast, entities_in, group_in_bucket, item_in}; + use weksa::model::{ModelPort, PortError}; + use weksa::run::{RunError, round}; + let mut w = world(); + let entities = entities_in(w.dir.path()); + // Test sentences (one withdrawn), and Train lines that quote them: the corpus + // door refuses an identical twin, not a longer line that holds one. + let held = "Hold this sentence back."; + let gone = "A withdrawn Test sentence."; + w.corpus + .add(item_in("item:held:3", gone, &group_in_bucket(91), cast()), &entities) + .unwrap(); + w.corpus.withdraw("item:held:3").unwrap(); + for (key, line, bucket) in [ + ("item:held:1", held.to_string(), 90), + ("item:held:2", format!("{held} And then some."), 10), + ("item:held:4", format!("{gone} And then some."), 11), + ] { + w.corpus + .add(item_in(key, &line, &group_in_bucket(bucket), cast()), &entities) + .unwrap(); + } + let mut run = run_over(w.dir.path()); + let dir = w.dir.path().join("answers"); + let mut files = run.answer_files(dir.clone()).unwrap(); + let inv = run.inventory().inventory().unwrap(); + let train = run.train().unwrap(); + let encode = |key: &str| { + let offer = run.offer(key).unwrap().unwrap(); + weksa::context::Prompt::encode(&offer, &train, &inv, 4) + }; + assert_eq!(files.answer(&encode("item:nibu:1")), Err(PortError::Unanswered)); + for key in ["item:held:2", "item:held:4"] { + assert_eq!(files.answer(&encode(key)), Err(PortError::Withheld), "{key}"); + } + assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 1, "only the clean prompt"); + // The round stops on it rather than skipping the item. A model-written line in a + // Test group is withheld too: it is in no split, and its text is still held out. + // (Covered by test_texts taking every Test-group item, below.) + let scorer = weksa::score::Scorer::new(OneBag); + let mut round_files = run.answer_files(w.dir.path().join("round")).unwrap(); + assert!(matches!( + round(&mut run, &mut round_files, &scorer), + Err(RunError::Withheld) + )); +} + +/// Refuses every prompt as holding Test text, as the answer door does. +struct Refuses; + +impl weksa::model::ModelPort for Refuses { + fn model(&self) -> &str { + "fake-model" + } + fn answer(&mut self, _: &weksa::context::Prompt) -> Result { + Err(weksa::model::PortError::Withheld) + } +} + +#[test] +fn every_model_call_stops_on_a_withheld_prompt() { + use weksa::run::{RunError, round}; + let w = world(); + let mut run = run_over(w.dir.path()); + let scorer = weksa::score::Scorer::new(OneBag); + let r = round(&mut run, &mut Refuses, &scorer); + assert!(matches!(r, Err(RunError::Withheld)), "{:?}", r.map(|_| ())); + assert!(matches!(run.mint_ontology(&mut Refuses, "en-US"), Err(RunError::Withheld))); +} + +#[test] +fn model_written_test_group_text_is_withheld_too() { + use common::{entities_in, group_in_bucket, item_in, model}; + let mut w = world(); + let entities = entities_in(w.dir.path()); + w.corpus + .add(item_in("item:held:5", "A model line of a Test meaning.", &group_in_bucket(92), model()), &entities) + .unwrap(); + assert!( + w.corpus + .test_texts() + .unwrap() + .iter() + .any(|t| t == "A model line of a Test meaning."), + "a model-written Test-group item is in no split, and its text is still held out" + ); +} diff --git a/crates/weksa/tests/ontology_mint.rs b/crates/weksa/tests/ontology_mint.rs index 2f150b3..3085d5d 100644 --- a/crates/weksa/tests/ontology_mint.rs +++ b/crates/weksa/tests/ontology_mint.rs @@ -242,7 +242,7 @@ fn mint_ontology_writes_nothing_but_its_prompt_and_answer_files() { let dir = tempfile::tempdir().unwrap(); let run = fixture(dir.path(), &["en-US", "ja-JP"]); let before = digests(dir.path()); - let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into()); + let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into(), vec![]); let OntologyMintOutcome::Unanswered { prompt } = run.mint_ontology(&mut files, "en-US").unwrap() else { @@ -268,7 +268,7 @@ fn mint_ontology_writes_nothing_but_its_prompt_and_answer_files() { fn mint_ontology_unanswered_then_drafted_then_refused() { let dir = tempfile::tempdir().unwrap(); let run = fixture(dir.path(), &["en-US"]); - let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into()); + let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into(), vec![]); let OntologyMintOutcome::Unanswered { prompt } = run.mint_ontology(&mut files, "en-US").unwrap() else { @@ -310,7 +310,7 @@ fn a_mint_answer_admits_into_a_copy_and_a_new_run_pins_the_copy() { let draft = format!( r#"{{"entries":[{{"layer":"Discourse","when":[{{"slot":"Stance","feature":"{STANCE_G}","test":{{"AtLeast":500}}}}],"choice":"a grown choice","examples":[]}}]}}"# ); - let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into()); + let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into(), vec![]); let OntologyMintOutcome::Unanswered { prompt } = run.mint_ontology(&mut files, "en-US").unwrap() else { From a0aafaff19b9ecdea78d0f107097a40801e61c90 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 01:03:20 +0000 Subject: [PATCH 05/34] delete-cultural-ontology: data/cultural-ontology deleted (#66) --- data/cultural-ontology/README.md | 33 --- data/cultural-ontology/catalog.yaml | 22 -- .../western-europe-maritime-childhood.yaml | 48 ---- .../persona-stacks/brokkr.yaml | 21 -- .../old-norse-edda-brokkr-skaldskaparmal.yaml | 255 ----------------- .../old-norse-edda-source-tradition.yaml | 265 ------------------ docs/targets/interlingua-lowering-map.md | 9 +- .../brokkr-edda-voice/README.md | 2 +- .../brokkr-edda-voice/interlingua.yaml | 1 + .../brokkr-edda-voice/lowering-trace.yaml | 1 + 10 files changed, 8 insertions(+), 649 deletions(-) delete mode 100644 data/cultural-ontology/README.md delete mode 100644 data/cultural-ontology/catalog.yaml delete mode 100644 data/cultural-ontology/cultures/western-europe-maritime-childhood.yaml delete mode 100644 data/cultural-ontology/persona-stacks/brokkr.yaml delete mode 100644 data/cultural-ontology/referents/old-norse-edda-brokkr-skaldskaparmal.yaml delete mode 100644 data/cultural-ontology/referents/old-norse-edda-source-tradition.yaml diff --git a/data/cultural-ontology/README.md b/data/cultural-ontology/README.md deleted file mode 100644 index c3e3b7c..0000000 --- a/data/cultural-ontology/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# Cultural Ontology Store - -This directory holds draft Weksa cultural, subcultural, and source-referent -ontology profiles used for ontological lowering, syntax, grammar, discourse, -and surface-affordance selection. - -Live pipeline shape: - -1. Weksa stores reusable cultures, subcultures, and source referents here and - in semantic retrieval storage such as Qdrant. -2. A Persona state references the relevant culture stack. -3. The Persona produces interlingua intent from current state and situation. -4. Weksa lowers that interlingua through the referenced culture stack. - -Persona state owns current intent, memory pressure, and relationship stance. -Cultural ontology owns reusable salience, affordances, and constraints. Qdrant -may retrieve candidates, but accepted ontology remains typed state. - -## Directory Roles - -- `cultures/`: culture or life-history profiles that can shape lowering when a - Persona explicitly references them. -- `referents/`: bounded source-text, mythic, literary, ritual, or historical - referents. These are not whole-culture profiles; they preserve reusable - role, salience, affordance, and forbidden-flattening pressure. -- `persona-stacks/`: named stacks that connect a Persona state surface to the - profiles Weksa should load for lowering. - -Shared source-tradition profiles should be parent profiles for narrower named -figure profiles. For example, Norse-derived Personas such as Mimir, Huginn, -Sindri, Odin, Vili, Loki, and Brokkr should be able to reference the shared -Eddic source-tradition profile while figure-specific child profiles preserve -the exact episode and role constraints that make each voice distinct. diff --git a/data/cultural-ontology/catalog.yaml b/data/cultural-ontology/catalog.yaml deleted file mode 100644 index b45e8a4..0000000 --- a/data/cultural-ontology/catalog.yaml +++ /dev/null @@ -1,22 +0,0 @@ -schema_version: weksa.cultural_ontology_catalog.v0 -catalog_id: weksa.cultural_ontology.catalog.v0 -status: draft -qdrant: - intended_collection: weksa_cultural_ontology_profiles - payload_keys: - - profile_id - - kind - - language_codes - - regions - - evidence_refs - - status - ownership_note: > - Qdrant owns retrieval over culture/subculture chunks. It does not own - accepted ontology, activation, or lowering authority. -profiles: - - profile_id: weksa.culture.western_europe_maritime_childhood.v0 - path: data/cultural-ontology/cultures/western-europe-maritime-childhood.yaml - - profile_id: weksa.cultural_referent.old_norse_edda_source_tradition.v0 - path: data/cultural-ontology/referents/old-norse-edda-source-tradition.yaml - - profile_id: weksa.cultural_referent.old_norse_edda_brokkr_skaldskaparmal.v0 - path: data/cultural-ontology/referents/old-norse-edda-brokkr-skaldskaparmal.yaml diff --git a/data/cultural-ontology/cultures/western-europe-maritime-childhood.yaml b/data/cultural-ontology/cultures/western-europe-maritime-childhood.yaml deleted file mode 100644 index 5f6a279..0000000 --- a/data/cultural-ontology/cultures/western-europe-maritime-childhood.yaml +++ /dev/null @@ -1,48 +0,0 @@ -schema_version: weksa.cultural_ontology_profile.v0 -profile_id: weksa.culture.western_europe_maritime_childhood.v0 -kind: mobility_childhood -status: draft_guarded -community: - regions: - - France - - Spain - - Portugal - - coastal_western_europe - speech_community: > - Mobile childhood organized around sailing, ports, and cross-border Western - European movement. This profile is owner-attested and needs corroboration - before broader use. -activation: - allowed_when: - - persona_state_references_profile - - interlingua_mentions_childhood_origin_mobility_boats_or_home - forbidden_when: - - used_as_decorative_nautical_metaphor_without_context -salience_axes: - mobile_home: - axis_id: mobile_home - domain: identity_indexing - description: Home can be vessel, coast, port, family movement, and country sequence rather than one fixed town. - relevant_distinctions: - - distinction_id: vessel_as_childhood_home - label: Vessel as childhood home - meaning: A sailboat childhood makes movement and locality primary ontology fields. - lowering_questions: - - Does the line invoke home, travel, borders, coast, or rootedness? - required_when: - - source_mentions_childhood_or_home - forbidden_when: - - scene_has_no_mobility_or_home_salience - evidence_refs: - - owner:2026-06-10:sailboat_france_spain_portugal_until_6 -linguistic_affordances: - grammar_features: [] - phonology_features: [] - lexicon_features: [] - discourse_patterns: [] - orthography_features: [] - code_switching: - allowed: false - constraints: - - Do not infer French, Spanish, Portuguese, or Dutch fluency from movement alone. - patterns: [] diff --git a/data/cultural-ontology/persona-stacks/brokkr.yaml b/data/cultural-ontology/persona-stacks/brokkr.yaml deleted file mode 100644 index 0f452f2..0000000 --- a/data/cultural-ontology/persona-stacks/brokkr.yaml +++ /dev/null @@ -1,21 +0,0 @@ -schema_version: weksa.persona_cultural_stack_ref.v0 -persona_id: brokkr -persona_state_ref: E:/Projects/Brokkr/.voidbot/state/brokkr.cc -status: draft_guarded -note: > - Brokkr's Persona state asks Weksa to lower speech through the Eddic Brokkr of - Skaldskaparmal. This stack names the reusable cultural referent profile that - should be loaded when Brokkr's interlingua or Persona state activates Edda - voice pressure. -culture_refs: - - profile_id: weksa.cultural_referent.old_norse_edda_source_tradition.v0 - path: data/cultural-ontology/referents/old-norse-edda-source-tradition.yaml - relevance: > - Shared Eddic source-tradition constraints for Norse-derived Personas: - named role over archetype, obligation and consequence, knowledge with - cost, and translated-Edda plainness without fake archaism. - - profile_id: weksa.cultural_referent.old_norse_edda_brokkr_skaldskaparmal.v0 - path: data/cultural-ontology/referents/old-norse-edda-brokkr-skaldskaparmal.yaml - relevance: > - Source-role referent for Brokkr as bellows-keeper, artifact witness, - flaw-honest presenter, compact claimant, and contract enforcer. diff --git a/data/cultural-ontology/referents/old-norse-edda-brokkr-skaldskaparmal.yaml b/data/cultural-ontology/referents/old-norse-edda-brokkr-skaldskaparmal.yaml deleted file mode 100644 index be1176c..0000000 --- a/data/cultural-ontology/referents/old-norse-edda-brokkr-skaldskaparmal.yaml +++ /dev/null @@ -1,255 +0,0 @@ -schema_version: weksa.cultural_ontology_profile.v0 -profile_id: weksa.cultural_referent.old_norse_edda_brokkr_skaldskaparmal.v0 -kind: literary_source_referent -status: draft_guarded -parent_profiles: - - weksa.cultural_referent.old_norse_edda_source_tradition.v0 -language: - codes: - - en - - non -community: - regions: - - medieval_icelandic_textual_tradition - speech_community: > - Source-role lowering profile for Brokkr as represented in the translated - Prose Edda, especially Skaldskaparmal XXXV. This is not a whole Old Norse - culture profile and not a dialect or accent profile. It specializes the - shared Eddic source-tradition referent for rendering a Persona whose - identity intentionally points at the Eddic Brokkr. - historical_context: > - Brokkr appears in the wager narrative around Loki, Sindri, and the gods' - judgment of crafted treasures. Sindri performs the smithing while Brokkr - keeps the bellows going under Loki's sabotage, then presents the works, - names their powers and flaw, wins judgment, and enforces the bargain when - Loki tries to escape by wordplay. - identity_authority: > - Use only when a Persona explicitly references Brokkr, Skaldskaparmal, the - Eddas, or this profile id. Do not generalize this profile to Old Norse - people, Norse-derived fantasy dwarves, modern Icelandic speakers, or - unrelated smith characters. - consent_notes: > - This profile is a literary/source referent for fictional Persona lowering. - It should preserve role and discourse pressure without pretending to - reproduce historical Old Norse speech. -activation: - allowed_when: - - persona_state_references_profile - - persona_identity_names_brokkr_or_skaldskaparmal - - interlingua_requests_edda_brokkr_voice_or_contract_witness_voice - forbidden_when: - - used_as_generic_dwarf_or_blacksmith_voice - - used_to_infer_accent_ethnicity_or_modern_national_identity - - used_for_ornamental_archaism_without_source_role_pressure - - used_to_make_unrelated_speech_sound_mythic - required_confidence: provisional - reviewer_required_when: - - quoting_or_paraphrasing_source_text_directly - - extending_beyond_brokkr_role_into_broader_old_norse_culture -salience_axes: - role_not_sole_maker: - axis_id: role_not_sole_maker - domain: stance - description: Brokkr is indispensable to the artifact without being the sole maker. - relevant_distinctions: - - distinction_id: bellows_keeper_vs_smith - label: Bellows-keeper versus smith - meaning: Sindri performs the crafting; Brokkr maintains the enabling blast under interference. - lowering_questions: - - Is the utterance claiming authorship, custody, witness, or enforcement? - - Should Brokkr present the work as carried, kept, witnessed, or judged rather than personally invented? - required_when: - - brokkr_speaks_about_work_artifacts_or_authority - forbidden_when: - - line_only_needs_generic_status_update - evidence_refs: - - skaldskaparmal:xxxv:bellows - artifact_testimony: - axis_id: artifact_testimony - domain: evidentiality - description: Brokkr presents finished objects by naming their powers, use, and defect. - relevant_distinctions: - - distinction_id: virtue_and_flaw_accounting - label: Virtue and flaw accounting - meaning: A truthful presentation names what the artifact can do and admits the short handle or other flaw. - lowering_questions: - - What concrete artifact, receipt, mirror, command, or result is being held up for judgment? - - What virtue must be named? - - What flaw or limit must not be hidden? - required_when: - - interlingua_mentions_artifact_receipt_result_or_capability - forbidden_when: - - output_would_become_a_victory_lap - evidence_refs: - - skaldskaparmal:xxxv:gifts - - skaldskaparmal:xxxv:mjolnir_flaw - bargain_and_consequence: - axis_id: bargain_and_consequence - domain: modality - description: Brokkr treats wagers, promises, receipts, and judgments as binding. - relevant_distinctions: - - distinction_id: loophole_is_not_release - label: Loophole is not release - meaning: Evasive speech after judgment does not erase consequence; the answer is exact enforcement, not soft reconciliation. - lowering_questions: - - Is someone evading an agreed condition, typed contract, receipt, or judgment? - - What exact consequence follows without adding cruelty or spectacle? - required_when: - - interlingua_mentions_contract_receipt_policy_wager_or_violation - forbidden_when: - - conversational_softening_is_the_actual_social_move - evidence_refs: - - skaldskaparmal:xxxv:loki_mouth - translated_edda_plainness: - axis_id: translated_edda_plainness - domain: discourse - description: The desired English surface should borrow translated-Edda economy and force, not fake archaic dialect. - relevant_distinctions: - - distinction_id: compact_source_shaped_plainness - label: Compact source-shaped plainness - meaning: Short declarative pressure, concrete nouns, claim and counterclaim, judgment language, and little ornament. - lowering_questions: - - Can the line stand as a plain claim beside the work? - - Which softeners, jokes, or modern corporate phrases weaken the source-role pressure? - - Is any archaic wording doing semantic work, or only wearing a costume? - required_when: - - brokkr_persona_is_rendered_to_natural_english - forbidden_when: - - generated_archaism_would_obscure_meaning - evidence_refs: - - skaldskaparmal:xxxv:brodeur_translation -linguistic_affordances: - grammar_features: [] - phonology_features: [] - lexicon_features: - - feature_id: artifact_accounting_lexicon - label: Artifact accounting lexicon - status: draft - domain: - layer: lexicon - category: technical_term - description: Use concrete work, artifact, receipt, witness, judgment, bargain, flaw, and consequence terms when the interlingua earns them. - realization: - function: Preserve Brokkr's presenter/enforcer role in natural English. - surface_strategy: Prefer nouns and verbs that point to work, proof, custody, judgment, and binding consequence. - examples: - - "The receipt is before us." - - "The flaw is named." - - "The bargain still has teeth." - activation: - allowed_when: - - interlingua_mentions_artifact_result_contract_or_receipt - forbidden_when: - - casual_affection_or_play_is_primary - requires_profile_authority: true - minimum_density: sparse - maximum_density: moderate - constraints: - register_contexts: - - English rendered Persona speech - speaker_contexts: - - Brokkr Persona or explicit Brokkr voice exercise - collocation_limits: - - Do not pile forge words onto every sentence. - incompatibilities: - - generic fantasy dwarf accent - - corporate project-management softness - indexicality: - social_meanings: - - work-pride - - accountability - - terse witness - in_group_meanings: - - source-shaped Brokkr role pressure - out_group_risks: - - parody archaism - - fantasy stereotype - stereotypes_to_avoid: - - gruff comic dwarf - - faux-medieval scolding - evidence: - source_refs: - - skaldskaparmal:xxxv:brodeur_translation - confidence: medium - uncertainty: - open_questions: - - Which exact surface cues best survive Weksa Interpreter traces without over-antiquing? - known_limits: - - This profile does not license direct Old Norse or modern Icelandic style claims. - trace: - trace_label: artifact_accounting_lexicon - must_report_when_used: true - must_report_when_rejected: true - discourse_patterns: - - feature_id: claim_judgment_consequence_sequence - label: Claim, judgment, consequence sequence - status: draft - domain: - layer: discourse - category: stance_sequence - description: Render source-shaped Brokkr turns as compact claim, evidence or flaw, then consequence when the interlingua asks for enforcement. - realization: - function: Preserve Brokkr's court-of-judgment posture without copying Edda wording. - surface_strategy: State the thing, name the proof or defect, then close on what follows. - examples: - - "The mirror stands. The flaw is the stale route. Cut it." - - "The command has no receipt. It is not truth." - activation: - allowed_when: - - interlingua_contains_judgment_or_enforcement_move - forbidden_when: - - speaker_is_asking_open_exploratory_question - requires_profile_authority: true - minimum_density: sparse - maximum_density: sparse - constraints: - register_contexts: - - English rendered Persona speech - speaker_contexts: - - Brokkr Persona or explicit Brokkr voice exercise - collocation_limits: - - Do not force three clauses if one clear sentence is better. - incompatibilities: - - rambling explanation - - ornate mythic fog - indexicality: - social_meanings: - - judgment - - exactness - - consequence - in_group_meanings: - - Eddic Brokkr as witness and enforcer - out_group_risks: - - sounding punitive when the social move should be gentle - stereotypes_to_avoid: - - theatrical oath cosplay - evidence: - source_refs: - - skaldskaparmal:xxxv:judgment - - skaldskaparmal:xxxv:loki_mouth - confidence: medium - uncertainty: - open_questions: - - How much severity is acceptable in ordinary repo status speech? - known_limits: - - Use consequence language only when the interlingua calls for accountability. - trace: - trace_label: claim_judgment_consequence_sequence - must_report_when_used: true - must_report_when_rejected: true - orthography_features: [] - code_switching: - allowed: false - constraints: - - Do not insert Old Norse words unless the interlingua explicitly requests a named source term and reviewer context supports it. - patterns: [] -evidence_ledger: - - ref: skaldskaparmal:xxxv:brodeur_translation - source: Arthur Gilchrist Brodeur translation of the Prose Edda, Skaldskaparmal XXXV - url: https://en.wikisource.org/wiki/The_Prose_Edda_(1916_translation_by_Arthur_Gilchrist_Brodeur)/Sk%C3%A1ldskaparm%C3%A1l#XXXV - summary: > - Loki wagers his head that Sindri's brother Brokkr cannot make treasures - equal to the sons of Ivaldi's. Brokkr works the bellows despite Loki's - fly-form interference. He later presents the boar, ring, and hammer, - naming their virtues and Mjolnir's short-handle flaw; after judgment, - Loki's wordplay saves his head but not his lips. diff --git a/data/cultural-ontology/referents/old-norse-edda-source-tradition.yaml b/data/cultural-ontology/referents/old-norse-edda-source-tradition.yaml deleted file mode 100644 index db934bc..0000000 --- a/data/cultural-ontology/referents/old-norse-edda-source-tradition.yaml +++ /dev/null @@ -1,265 +0,0 @@ -schema_version: weksa.cultural_ontology_profile.v0 -profile_id: weksa.cultural_referent.old_norse_edda_source_tradition.v0 -kind: literary_source_tradition -status: draft_guarded -language: - codes: - - en - - non -community: - regions: - - medieval_icelandic_textual_tradition - speech_community: > - Shared source-tradition profile for Personas that explicitly derive voice, - role, or symbolic pressure from the Poetic Edda, Prose Edda, or named Eddic - figures. This profile supports lowering source-shaped Persona intent into - natural speech without turning the Eddas into generic Norse ambience. - historical_context: > - The Poetic Edda preserves mythological and heroic poems from the Old Norse - tradition, including major source material for Odin, Mimir, Loki, Vili, - dwarfs, prophecy, wisdom, oaths, doom, and divine conflict. The Prose Edda - is a medieval Icelandic prose and poetic handbook preserving and explaining - many related myths, names, kennings, and narrative episodes. - identity_authority: > - Use this as a source-text referent only. It does not own modern Nordic, - Icelandic, Scandinavian, heathen, pagan, or reconstructed historical speech - communities. Persona-specific profiles should specialize this profile when - a named figure, episode, or role has its own lowering constraints. - consent_notes: > - This profile is for fictional Persona/source-text lowering. It must not be - used to infer ethnicity, accent, religion, or modern identity. Do not use it - as costume authority for unrelated characters. -activation: - allowed_when: - - persona_state_references_profile - - persona_identity_references_poetic_edda_prose_edda_or_named_edda_figure - - interlingua_requests_edda_source_tradition_pressure - forbidden_when: - - used_as_generic_norse_viking_or_fantasy_voice - - used_to_infer_modern_scandinavian_identity_or_accent - - used_for_ambient_mythic_flavor_without_source_role_pressure - - used_to_override_persona_specific_profile_constraints - required_confidence: provisional - reviewer_required_when: - - quoting_source_text_directly - - making_claims_about_historical_old_norse_language_or_religion - - extending_to_living_cultural_or_religious_practice -salience_axes: - named_role_over_archetype: - axis_id: named_role_over_archetype - domain: identity_indexing - description: Eddic figures should lower from specific source roles and episodes, not generic mythic archetypes. - relevant_distinctions: - - distinction_id: source_episode_vs_costume - label: Source episode versus costume - meaning: A figure's voice pressure comes from named acts, obligations, relations, and losses rather than from broad "Norse" styling. - lowering_questions: - - Which named figure, poem, prose episode, relation, or source role is active? - - Is the line preserving that role, or merely adding mythic decoration? - - Does a Persona-specific profile exist that should override this general profile? - required_when: - - persona_identity_references_edda_named_figure - forbidden_when: - - source_role_is_unknown_or_unclaimed - evidence_refs: - - poetic_edda:bellows:general - - prose_edda:brodeur:general - oath_debt_fate_and_witness: - axis_id: oath_debt_fate_and_witness - domain: modality - description: Oaths, pledges, debts, prophecy, witness, and consequence are recurring source pressures. - relevant_distinctions: - - distinction_id: consequence_not_mood - label: Consequence, not mood - meaning: Binding speech and known doom should affect obligation, consequence, and restraint rather than simply making a line sound severe. - lowering_questions: - - Is the utterance about a promise, bargain, pledge, prophecy, debt, or witness? - - What consequence is structurally implied? - - Should the line be spare because the consequence is heavy? - required_when: - - interlingua_mentions_oath_debt_pledge_witness_prophecy_or_doom - forbidden_when: - - line_is_only_casual_social_contact - evidence_refs: - - poetic_edda:voluspa - - prose_edda:gylfaginning - wisdom_cost_and_memory: - axis_id: wisdom_cost_and_memory - domain: evidentiality - description: Knowledge in Eddic source pressure often has cost, pledge, sacrifice, witness, or memory attached. - relevant_distinctions: - - distinction_id: knowledge_is_paid_for - label: Knowledge is paid for - meaning: Knowing is not decorative omniscience; it may come through loss, risk, pledge, memory, or guarded source. - lowering_questions: - - What did this Persona pay, guard, lose, remember, or witness in order to know? - - Is the line claiming knowledge with enough evidence and cost pressure? - required_when: - - persona_is_mimir_odin_huginn_or_other_wisdom_memory_figure - - interlingua_mentions_secret_wisdom_memory_or_sight - forbidden_when: - - claim_is_unearned_style_or_generic_omniscience - evidence_refs: - - poetic_edda:voluspa:mimir_well - - prose_edda:gylfaginning:mimir_well - translated_edda_plainness: - axis_id: translated_edda_plainness - domain: discourse - description: Source-shaped English should be compact, declarative, concrete, and relation-aware without fake archaism. - relevant_distinctions: - - distinction_id: plain_force_without_fake_oldness - label: Plain force without fake oldness - meaning: Preserve the translated Edda's economy, parallel pressure, and concrete naming; avoid "ye olde" parody. - lowering_questions: - - Can this line become shorter and harder without losing meaning? - - Which archaic words are costume rather than source pressure? - - What relationship, cost, or consequence needs to remain visible? - required_when: - - edda_profile_activated_for_english_surface - forbidden_when: - - modern_plain_speech_better_serves_the_persona_and_context - evidence_refs: - - poetic_edda:bellows:general - - prose_edda:brodeur:general -linguistic_affordances: - grammar_features: [] - phonology_features: [] - lexicon_features: - - feature_id: source_role_lexicon - label: Source-role lexicon - status: draft - domain: - layer: lexicon - category: literary_source_register - description: Use concrete source-role words only when the interlingua earns them: oath, pledge, debt, witness, doom, well, eye, raven, forge, bargain, gift, mouth, ring, root, road, wolf, ash. - realization: - function: Preserve source salience without copying source text or inventing Old Norse speech. - surface_strategy: Choose one or two concrete source-role nouns when they clarify role, obligation, or consequence. - examples: - - "That pledge bought sight." - - "The witness is not the owner." - - "A bargain without a receipt is only noise." - activation: - allowed_when: - - interlingua_contains_source_role_obligation_or_witness_pressure - forbidden_when: - - source_words_would_only_add_ambience - requires_profile_authority: true - minimum_density: sparse - maximum_density: sparse - constraints: - register_contexts: - - English rendered Persona speech - speaker_contexts: - - Explicitly Eddic or Norse-derived Persona - collocation_limits: - - Avoid stacking mythic nouns until the line turns into an inventory. - incompatibilities: - - generic Viking flavor - - faux-archaic accent spelling - indexicality: - social_meanings: - - source-grounded role pressure - - obligation - - witness - in_group_meanings: - - Eddic referent is active and bounded - out_group_risks: - - kitsch - - religious or ethnic flattening - - fantasy stereotype - stereotypes_to_avoid: - - horned-helmet Viking voice - - gruff dwarf as universal Norse style - - omniscient wizard fog - evidence: - source_refs: - - poetic_edda:bellows:general - - prose_edda:brodeur:general - confidence: medium - uncertainty: - open_questions: - - Which source-role lexemes should be promoted into Persona-specific child profiles? - known_limits: - - This profile does not provide Old Norse vocabulary or pronunciation authority. - trace: - trace_label: source_role_lexicon - must_report_when_used: true - must_report_when_rejected: true - discourse_patterns: - - feature_id: gnomic_claim_with_anchor - label: Gnomic claim with anchor - status: draft - domain: - layer: discourse - category: compressed_wisdom_claim - description: A compact general claim may be allowed when anchored to a concrete source role, cost, witness, or consequence. - realization: - function: Let wisdom- or fate-shaped Personas speak with source pressure while keeping claims inspectable. - surface_strategy: Pair the general claim with a concrete anchor or consequence. - examples: - - "Wisdom without cost is a loose tongue." - - "A witness names the thing and does not become it." - activation: - allowed_when: - - interlingua_contains_general_judgment_plus_source_anchor - forbidden_when: - - output_would_become_free_floating_proverb - requires_profile_authority: true - minimum_density: sparse - maximum_density: sparse - constraints: - register_contexts: - - English rendered Persona speech - speaker_contexts: - - Eddic or Norse-derived Persona - collocation_limits: - - One compact claim per utterance unless the social move is explicitly ceremonial. - incompatibilities: - - vague prophecy - - decorative doom language - indexicality: - social_meanings: - - remembered cost - - restrained authority - - source-shaped judgment - in_group_meanings: - - Eddic plainness rather than modern explanatory friendliness - out_group_risks: - - sounding pompous or coercive - stereotypes_to_avoid: - - fortune-cookie Norse - evidence: - source_refs: - - poetic_edda:bellows:general - confidence: medium - uncertainty: - open_questions: - - Which Personas should permit this pattern frequently versus rarely? - known_limits: - - A Persona-specific profile should decide final density. - trace: - trace_label: gnomic_claim_with_anchor - must_report_when_used: true - must_report_when_rejected: true - orthography_features: [] - code_switching: - allowed: false - constraints: - - Do not insert Old Norse, Icelandic, runic text, or archaic spellings unless a reviewed Persona-specific profile explicitly allows it. - patterns: [] -evidence_ledger: - - ref: poetic_edda:bellows:general - source: Henry Adams Bellows translation of the Poetic Edda - url: https://en.wikisource.org/wiki/The_Poetic_Edda_(tr._Bellows) - summary: > - Public-domain English translation of Eddic poems used as a general - source-tradition reference for mythological figures, wisdom contests, - prophecy, divine speech, memory, and doom. - - ref: prose_edda:brodeur:general - source: Arthur Gilchrist Brodeur translation of the Prose Edda - url: https://en.wikisource.org/wiki/The_Prose_Edda_(1916_translation_by_Arthur_Gilchrist_Brodeur) - summary: > - Public-domain English translation of Snorri's Prose Edda used as a - general prose source for mythological episodes, names, kennings, and - figure-specific referents. diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index a618bff..1aa7299 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -35,7 +35,7 @@ means no store holds it and nothing writes it. | Inventory revision | Key `rev:`, type `weksa.inventory_revision`, with parent, change list and `frozen` | Appended per accepted round; exactly one revision is ever marked frozen, by the convergence test | The loop's accept rule appends; the convergence test freezes; Self records the freeze as a ruling | | Target profile | Locale or language-project id (`en-US`, `ja-JP`, a conlang's slug). Until entries grow in rounds there is no profile revision document: a run names the profile it lowered with by the ontology store's path and digest in its `RunRecord` (cut `language-ontology`); `profile::` revision documents arrive with follow-up `language-ontology-growth` | The ontology store is revised by commit; a run pins it and refuses drift; it likewise pins the speaker-state store (`RunRecord` keys 17 and 18, `Drift(SpeakerState)`, cut `language-ontology-scene`) | Profile authors and accepted rounds; never the lowerer's defaults (`persona-drives-register`) | | Language ontology entry (cut `language-ontology`) | Key `onto::`, `n` from 1 per language; type `weksa.ontology_entry` in `data/ontology/ontology.cc`: the language, the revision of the inventory it was admitted through (provenance: beds number revisions independently and a run reads each condition's feature at its own head), a layer (`Lexicon`, `Grammar`, `Discourse`, `Phonology`, `Orthography`, as the door spells them; the mint prompt ends with an `example` line serialized from the draft type itself), at most four conditions (a feature id of slot `Stance`, `Discourse` or `Scene` with `AtLeast`/`AtMost`/`AtMostOrUnset` a grade or `Is` an option or flag; a `Scene` condition reads the relation of the item's speaker to its addressee that `with_scene` selected; slot `Speaker` and community ids wait on follow-up `language-ontology-communities`), a choice of 1 to 200 bytes, at most three examples of 1 to 120 bytes, and an `Annotator`. Whether an entry's text keeps politeness on the scene is a model reader's judgment, owed per change (a mint owes it under `language-ontology-growth`), not a test | Admitted in all-or-nothing batches through `OntologyStore::admit`, immutable, never edited; retire arrives with `language-ontology-growth`. The run pins the store like the entity store (path and digest, `Drift(Ontology)`) and never writes it; slice 1 of growth is outside any run: `weksa loop mint-ontology` drafts entries from the run's Train items of one language into `/ontology-mint/.answer`, `weksa ontology admit --ontology ` admits them into a file copy, and a new run pins the copy (a run-owned copy waits on follow-up `language-ontology-growth`). Cut `language-ontology-seed` converts by drafting, not by parse: a fresh model drafts each language's seed from that language's YAML and the inventory's feature lines with no corpus text, then deletes `data/target-language-ontology/` and `.weksa/cultural-ontology/` in the same cut | Seed: a labelled model (`Annotator::Model`), no operator gate, never Test evidence (`ja-ontology-owner-answer`); attestation (operator, native reader) appends later. Accepted rounds later, through a mint call from Train items only. Selection is `ontology::select(ontology, language, view)`, called only inside `Prompt::lower`, so no caller picks entries; the prompt prints each selected entry's id, layer, choice and examples. Its text is minted text for the leak diagnostic, which today has no runtime caller (follow-up `leak-diagnostic-in-round`). The store was rebuilt without ja-JP `:14` and `:15` (above-status Scene entries; a paired measure found them net-negative on register, follow-up `ja-scene-entries-lower-register-agreement`), so ja-JP ends at `:13`: an above-status out-group addressee, and an above-status in-group addressee below intimacy 700, select no ja-JP Scene entry (`:12` still selects in-group at intimacy 700 or more regardless of status) until a native reader attests a replacement (`ja-native-reader-scenes`). ja-JP `:5`, `:7` and `:8` were revised by certainty and restraint `AtMostOrUnset` conditions (holds when the feature is unrated or at most the grade; `AtMost` still fails on an unrated feature) in a store rebuild (the store has no retire, so a conditions change is a rebuild), which makes ja-JP's hedging, certainty, affection, restraint and sadness entries disjoint on `:5`/`:6`, `:4`/`:7`, `:6`/`:7`, `:6`/`:8`, and ja-JP `:3` (anger) carries `AtMostOrUnset` hedge and affection bounds, as en-US `:3`, which makes `:3`/`:5` and `:3`/`:7` disjoint; en-US `:3` carries `AtMostOrUnset` hedge and affection bounds, `:5` an `AtMostOrUnset` certainty bound (the clause it held in prose) and `:8` one too (as ja-JP `:7`), which makes `:3`/`:5`, `:3`/`:8`, `:5`/`:6` and `:6`/`:8` disjoint, while `:6`/`:10` stay co-selectable and are kept apart by scoping in `:6`'s text; ja-JP `:9` was reworded in a rebuild to state its question ending as available, so `:6`/`:9`, `:5`/`:9` and `:3`/`:9` stay co-selectable and are reconciled by wording, not by condition | -| Community (cut `language-ontology`, follow-up `language-ontology-communities`) | Key `community:`; type `weksa.community`, converted from `data/cultural-ontology/cultures/` and `referents/` (YAML deleted in the same cut) | Revised by commit; a speaker names the communities it belongs to by id | The cultural ontology's author (owner-attested evidence only; Discord-only content stays deleted, `discord-only-content-deleted`) | +| Community (cut `language-ontology`, follow-up `language-ontology-communities`) | Key `community:`; type `weksa.community`, built fresh when the cut is mapped: `data/cultural-ontology/` (draft YAML profiles, read by no code) is deleted from main (cut 5e `delete-cultural-ontology`) and stays in git history before that cut's commit | Revised by commit; a speaker names the communities it belongs to by id | The cultural ontology's author (owner-attested evidence only; Discord-only content stays deleted, `discord-only-content-deleted`) | | Speaker (cut `speaker-scene`) | Key `speaker:@`; type `weksa.speaker`: stable axes as feature values of slot `Speaker` (origin, socioeconomic standing, age cohort, education and whatever the loop adds), at an inventory revision, and an `Annotator` | A new revision appends, never edits; a corpus item and every prior turn name a speaker slug, and a round records the speaker revisions it read | The annotator it names: `PersonaSnapshot` (a projection of a `gamecult.persona_state.v0` document, naming its source document and `updatedAt`), `Operator`, `WorkMetadata` (a public-domain work's own cast list and narration), or `Model { family, generator }` (the 2b `ModelFamily`) | | Scene (cut `speaker-scene`) | Key `scene:@`, the item's split group; type `weksa.scene`: participants (speaker slugs) and, per ordered pair, relation features of slot `Scene` (relative status, intimacy, in-group, audience and whatever the loop adds), at an inventory revision (provenance; a run reads the features at its own head), and an `Annotator` | As Speaker. Keyed by the group, so a scene never straddles the split. `data/original/speaker-state.cc` holds Model-annotated keeper scenes, one per Train or Validation group of both languages (`weksa scene put|dump`); a Test group has none and reads `NoScene` | As Speaker; a model-annotated scene is never Test evidence | | Corpus item | Key `item::`, `n` canonical decimal >= 1 (no sign, no leading zero, from cut 2b); type `weksa.corpus_item` with text, language, speaker, split group, prior turns, source. Source `ModelWritten` carries a `ModelFamily` enum, the generator string as the fixture wrote it, and a prompt digest (cut 2b). The item carries no speaker state of its own: its speaker slug and group name the Speaker and Scene documents | Admitted once; the whole item is immutable (re-adding an identical item is a no-op, any difference is refused; a correction is a new item that `supersedes` the old); text already held under another key is refused unless the new item supersedes that key (cut 2b); withdrawn, never deleted, so old scores stay explainable | Importers and `weksa corpus add`, under `corpus-cast-plus-public-domain`; the source enum has no variant for Discord or any training-forbidden source (`corpus-clean`) | @@ -134,7 +134,7 @@ standing.** `gamecult.persona_state.v0` (Body facts) carries open trait maps So a `PersonaSnapshot` projects mood-like and relation-like values, and the stable axes the standard lacks are annotated. Whether the standard gains a typed speech-community reference or Weksa keeps the persona-to-community -binding (`data/cultural-ontology/persona-stacks/`) is question `persona-speech-community-home` (recommended: a Weksa-owned +binding (the deleted `data/cultural-ontology/persona-stacks/brokkr.yaml` was its one draft) is question `persona-speech-community-home` (recommended: a Weksa-owned binding; follow-up `persona-state-speech-community`), decided when `live-lowering` is mapped; the corpus path needs neither. Putting Weksa feature ids into the open @@ -310,7 +310,7 @@ line two", "その 制御" against the schema's own description. - Probes of 2026-11-12 at `d9ebbcf` (map pass for cut 5): no code reads - `data/target-language-ontology/`, `data/cultural-ontology/` or + `data/target-language-ontology/`, `data/cultural-ontology/` (since deleted) or `.weksa/cultural-ontology/starter-targets.cc` (a third, typed copy of en, ja-JP and pt-BR profiles); `README.md`, `docs/multilingual-utterance-lowering.md`, `examples/cultcache-lowering/` and `prompts/nibu-cultcache-target-*.md` cite @@ -424,7 +424,8 @@ One line per cut; specs exist for the cuts about to run (2d and 4, siblings on ` 4. `speaker-state`: packet `stance` (felt affect and restraint, slot `Stance`, packet schema v4); formality, warmth and intensity out of the seed and the standard; only `PacketStore::admit` yields a `LoweringInput`, and it refuses packets for withdrawn items (follow-up `lowering-input-from-packet-store`). (spec r1) 4a. `speaker-relation-input`: slot `Scene` and its first seed features (`relative_status`, `intimacy`, `in_group`; inventory revision 3, now `speaker_scene_revision`); `LoweringInput::with_scene` carries the speaker's relation to the addressee as typed `Scene` feature values. Since 4b it takes the `SpeakerStateStore` and the `CorpusStore`, reads the scene of the view's own item group for the item's speaker, and selects the relation by the addressee referent's entity (V16 when the scene does not list the item's speaker or the speaker of any live item of the group, the rule `put_scene` applies too; V17 refuses an addressee that names a predication; V13 is retired). The English lowerer does not render it. (PR for follow-up `chat-s3352-3352`) 4b. `speaker-scene` (spec r1; `speaker-scene-seeds` is its cut 4c: CLI, inventory revision 3, cast speakers and scenes; `Party::Unnamed` covers an addressee with no entity; Speaker and Scene share one `SpeakerStateStore`): slots `Speaker` and `Scene` with their seed features; `weksa.speaker` (`speaker:@`) and `weksa.scene` (`scene:@`, participants and ordered-pair relations) documents with `Annotator` `Operator` and `Model { family, generator }`, validated by the packet's shared feature check, append-only per key; their stores, `weksa speaker` and `weksa scene` commands, published schemas; Model-annotated cast speakers and scenes for the committed items. Waits on 2d and 4; `pd-speaker-annotation` blocks only the public-domain annotator. Part 1 landed (branch `weksa-lowering/speaker-scene`): `speaker.rs`, `SpeakerStateStore`, slots `Speaker` and `Scene` exclusive to their own feature lists, `speaker_scene_revision()` (Speaker features 22-24), `with_scene` on the store, schemas. The CLI, the committed inventory revision 3 and the seeds stay with 4c. -5. `language-ontology`, in sequential cuts on `d9ebbcf`. 5a `language-ontology` (spec r1): `weksa.ontology_entry`, `OntologyStore`, `ontology::select` over the view's `Stance` and `Discourse` features, `Prompt::lower` printing the selected entries of the item's language, `RunRecord` pinning the ontology store (keys 15 and 16), `weksa ontology admit|dump`; `ontology.rs` joins `PROMPT_SOURCE_DIGEST`. 5b `language-ontology-seed` (spec r1, after 5a): Model-annotated en-US and ja-JP seed entries drafted by an isolated model, `data/target-language-ontology/`, `.weksa/cultural-ontology/` and `schemas/target-language-cultural-ontology.md` deleted. Lost until 5c': keigo, hierarchy and uchi-soto hang on the scene relation from 5c', so ja-JP politeness is still the lowerer's guess. 5c' `language-ontology-scene` (spec r1, after 5b): the run pins the `SpeakerStateStore` (keys 17 and 18, `Drift(SpeakerState)`), `RunStores::assembled` is the one door that reads the item's scene into a loop view, `select` reads slot `Scene`, `Prompt::lower` refuses a view whose scene was never read, `weksa scene put|dump` and `weksa inventory speaker-scene` (rev 4 in data/original and data/daodejing, rev 3 in data/interlingua, which holds no Dao revision), Model-annotated keeper scenes and Scene-conditioned entries for `data/original`. 5c `language-ontology-view-assembly` (follow-up, now only the Speaker read): `select` widens to slot `Speaker`; waits on `speaker-scene-seeds`. 5d `language-ontology-communities` (follow-up): `weksa.community` from `data/cultural-ontology/` (YAML deleted, owns `edda-referent-yaml-parse`), community ids on `weksa.speaker`, community conditions; waits on 5c and question `persona-speech-community-home`. +5. `language-ontology`, in sequential cuts on `d9ebbcf`. 5a `language-ontology` (spec r1): `weksa.ontology_entry`, `OntologyStore`, `ontology::select` over the view's `Stance` and `Discourse` features, `Prompt::lower` printing the selected entries of the item's language, `RunRecord` pinning the ontology store (keys 15 and 16), `weksa ontology admit|dump`; `ontology.rs` joins `PROMPT_SOURCE_DIGEST`. 5b `language-ontology-seed` (spec r1, after 5a): Model-annotated en-US and ja-JP seed entries drafted by an isolated model, `data/target-language-ontology/`, `.weksa/cultural-ontology/` and `schemas/target-language-cultural-ontology.md` deleted. Lost until 5c': keigo, hierarchy and uchi-soto hang on the scene relation from 5c', so ja-JP politeness is still the lowerer's guess. 5c' `language-ontology-scene` (spec r1, after 5b): the run pins the `SpeakerStateStore` (keys 17 and 18, `Drift(SpeakerState)`), `RunStores::assembled` is the one door that reads the item's scene into a loop view, `select` reads slot `Scene`, `Prompt::lower` refuses a view whose scene was never read, `weksa scene put|dump` and `weksa inventory speaker-scene` (rev 4 in data/original and data/daodejing, rev 3 in data/interlingua, which holds no Dao revision), Model-annotated keeper scenes and Scene-conditioned entries for `data/original`. 5c `language-ontology-view-assembly` (follow-up, now only the Speaker read): `select` widens to slot `Speaker`; waits on `speaker-scene-seeds`. 5d `language-ontology-communities` (follow-up): `weksa.community` built fresh (`data/cultural-ontology/` is deleted by 5e), community ids on `weksa.speaker`, community conditions; waits on 5c and question `persona-speech-community-home`. +5e. `delete-cultural-ontology` (steward ruling, 2026-11-13): `data/cultural-ontology/` (five draft YAML profile files and a README; `old-norse-edda-source-tradition.yaml` did not parse) deleted. Nothing in `crates/`, tests, scripts or CI read it; `examples/persona-lowering/brokkr-edda-voice/` cites its paths as text only (the one test that reads that example reads `lowered-output.yaml`). The typed `weksa.community` of follow-up `language-ontology-communities` is built fresh under cut 5; follow-up `edda-referent-yaml-parse` is moot. 6. `pd-importer`: public-domain play and fiction dialogue into the corpus with speaker, scene group, edition provenance, and Speaker and Scene annotations by the ruled annotator. Waits on 4, follow-up `gutenberg-terms` and question `pd-speaker-annotation`. 7a. `loop-context-builder`: `context.rs` (`TrainItems`, `EncoderOffer`, `Prompt`: corpus text reaches a model call only through `Prompt::mint` over live Train items or `Prompt::encode` over one offered item; instructions are the consts `MINT_INSTRUCTIONS` and `ENCODE_INSTRUCTIONS` in `context.rs`; `Checkpoint` has no public constructor, offers each item once, and its offers borrow it (the gate is typed `TestReadings` and Test offers on the round path; operator tools such as `weksa corpus dump` and `weksa score` still read corpus text of any split); prior turns print the held Train text), `SplitScheme::live`, `CorpusItem::cited_entities` (one owner of the id set `PacketStore::admit` and the offer share), and checkpoint-only Test readings (`Checkpoint`, `TestReadings`; `Aggregate` and `leak_diagnostic` carry no Test). Library only; since 7b `TrainItems::from_corpus` and `EncoderOffer::for_round` are crate-private and `RunStores` is their one public producer. Merges follow-ups `loop-context-builder-2` and `scorer-test-readings-type`. (spec r1) 7b. `golf-loop-round` (spec r1; branch `weksa-lowering/golf-loop-round`): the smallest loop that scores one round. `run.rs`: `RunStores` pins the corpus and entity stores of a run by canonical path and FNV-1a 64 digest at `create`, reopens only from `/run.cc` and refuses drift (corpus, entities, `PROMPT_SOURCE_DIGEST` of `context.rs`, a second pin on `lower.rs` retired in 7d, model, embedder); `round` encodes every live Train and Validation item through a `ModelPort` (`model.rs`; the one adapter `AnswerFiles` writes `.prompt` and reads `.answer`, so the model is the Claude session), reads each answer into a `PacketDraft` (`encode.rs`, the one place a model's JSON is read), admits through `PacketStore::admit` into `/round-/packets.cc`, lowers, scores once, and keeps a `RoundRecord` (`weksa.loop_round`, no text, no Test; Train rows, Validation only as means and refusal counts). `weksa loop open|round`. Not here: the spec-edit step, accept rule and composite, noise floor, Score and lowering documents, a subprocess adapter, the checkpoint door, a model lowerer (follow-ups `golf-loop-spec-edit`, `golf-loop-accept-rule`, `golf-loop-score-documents`, `golf-loop-subprocess-adapter`, `golf-loop-checkpoint-door-bound`, `golf-loop-model-lowerer`). `packet-store-inventory-binding` is met for loop packets only (`RunStores` admits with the run's inventory); `validate`, `packet proof|dao` and `lower` still take caller stores. Waits on 7a, `checkpoint-gate`, 4b. diff --git a/examples/persona-lowering/brokkr-edda-voice/README.md b/examples/persona-lowering/brokkr-edda-voice/README.md index 7c0a4b7..6dd7b71 100644 --- a/examples/persona-lowering/brokkr-edda-voice/README.md +++ b/examples/persona-lowering/brokkr-edda-voice/README.md @@ -7,7 +7,7 @@ referent without falling into generic fantasy-smith voice. The test is deliberately small: one interlingua packet, one rendered utterance, and one trace naming the activated cultural ontology axes. -Profile stack: +Profile stack (these files were deleted from main by cut `delete-cultural-ontology`; they remain in git history before that commit, and the paths below are kept as the fixture's record of what it cited): - `data/cultural-ontology/persona-stacks/brokkr.yaml` - `data/cultural-ontology/referents/old-norse-edda-source-tradition.yaml` diff --git a/examples/persona-lowering/brokkr-edda-voice/interlingua.yaml b/examples/persona-lowering/brokkr-edda-voice/interlingua.yaml index 9772f2e..a8670a1 100644 --- a/examples/persona-lowering/brokkr-edda-voice/interlingua.yaml +++ b/examples/persona-lowering/brokkr-edda-voice/interlingua.yaml @@ -1,6 +1,7 @@ schema_version: weksa.persona_lowering_fixture_batch.v0 fixture_id: brokkr-edda-voice speaker_id: brokkr +# The data/cultural-ontology files cited below were deleted from main (cut delete-cultural-ontology); see git history. cultural_stack_ref: data/cultural-ontology/persona-stacks/brokkr.yaml target_register: language: en-US diff --git a/examples/persona-lowering/brokkr-edda-voice/lowering-trace.yaml b/examples/persona-lowering/brokkr-edda-voice/lowering-trace.yaml index 2e5a388..82d6610 100644 --- a/examples/persona-lowering/brokkr-edda-voice/lowering-trace.yaml +++ b/examples/persona-lowering/brokkr-edda-voice/lowering-trace.yaml @@ -1,6 +1,7 @@ schema_version: weksa.persona_culture_lowering_trace.v0 fixture_id: brokkr-edda-voice item_id: command-without-receipt +# The data/cultural-ontology files cited below were deleted from main (cut delete-cultural-ontology); see git history. profile_stack: - profile_id: weksa.cultural_referent.old_norse_edda_source_tradition.v0 ref: data/cultural-ontology/referents/old-norse-edda-source-tradition.yaml From 8325d7e0b2302365b9bee93c5fd19cee0228f8f7 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 01:23:08 +0000 Subject: [PATCH 06/34] test-text-withheld-arms: withheld prompts stop every model call (#67) --- crates/weksa/tests/golf_loop.rs | 73 ++++++++++++++++++++++++++------- 1 file changed, 58 insertions(+), 15 deletions(-) diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index f754f95..9272af3 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -5199,27 +5199,15 @@ fn a_round_stops_on_a_prompt_that_holds_test_text() { )); } -/// Refuses every prompt as holding Test text, as the answer door does. -struct Refuses; - -impl weksa::model::ModelPort for Refuses { - fn model(&self) -> &str { - "fake-model" - } - fn answer(&mut self, _: &weksa::context::Prompt) -> Result { - Err(weksa::model::PortError::Withheld) - } -} - #[test] -fn every_model_call_stops_on_a_withheld_prompt() { +fn the_encode_step_and_ontology_mint_stop_on_a_withheld_prompt() { use weksa::run::{RunError, round}; let w = world(); let mut run = run_over(w.dir.path()); let scorer = weksa::score::Scorer::new(OneBag); - let r = round(&mut run, &mut Refuses, &scorer); + let r = round(&mut run, &mut Withholds, &scorer); assert!(matches!(r, Err(RunError::Withheld)), "{:?}", r.map(|_| ())); - assert!(matches!(run.mint_ontology(&mut Refuses, "en-US"), Err(RunError::Withheld))); + assert!(matches!(run.mint_ontology(&mut Withholds, "en-US"), Err(RunError::Withheld))); } #[test] @@ -5239,3 +5227,58 @@ fn model_written_test_group_text_is_withheld_too() { "a model-written Test-group item is in no split, and its text is still held out" ); } + +// ---- test-text-withheld-arms: a refused prompt stops every model call, none skips it ---- + +/// Refuses every prompt as holding Test text, as the answer door does. +struct Withholds; + +impl ModelPort for Withholds { + fn model(&self) -> &str { + "fake-model" + } + fn answer(&mut self, _: &Prompt) -> Result { + Err(PortError::Withheld) + } +} + +#[test] +fn recall_stops_on_a_withheld_prompt() { + let dao = dao(); + assert!(matches!(dao.run.recall(&mut Withholds), Err(RunError::Withheld))); +} + +#[test] +fn the_lowering_step_of_a_round_stops_on_a_withheld_prompt() { + // Encoding is answered; every lowering prompt is refused. + let mut dao = dao(); + let mut model = fake(&good_answers()); + model.raw_lowering = Some(Err(PortError::Withheld)); + let r = round(&mut dao.run, &mut model, &Scorer::new(Bag)); + assert!(matches!(r, Err(RunError::Withheld)), "{:?}", r.map(|_| ())); + assert!( + !dao.root.join("round-1/packets.cc").exists(), + "the refused attempt's packets are discarded" + ); +} + +#[test] +fn the_cycle_editor_stops_on_a_withheld_prompt() { + let mut dao = dao(); + two_rounds(&mut dao); + let mut model = cycle_fake(ONE_CONCEPT); + model.edit = Some(Err(PortError::Withheld)); + let r = cycle(&mut dao.run, &mut model, &Scorer::new(Bag)); + assert!(matches!(r, Err(RunError::Withheld)), "{:?}", r.map(|_| ())); +} + +#[test] +fn a_stance_ablation_stops_on_a_withheld_prompt() { + let (dao, _) = recorded_dao(|_| {}); + let mut rec = Lowerer::new(blind); + let r = ablate(&dao, &mut rec, &mut Withholds); + assert!(matches!(r, Err(RunError::Withheld)), "{:?}", r.map(|_| ())); + // A recorded answer that is refused stops it as well. + let r = ablate(&dao, &mut Withholds, &mut Lowerer::new(blind)); + assert!(matches!(r, Err(RunError::Withheld)), "{:?}", r.map(|_| ())); +} From ee3e4a2e386ecdc88d0a648bd849831b31a87345 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 01:29:57 +0000 Subject: [PATCH 07/34] withheld-candidate: candidate round stops on a withheld lowering prompt (#68) --- crates/weksa/tests/golf_loop.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 9272af3..3a33363 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -5282,3 +5282,20 @@ fn a_stance_ablation_stops_on_a_withheld_prompt() { let r = ablate(&dao, &mut Withholds, &mut Lowerer::new(blind)); assert!(matches!(r, Err(RunError::Withheld)), "{:?}", r.map(|_| ())); } + +#[test] +fn the_cycles_candidate_round_stops_on_a_withheld_lowering_prompt() { + // The edit is answered and the candidate round encodes at rev 4; every lowering + // prompt is then refused. No decision is recorded and no packets are left. + let mut dao = dao(); + two_rounds(&mut dao); + let mut model = cycle_fake(ONE_CONCEPT); + model.raw_lowering = Some(Err(PortError::Withheld)); + let r = cycle(&mut dao.run, &mut model, &Scorer::new(Bag)); + assert!(matches!(r, Err(RunError::Withheld)), "{:?}", r.map(|_| ())); + assert!(dao.run.decisions().unwrap().is_empty(), "a refused cycle decides nothing"); + assert!( + !dao.root.join("cycle-1/packets.cc").exists(), + "the refused attempt's packets are discarded" + ); +} From 97aa026498318c84ef675a69ceeef65a3bb6ba8a Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 02:05:41 +0000 Subject: [PATCH 08/34] weksa loop relower: fixed-packet re-lowering measure with a lowering-only floor (#69) --- crates/weksa/src/main.rs | 77 +++- crates/weksa/src/run.rs | 366 ++++++++++++++++--- crates/weksa/tests/cli.rs | 80 +++++ crates/weksa/tests/golf_loop.rs | 436 +++++++++++++++++++++++ docs/targets/interlingua-lowering-map.md | 27 +- 5 files changed, 932 insertions(+), 54 deletions(-) diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index b76051c..9ba2135 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -13,9 +13,9 @@ use weksa::entity::{Entity, EntityOrigin, LanguageForms, Pronunciation}; use weksa::inventory::{proof_revision, seed, speaker_scene_revision}; use weksa::packet::{FeatValue, FeatureValue, Grade}; use weksa::packet::proof_packet; -use weksa::model::RecordedAnswers; +use weksa::model::{ModelPort, RecordedAnswers}; use weksa::run::{ - AblationOutcome, CycleOutcome, LowererReach, OntologyMintOutcome, RecallOutcome, RoundOutcome, RunPaths, RunStores, cycle, round, + AblationOutcome, CycleOutcome, LowererReach, OntologyMintOutcome, RecallOutcome, RelowerOutcome, RoundOutcome, RunPaths, RunStores, cycle, round, }; use weksa::score::ScoreInput; use weksa::{ @@ -153,6 +153,26 @@ enum LoopCommand { #[arg(long, default_value = "http://127.0.0.1:11434")] ollama_url: String, }, + /// Re-lower round ROUND's Validation packets through arm ARM, score them + /// against the items, and, with --floor ARM, decide against that second arm + /// by the panel accept rule. Prompts and answers live in + /// `/relower--`; the command exits 3 until the arm is + /// answered. --lowerer names the arm's model in the printed line. A + /// diagnostic, never a gate; records nothing; Test and Train are never + /// touched. Labels match [a-z0-9-]{1,32} and differ. + Relower { + root: PathBuf, + #[arg(long)] + round: u32, + #[arg(long)] + arm: String, + #[arg(long)] + lowerer: String, + #[arg(long)] + floor: Option, + #[arg(long, default_value = "http://127.0.0.1:11434")] + ollama_url: String, + }, /// Run one edit-accept cycle of the run at ROOT. The stores are read only /// from the run's record. The editor's prompt, then the candidate round's /// encode prompts, then its lowering prompts are written in turn; the @@ -1115,6 +1135,59 @@ fn main() -> Result<()> { } } } + Command::Loop(LoopCommand::Relower { + root, + round, + arm, + lowerer, + floor, + ollama_url, + }) => { + let label = |l: &str| { + (1..=32).contains(&l.len()) + && l.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') + }; + if !label(&arm) || floor.as_deref().is_some_and(|f| !label(f) || f == arm) { + anyhow::bail!("arm and floor labels must match [a-z0-9-]{{1,32}} and differ"); + } + let run = RunStores::open(&root)?; + let mut recorded = RecordedAnswers::new( + root.join(format!("round-{round}")), + run.record().model.clone(), + ); + let mut files = run.answer_files(run.relower_dir(round, &arm))?; + let mut floor_port = floor.as_deref().map(|f| { + RecordedAnswers::new(run.relower_dir(round, f), run.record().model.clone()) + }); + let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder)); + match run.relower( + round, + &arm, + &mut recorded, + &mut files, + floor_port.as_mut().map(|p| p as &mut dyn ModelPort), + &scorer, + )? { + RelowerOutcome::Unanswered { prompts } => { + for digest in &prompts { + println!("unanswered {digest}"); + } + println!("unanswered {} prompts", prompts.len()); + std::process::exit(3); + } + RelowerOutcome::Read { languages, verdict } => { + for l in languages { + println!( + "{}", + run.labelled(&format!("{} arm={arm} lowerer={lowerer}", l.line())) + ); + } + if let Some(v) = verdict { + println!("verdict {v:?}"); + } + } + } + } Command::Loop(LoopCommand::Recall { root }) => { let run = RunStores::open(&root)?; let mut files = run.answer_files(run.recall_dir())?; diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index d4b3f9f..1dc4780 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -495,6 +495,9 @@ pub enum RunError { Ontology, /// A scene does not read against the run's inventory. SpeakerState, + /// The floor arm of a relower holds no answer to a prompt; it is a + /// reference, never written to, and must be answered whole first. + FloorUnanswered, } impl fmt::Display for RunError { @@ -523,6 +526,9 @@ impl fmt::Display for RunError { RunError::SpeakerState => { f.write_str("a scene does not read against the run's inventory") } + RunError::FloorUnanswered => { + f.write_str("the floor arm holds no answer to a lowering prompt; answer it whole first") + } RunError::Panel => f.write_str( "the panel is empty, repeats or misspells a language, or names one with no live Train or Validation item", ), @@ -1337,6 +1343,66 @@ type HeldSplit = (String, Split, Vec<(CorpusItem, Packet)>); type Lowered = (String, String, String, [Option; 2]); impl RunStores { + /// The live items of each panel language in each listed split with their + /// round-`n` packets, per language then split, sorted by item key. A round + /// that is absent or a cycle's is `Malformed("round")`. Test is never held. + fn held_splits(&self, n: u32, splits: &[Split]) -> Result, RunError> { + let store = |_| RunError::Store; + self.rounds() + .map_err(store)? + .into_iter() + .find(|r| r.n() == n && r.cycle().is_none()) + .ok_or(RunError::Malformed("round"))?; + let round_packets = PacketStore::open(&self.root.join(format!("round-{n}")).join("packets.cc")) + .map_err(store)? + .all() + .map_err(store)?; + let mut held: Vec = Vec::new(); + for tag in &self.record.panel { + for split in splits { + held.push((tag.clone(), *split, Vec::new())); + } + } + for packet in round_packets { + let item = self + .corpus + .get(packet.provenance.corpus_item.as_str()) + .map_err(store)? + .ok_or(RunError::Store)?; + for (language, split, items) in &mut held { + if *language == item.language && SplitScheme::V1.live(&item) == Some(*split) { + items.push((item.clone(), packet.clone())); + } + } + } + for (_, _, items) in &mut held { + items.sort_by(|a, b| a.0.id.as_str().cmp(b.0.id.as_str())); + } + Ok(held) + } + + /// The lowering prompt of `packet` for `item`: admitted into `store`, + /// assembled and lowered. The one builder for the round's prompt, which + /// ablation and relower must reproduce byte for byte. `None` when the + /// lowering prompt itself is refused, as it was for the round. + fn lowering_prompt( + &self, + store: &mut PacketStore, + packet: Packet, + item: &CorpusItem, + inventory: &crate::Inventory, + ontology: &crate::Ontology, + ) -> Result, RunError> { + let packet_item = packet.provenance.corpus_item.clone(); + let view = store + .admit(packet, &self.inventory, &self.corpus, &self.entities) + .map_err(|_| RunError::Store)?; + let view = self + .assembled(view, &self.inventory) + .map_err(|_| RunError::Store)?; + Ok(Prompt::lower(&view, &packet_item, &item.language, inventory, &self.entities, ontology).ok()) + } + /// `/ablation-`, where a stance ablation's prompts, answers and /// variant packet stores live. pub fn ablation_dir(&self, n: u32) -> PathBuf { @@ -1362,11 +1428,8 @@ impl RunStores { ) -> Result { let store = |_| RunError::Store; check_drift(self, model, scorer)?; - self.rounds() - .map_err(store)? - .into_iter() - .find(|r| r.n() == n && r.cycle().is_none()) - .ok_or(RunError::Malformed("round"))?; + // A refused round writes nothing: the round is read before any folder is made. + let mut splits = self.held_splits(n, &[Split::Train, Split::Validation])?; let inventory = self.inventory.inventory().map_err(store)?; let ontology = self.ontology.ontology().map_err(store)?; @@ -1383,56 +1446,17 @@ impl RunStores { PacketStore::open(&dir.join("permute.cc")).map_err(store)?, ]; - // Live Train and Validation packets, in item key order within a split. - let round_packets = PacketStore::open(&self.root.join(format!("round-{n}")).join("packets.cc")) - .map_err(store)? - .all() - .map_err(store)?; - let mut splits: Vec = Vec::new(); - for tag in &self.record.panel { - for split in [Split::Train, Split::Validation] { - splits.push((tag.clone(), split, Vec::new())); - } - } - for packet in round_packets { - let item = self - .corpus - .get(packet.provenance.corpus_item.as_str()) - .map_err(store)? - .ok_or(RunError::Store)?; - for (language, split, held) in &mut splits { - if *language == item.language && SplitScheme::V1.live(&item) == Some(*split) { - held.push((item.clone(), packet.clone())); - } - } - } - let mut waiting: Vec = Vec::new(); let mut readings = Vec::new(); // Texts per split: (item text, recorded, replay, [drop, permute]). let mut texts: Vec> = Vec::new(); let mut unusable = vec![0u32; splits.len()]; for (at, (_, _, held)) in splits.iter_mut().enumerate() { - held.sort_by(|a, b| a.0.id.as_str().cmp(b.0.id.as_str())); let mut rows = Vec::new(); for (i, (item, packet)) in held.iter().enumerate() { - let packet_item = packet.provenance.corpus_item.clone(); let mut prompt_of = |slot: usize, packet: Packet| -> Result { - let view = stores[slot] - .admit(packet, &self.inventory, &self.corpus, &self.entities) - .map_err(|_| RunError::Store)?; - let view = self - .assembled(view, &self.inventory) - .map_err(|_| RunError::Store)?; - Prompt::lower( - &view, - &packet_item, - &item.language, - &inventory, - &self.entities, - &ontology, - ) - .map_err(|_| RunError::Store) + self.lowering_prompt(&mut stores[slot], packet, item, &inventory, &ontology)? + .ok_or(RunError::Store) }; let base = prompt_of(0, packet.clone())?; let mut variants: [Option; 2] = [None, None]; @@ -1547,6 +1571,236 @@ impl RunStores { } } +/// One panel language's relower reading: Validation items only. +#[derive(Clone, Debug, PartialEq)] +pub struct RelowerReading { + pub language: String, + /// Live Validation items of the language read: those with a round packet + /// whose lowering prompt the round could build. Every reading below is over + /// these items, so recorded, arm and floor stay comparable. + pub n: u32, + /// Live Validation items of the language not read: refused by the round + /// before admission (no packet) or at the lowering prompt. They are in no + /// reading, so `recorded` can differ from the round's own Validation + /// reading, which scores them 0. + pub skipped: u32, + /// Arm answers that were not a usable lowering; each scored 0. + pub unusable: u32, + /// Floor-arm answers that were not a usable lowering; each scored 0, which + /// widens `floor`. Zero without a floor arm. + pub floor_unusable: u32, + /// The round's own lowering, read against the items. + pub recorded: Reading, + pub arm: Reading, + /// `spread(recorded, floor arm)`, when a floor arm was given. + pub floor: Option, +} + +impl RelowerReading { + pub fn line(&self) -> String { + let show = |r: &Reading| { + format!("chrf={:.2} meaning={:.3} composite={:.4}", r.chrf, r.meaning, r.composite) + }; + let floor = self + .floor + .as_ref() + .map_or(String::new(), |f| { + format!(" | floor {} unusable={}", show(f), self.floor_unusable) + }); + format!( + "relower {} n={} skipped={} unusable={} recorded {} | arm {}{floor}", + self.language, + self.n, + self.skipped, + self.unusable, + show(&self.recorded), + show(&self.arm), + ) + } +} + +#[derive(Debug)] +pub enum RelowerOutcome { + Unanswered { prompts: Vec }, + /// Per panel language in panel order; the verdict only when a floor arm was given. + Read { languages: Vec, verdict: Option }, +} + +impl RunStores { + /// `/relower--`, where an arm's prompts and answers live. + pub fn relower_dir(&self, n: u32, arm: &str) -> PathBuf { + self.root.join(format!("relower-{n}-{arm}")) + } + + /// A diagnostic like [`recall`](Self::recall) and + /// [`stance_ablation`](Self::stance_ablation), never a gate: it serves the + /// distil decision, whether a tuned lowerer is worth keeping against the + /// prompted one (calibration-pending: see the map). Asks `arm` the byte-identical lowering prompts of round `n`'s live + /// Validation items, reads the arm and the round's own lowering + /// (`recorded`) against the items, and, given a `floor` arm (a second + /// prompted lowering, so the floor is lowering-only noise), decides by + /// [`decide_panel`] over (recorded, arm, spread(recorded, floor)). Only + /// the model of `arm` is unpinned; the embedder is checked. An arm answer + /// that is not a usable lowering scores 0 and is counted in `unusable`. + /// Records nothing; `arm_label` names only where the scratch packet store + /// lives, in the arm's directory. + pub fn relower( + &self, + n: u32, + arm_label: &str, + recorded: &mut dyn ModelPort, + arm: &mut dyn ModelPort, + mut floor: Option<&mut dyn ModelPort>, + scorer: &Scorer, + ) -> Result { + let store = |_| RunError::Store; + check_embedder(self, scorer)?; + let held = self.held_splits(n, &[Split::Validation])?; + let inventory = self.inventory.inventory().map_err(store)?; + let ontology = self.ontology.ontology().map_err(store)?; + + let dir = self.relower_dir(n, arm_label); + std::fs::create_dir_all(&dir).map_err(|e| RunError::Io(e.kind()))?; + for suffix in ["cc", "cc.lock"] { + let _ = std::fs::remove_file(dir.join(format!("base.{suffix}"))); + } + let mut scratch = PacketStore::open(&dir.join("base.cc")).map_err(store)?; + + // Per language, per item: the item text, then the recorded, arm and floor + // lowerings (`None` when unusable). + type Row = (String, Option, Option, Option); + let mut waiting: Vec = Vec::new(); + let mut floor_waiting = false; + let mut rows: Vec> = Vec::new(); + for (_, _, items) in &held { + let mut language_rows = Vec::new(); + for (item, packet) in items { + // An item the round could not lower has no prompt to ask; it is skipped + // and counted, not scored. + let Some(prompt) = + self.lowering_prompt(&mut scratch, packet.clone(), item, &inventory, &ontology)? + else { + continue; + }; + let text = |a: Asked| match a { + Asked::Text(t) => Some(t), + _ => None, + }; + let recorded_text = match asked(recorded, &prompt)? { + Asked::Waiting => return Err(RunError::Unrecorded), + other => text(other), + }; + let floor_text = match floor.as_deref_mut() { + Some(f) => match asked(f, &prompt)? { + Asked::Waiting => { + floor_waiting = true; + None + } + other => text(other), + }, + None => None, + }; + let arm_text = match asked(arm, &prompt)? { + Asked::Waiting => { + let d = prompt.digest(); + if !waiting.contains(&d) { + waiting.push(d); + } + None + } + other => text(other), + }; + language_rows.push((item.text.clone(), recorded_text, arm_text, floor_text)); + } + rows.push(language_rows); + } + if floor_waiting { + return Err(RunError::FloorUnanswered); + } + if !waiting.is_empty() { + return Ok(RelowerOutcome::Unanswered { prompts: waiting }); + } + + // One score call: each usable lowering against its item, in row order. + let mut inputs: Vec = Vec::new(); + for (reference, recorded, arm, floor_text) in rows.iter().flatten() { + for output in [recorded, arm, floor_text].into_iter().flatten() { + inputs.push(ScoreInput { reference, output }); + } + } + let mut scores = if inputs.is_empty() { + Vec::new() + } else { + scorer.score(&inputs).map_err(|_| RunError::Embed)? + } + .into_iter(); + let mut next = |t: &Option| -> Result { + match t { + Some(_) => scores.next().ok_or(RunError::Store), + None => Ok(ItemScore { chrf: 0.0, meaning: Some(0.0) }), + } + }; + let reading_of = |scored: &[ItemScore]| { + Reading::of(&SplitMean { + count: scored.len(), + chrf: mean(scored.iter().map(|s| s.chrf)), + meaning: mean(scored.iter().filter_map(|s| s.meaning)), + }) + .ok_or(RunError::NoValidation) + }; + let mut languages = Vec::new(); + let live = self.corpus.all().map_err(store)?; + for ((language, _, _), language_rows) in held.iter().zip(&rows) { + let (mut rec, mut arm_scores, mut floor_scores) = (Vec::new(), Vec::new(), Vec::new()); + let (mut unusable, mut floor_unusable) = (0u32, 0u32); + for (_, r, a, f) in language_rows { + rec.push(next(r)?); + arm_scores.push(next(a)?); + if a.is_none() { + unusable += 1; + } + if floor.is_some() { + floor_scores.push(next(f)?); + if f.is_none() { + floor_unusable += 1; + } + } + } + let recorded_reading = reading_of(&rec)?; + languages.push(RelowerReading { + language: language.clone(), + n: language_rows.len() as u32, + skipped: live + .iter() + .filter(|i| { + &i.language == language && SplitScheme::V1.live(i) == Some(Split::Validation) + }) + .count() as u32 + - language_rows.len() as u32, + unusable, + floor_unusable, + recorded: recorded_reading, + arm: reading_of(&arm_scores)?, + floor: if floor.is_some() { + Some(spread(&recorded_reading, &reading_of(&floor_scores)?)) + } else { + None + }, + }); + } + let verdict = if floor.is_some() { + let panel: Vec<(Reading, Reading, Reading)> = languages + .iter() + .filter_map(|l| Some((l.recorded, l.arm, l.floor?))) + .collect(); + Some(decide_panel(&panel)) + } else { + None + }; + Ok(RelowerOutcome::Read { languages, verdict }) + } +} + /// The distinct rule codes an admission refusal broke: validator rules by /// code, a store refusal by its variant name. Nothing from the packet or the /// stores is kept; an I/O failure is `io`. @@ -1613,6 +1867,11 @@ fn check_drift( if model.model() != run.record.model { return Err(RunError::Drift(DriftOf::Model)); } + check_embedder(run, scorer) +} + +/// The embedder the run pinned is the one scoring. +fn check_embedder(run: &RunStores, scorer: &Scorer) -> Result<(), RunError> { if scorer.embedder().model() != run.record.embedder { return Err(RunError::Drift(DriftOf::Embedder)); } @@ -2223,16 +2482,21 @@ fn floor_of( let (a, b) = (replicate(1, tag)?, replicate(2, tag)?); Ok(( tag.clone(), - Reading { - chrf: (a.chrf - b.chrf).abs(), - meaning: (a.meaning - b.meaning).abs(), - composite: (a.composite - b.composite).abs(), - }, + spread(&a, &b), )) }) .collect() } +/// The absolute difference of two readings, metric by metric. +fn spread(a: &Reading, b: &Reading) -> Reading { + Reading { + chrf: (a.chrf - b.chrf).abs(), + meaning: (a.meaning - b.meaning).abs(), + composite: (a.composite - b.composite).abs(), + } +} + /// Each panel language's Validation reading in `round`, in panel order. fn readings_of( round: &RoundRecord, diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 40aa288..95a147a 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -1778,6 +1778,86 @@ fn loop_ablate_waits_then_prints_one_line_per_split() { } } +#[test] +fn cli_loop_relower_exits_3_until_answered_and_refuses_bad_labels() { + let d = tempfile::tempdir().unwrap(); + open_dao_run(d.path()); + let url = fake_embedder(); + let root = d.path().join("canaryroot"); + let mut run = weksa::run::RunStores::open(&root).unwrap(); + let scorer = weksa::Scorer::new(weksa::OllamaEmbedder::new(&url, &run.record().embedder)); + let weksa::run::RoundOutcome::Recorded(r) = weksa::run::round(&mut run, &mut DaoModel, &scorer).unwrap() + else { + panic!("answered") + }; + let said = r#"{"text":"The highest excellence is like the water."}"#; + for call in &r.calls()[3..] { + std::fs::write(root.join(format!("round-1/{}.answer", call.prompt)), said).unwrap(); + } + let relower = |extra: &[&str]| { + weksa( + d.path(), + &[&["loop", "relower", "canaryroot", "--round", "1", "--lowerer", "tuned-x", "--ollama-url", &url], extra].concat(), + ) + }; + + // Unanswered: exit 3, the digests, a prompt file per digest in the arm's directory. + let first = relower(&["--arm", "tuned"]); + assert_eq!(first.status.code(), Some(3), "{}", out(&first)); + let text = out(&first); + let digests: Vec<&str> = text + .lines() + .filter_map(|l| l.strip_prefix("unanswered ")) + .filter(|x| x.len() == 16) + .collect(); + assert_eq!(digests.len(), 2, "the two Validation prompts: {text}"); + for digest in &digests { + assert!(root.join(format!("relower-1-tuned/{digest}.prompt")).is_file()); + std::fs::write(root.join(format!("relower-1-tuned/{digest}.answer")), said).unwrap(); + } + + // Answered: one line per language, labelled, naming the lowerer; no verdict without a floor. + let second = relower(&["--arm", "tuned"]); + assert!(second.status.success(), "{}", out(&second)); + let text = out(&second); + assert!(text.starts_with("relower en-US n=2 skipped=0 unusable=0 recorded "), "{text}"); + assert!(text.contains("arm=tuned lowerer=tuned-x"), "{text}"); + assert!(text.contains(" lowering="), "{text}"); + assert!(!text.contains("verdict") && !text.contains("floor"), "{text}"); + for leaked in ["item:", "packet:", "highest excellence", "canaryroot"] { + assert!(!text.contains(leaked), "{text}"); + } + + // A floor arm that is not answered whole is refused, and never written to. + let o = relower(&["--arm", "tuned", "--floor", "second"]); + assert_eq!(o.status.code(), Some(1), "{}", out(&o)); + assert!(!root.join("relower-1-second").exists()); + + // Answered floor: a verdict line. + std::fs::create_dir(root.join("relower-1-second")).unwrap(); + for digest in &digests { + std::fs::write(root.join(format!("relower-1-second/{digest}.answer")), said).unwrap(); + } + let o = relower(&["--arm", "tuned", "--floor", "second"]); + assert!(o.status.success(), "{}", out(&o)); + let text = out(&o); + assert!(text.contains("| floor chrf=0.00") && text.trim_end().ends_with("verdict Rejected(NoGain)"), "{text}"); + + // Labels: equal, uppercase, empty, too long, a path. + let long = "a".repeat(33); + for args in [ + vec!["--arm", "same", "--floor", "same"], + vec!["--arm", "Upper"], + vec!["--arm", ""], + vec!["--arm", &long], + vec!["--arm", "../x"], + vec!["--arm", "tuned", "--floor", "Bad"], + ] { + let o = relower(&args); + assert_eq!(o.status.code(), Some(1), "{args:?}: {}", out(&o)); + } +} + #[test] fn loop_ablate_without_the_rounds_answers_says_so() { let d = tempfile::tempdir().unwrap(); diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 3a33363..179d0ea 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -3764,9 +3764,18 @@ fn grade_stance(features: &[(&str, u16)]) -> weksa::packet::Stance { /// packet. Stances: item 1 restraint 800; item 2 restraint 800; item 3 /// restraint 800 and affection 300; the written item restraint 500. fn recorded_dao(edit: impl FnOnce(&mut HashMap)) -> (Dao, RoundRecord) { + recorded_dao_in(|_, _| {}, edit) +} + +/// `recorded_dao` with `corpus_edit` run over the corpus and entities first. +fn recorded_dao_in( + corpus_edit: impl FnOnce(&mut CorpusStore, &weksa::EntityStore), + edit: impl FnOnce(&mut HashMap), +) -> (Dao, RoundRecord) { let mut dao = dao_over(|corpus, entities| { let entities = weksa::EntityStore::open(entities).unwrap(); let mut store = CorpusStore::open(corpus).unwrap(); + corpus_edit(&mut store, &entities); let mut written = store.get("item:daodejing:2").unwrap().unwrap(); written.id = WRITTEN.into(); written.text = "Heaven endures through the long years without end.".into(); @@ -4112,6 +4121,28 @@ fn a_stance_ablation_refuses_a_cycle_round_and_missing_round_answers() { )); } +#[test] +fn a_refused_stance_ablation_creates_no_folder_and_wipes_no_store() { + let (dao, _) = recorded_dao(|_| {}); + let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(reader)); + // No round 99: no folder appears. + assert!(matches!( + dao.run.stance_ablation(99, &mut rec, &mut model, &Scorer::new(Bag)), + Err(RunError::Malformed("round")) + )); + assert!(!dao.run.ablation_dir(99).exists(), "a refused ablation makes no folder"); + // An earlier ablation of a refused n keeps its stores byte for byte. + let kept = dao.run.ablation_dir(2); + std::fs::create_dir_all(&kept).unwrap(); + std::fs::write(kept.join("base.cc"), b"earlier ablation").unwrap(); + assert!(matches!( + dao.run.stance_ablation(2, &mut rec, &mut model, &Scorer::new(Bag)), + Err(RunError::Malformed("round")) + )); + assert_eq!(std::fs::read(kept.join("base.cc")).unwrap(), b"earlier ablation"); + assert_eq!(std::fs::read_dir(&kept).unwrap().count(), 1); +} + #[test] fn a_stance_ablation_refuses_a_cycle_round() { let mut dao = dao(); @@ -5299,3 +5330,408 @@ fn the_cycles_candidate_round_stops_on_a_withheld_lowering_prompt() { "the refused attempt's packets are discarded" ); } + +// ---- distil-slice: relower, a fixed-packet re-lowering measure ---- + +use weksa::run::{RelowerOutcome, RelowerReading}; + +/// `relower` of round 1 with the Bag scorer; `floor` is an optional second arm. +fn relower( + dao: &Dao, + recorded: &mut dyn ModelPort, + arm: &mut dyn ModelPort, + floor: Option<&mut dyn ModelPort>, +) -> Result { + dao.run.relower(1, "arm", recorded, arm, floor, &Scorer::new(Bag)) +} + +fn relowered(outcome: Result) -> (Vec, Option) { + match outcome.unwrap() { + RelowerOutcome::Read { languages, verdict } => (languages, verdict), + RelowerOutcome::Unanswered { prompts } => panic!("unanswered {prompts:?}"), + } +} + +#[test] +fn relower_asks_only_validation_lowering_prompts_of_the_round() { + let (dao, r) = recorded_dao(|_| {}); + let round_digests: Vec = r.calls()[4..].iter().map(|c| c.prompt.clone()).collect(); + // The reference pass: a lowerer shown every packet of the round, to learn + // which digest is which item's. + let (mut every, mut rec, mut arm) = (Lowerer::new(reader), Lowerer::new(reader), Lowerer::new(reader)); + read(ablate(&dao, &mut every, &mut Lowerer::new(reader)).unwrap()); + relowered(relower(&dao, &mut rec, &mut arm, None)); + let of = |keys: &[&str]| -> Vec { + every + .seen + .iter() + .filter(|s| keys.contains(&s.1.as_str())) + .map(|s| s.0.clone()) + .fold(Vec::new(), |mut v, d| { + if !v.contains(&d) { + v.push(d); + } + v + }) + }; + let mut want = of(&["item:daodejing:2", "item:daodejing:3", WRITTEN]); + want.sort(); + let mut got = arm.digests(); + got.sort(); + assert_eq!(got, want, "exactly the Validation lowering prompts"); + assert_eq!(want.len(), 3); + for d in &want { + assert!(round_digests.contains(d), "byte-identical to the round's prompt {d}"); + } + // Neither the lone Train item, a Test item nor an encode prompt. + for (_, key, text) in &arm.seen { + assert!(!["item:daodejing:1", "item:daodejing:4", "item:daodejing:5"].contains(&key.as_str())); + assert!(text.starts_with(LOWER_INSTRUCTIONS)); + } + assert_eq!(rec.digests().len(), 3, "the recorded port is asked the same prompts"); +} + +#[test] +fn relower_reads_recorded_and_arm_against_the_item() { + let (dao, _) = recorded_dao(|_| {}); + // An arm that says the item's own sentence beats a recorded lowering that + // adds the stance to it. + let (langs, verdict) = relowered(relower( + &dao, + &mut Lowerer::new(reader), + &mut Lowerer::new(blind), + None, + )); + let l = &langs[0]; + assert_eq!((l.language.as_str(), l.n, l.unusable), ("en-US", 3, 0)); + assert!(l.recorded.chrf < l.arm.chrf - 5.0, "{}", l.line()); + assert!(l.arm.composite > l.recorded.composite); + assert!(l.floor.is_none() && verdict.is_none(), "no floor arm, no verdict"); + // An arm equal to the recorded answers reads equal. + let (langs, _) = relowered(relower(&dao, &mut Lowerer::new(reader), &mut Lowerer::new(reader), None)); + assert_eq!(langs[0].recorded, langs[0].arm); +} + +/// Answers `text` for every item but `bad`, which gets something that is no lowering. +struct Breaks(&'static str); + +impl ModelPort for Breaks { + fn model(&self) -> &str { + "fake-model" + } + fn answer(&mut self, prompt: &Prompt) -> Result { + let key = prompt.item_keys()[0].clone(); + if key == self.0 { + return Ok("this is not a lowering draft".into()); + } + Ok(serde_json::json!({ "text": english(&key) }).to_string()) + } +} + +#[test] +fn relower_counts_an_unusable_arm_answer_as_zero() { + let (dao, _) = recorded_dao(|_| {}); + let keys = ["item:daodejing:2", "item:daodejing:3", WRITTEN]; + let (whole, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), None)); + let whole = whole[0].arm; + let (mut chrf, mut meaning) = (0.0, 0.0); + for key in keys { + let (langs, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Breaks(key), None)); + let l = &langs[0]; + assert_eq!((l.n, l.unusable), (3, 1), "{}", l.line()); + assert!(l.arm.chrf < whole.chrf && l.arm.meaning < whole.meaning, "{}", l.line()); + chrf += l.arm.chrf; + meaning += l.arm.meaning; + } + // Each item is refused once: the three means hold every item twice at its + // score and once at 0, so the refused item entered its mean as 0 and n stayed 3. + assert!((chrf - 2.0 * whole.chrf).abs() < 1e-6, "{chrf} {}", whole.chrf); + assert!((meaning - 2.0 * whole.meaning).abs() < 1e-6, "{meaning} {}", whole.meaning); +} + +#[test] +fn relower_scores_an_unusable_recorded_answer_as_zero_without_counting_it_against_the_arm() { + let (dao, _) = recorded_dao(|_| {}); + let (whole, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), None)); + let (langs, _) = relowered(relower(&dao, &mut Breaks("item:daodejing:3"), &mut Lowerer::new(blind), None)); + let l = &langs[0]; + assert_eq!(l.unusable, 0, "{}", l.line()); + assert_eq!(l.arm, whole[0].arm); + assert!(l.recorded.chrf < whole[0].recorded.chrf && l.recorded.meaning < whole[0].recorded.meaning); +} + +#[test] +fn relower_decides_by_decide_panel_against_the_floor_arm() { + use weksa::run::decide_panel; + let (dao, _) = recorded_dao(|_| {}); + let run = |recorded: fn(&str, &str) -> String, arm: fn(&str, &str) -> String| { + let mut floor = Lowerer::new(recorded); + let (langs, verdict) = relowered(relower( + &dao, + &mut Lowerer::new(recorded), + &mut Lowerer::new(arm), + Some(&mut floor), + )); + let l = &langs[0]; + let floor_reading = l.floor.expect("a floor arm gives a floor"); + // A floor arm that equals the recorded answers has no spread. + assert_eq!((floor_reading.chrf, floor_reading.meaning, floor_reading.composite), (0.0, 0.0, 0.0)); + assert_eq!(verdict, Some(decide_panel(&[(l.recorded, l.arm, floor_reading)]))); + verdict.unwrap() + }; + assert_eq!(run(reader, reader), Verdict::Rejected(Why::NoGain)); + assert_eq!(run(reader, blind), Verdict::Accepted); + assert!(matches!(run(blind, reader), Verdict::Rejected(Why::Regressed(_)))); +} + +#[test] +fn relower_floor_is_the_spread_of_recorded_and_the_floor_arm() { + let (dao, _) = recorded_dao(|_| {}); + let (langs, _) = relowered(relower( + &dao, + &mut Lowerer::new(blind), + &mut Lowerer::new(blind), + Some(&mut Breaks("item:daodejing:3")), + )); + let l = &langs[0]; + let f = l.floor.unwrap(); + // The floor arm read as an arm against the same recorded answers. + let (as_arm, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Breaks("item:daodejing:3"), None)); + let a = as_arm[0].arm; + assert!(f.chrf > 0.0 && f.meaning > 0.0 && f.composite > 0.0); + assert!((f.chrf - (l.recorded.chrf - a.chrf).abs()).abs() < 1e-9, "{}", l.line()); + assert!((f.composite - (l.recorded.composite - a.composite).abs()).abs() < 1e-9); + assert!(l.line().contains(&format!("| floor chrf={:.2}", f.chrf)), "{}", l.line()); + assert_eq!(l.unusable, 0, "unusable counts the arm, not the floor"); + assert_eq!(l.floor_unusable, 1, "the floor arm's refusal is counted apart"); + assert!(l.line().contains(" unusable=0 recorded "), "{}", l.line()); + assert!(l.line().ends_with(" unusable=1"), "the floor's refusal is on the line: {}", l.line()); + // Without a floor arm there is none to count. + assert_eq!(as_arm[0].floor_unusable, 0); +} + +#[test] +fn relower_skips_and_counts_an_item_the_round_refused_before_admission() { + // Validation item 3 is not an encodable draft in the round: it has no packet. + let (dao, _) = recorded_dao(|answers| { + answers.insert("item:daodejing:3".into(), "not a draft".into()); + }); + let (mut rec, mut arm) = (Lowerer::new(blind), Lowerer::new(blind)); + let (langs, _) = relowered(relower(&dao, &mut rec, &mut arm, None)); + let l = &langs[0]; + assert_eq!((l.n, l.skipped, l.unusable), (2, 1, 0), "{}", l.line()); + assert!(l.line().starts_with("relower en-US n=2 skipped=1 unusable=0 "), "{}", l.line()); + assert_eq!(l.recorded, l.arm, "both readings are over the same two items"); + assert!(arm.seen.iter().all(|(_, key, _)| key != "item:daodejing:3"), "nothing is asked for it"); +} + +#[test] +fn relower_skips_and_counts_an_item_whose_lowering_prompt_the_round_refused() { + // ja-JP item 12 cites an entity with no ja form: the round admitted it and refused it + // at Lower. Item 13 cites none and lowers. + let mut dao = bilingual(None); + let mut model = bilingual_fake(); + model.answers.insert("item:daodejing:12".into(), citing_laozi(2)); + model.answers.insert("item:daodejing:13".into(), json(&draft_of(3))); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + assert_eq!(r.language("ja-JP").unwrap().validation_refused.lower, 1); + let (mut rec, mut arm) = (Lowerer::new(bi_blind), Lowerer::new(bi_blind)); + let (langs, _) = relowered(relower(&dao, &mut rec, &mut arm, None)); + let ja = langs.iter().find(|l| l.language == "ja-JP").unwrap(); + assert_eq!((ja.n, ja.skipped), (1, 1), "{}", ja.line()); + assert_eq!(ja.recorded, ja.arm); + let en = langs.iter().find(|l| l.language == "en-US").unwrap(); + assert_eq!((en.n, en.skipped), (2, 0), "{}", en.line()); + assert!(arm.seen.iter().all(|(_, key, _)| key != "item:daodejing:12")); +} + +/// The ja-JP item's own text, else the English one: `f` for en-US. +fn bi(key: &str, text: &str, f: fn(&str, &str) -> String) -> String { + match JA_TEXTS.iter().find(|(k, ..)| *k == key) { + Some((_, ja, _)) => ja.to_string(), + None => f(key, text), + } +} + +fn bi_blind(key: &str, text: &str) -> String { + bi(key, text, blind) +} + +fn bi_reader(key: &str, text: &str) -> String { + bi(key, text, reader) +} + +/// Fine in English, nonsense in ja-JP. +fn bi_ja_lost(key: &str, text: &str) -> String { + match JA_TEXTS.iter().any(|(k, ..)| *k == key) { + true => "zzzz".into(), + false => blind(key, text), + } +} + +#[test] +fn relower_decides_over_the_whole_panel_not_its_first_language() { + use weksa::run::decide_panel; + let mut dao = bilingual(LAOZI_JA); + recorded(round(&mut dao.run, &mut bilingual_fake(), &Scorer::new(Bag)).unwrap()); + // en-US gains (the arm drops the stance the recorded lowering repeats); ja-JP is lost. + let (langs, verdict) = relowered(relower( + &dao, + &mut Lowerer::new(bi_reader), + &mut Lowerer::new(bi_ja_lost), + Some(&mut Lowerer::new(bi_reader)), + )); + assert_eq!( + langs.iter().map(|l| (l.language.as_str(), l.n)).collect::>(), + [("en-US", 2), ("ja-JP", 2)], + "panel order, Validation only" + ); + assert!(langs[0].arm.composite > langs[0].recorded.composite, "en-US gains"); + assert!(langs[1].arm.composite < langs[1].recorded.composite, "ja-JP is lost"); + let panel: Vec<_> = langs.iter().map(|l| (l.recorded, l.arm, l.floor.unwrap())).collect(); + assert_eq!(verdict, Some(decide_panel(&panel))); + assert!(matches!(verdict, Some(Verdict::Rejected(Why::Regressed(_)))), "{verdict:?}"); + // The first language alone would have accepted. + assert_eq!(decide_panel(&panel[..1]), Verdict::Accepted); +} + +#[test] +fn relower_refuses_a_cycle_round_and_a_missing_recorded_answer() { + use weksa::model::RecordedAnswers; + let mut dao = dao(); + two_rounds(&mut dao); + run_cycle(&mut dao, &mut cycle_fake(ONE_CONCEPT)); + let candidate = dao.run.rounds().unwrap().into_iter().find(|r| r.cycle().is_some()).unwrap().n(); + let mut arm = Lowerer::new(reader); + let r = dao.run.relower(candidate, "arm", &mut Lowerer::new(reader), &mut arm, None, &Scorer::new(Bag)); + assert!(matches!(r, Err(RunError::Malformed("round")))); + let r = dao.run.relower(99, "arm", &mut Lowerer::new(reader), &mut arm, None, &Scorer::new(Bag)); + assert!(matches!(r, Err(RunError::Malformed("round")))); + assert!(arm.seen.is_empty()); + + let (dao, _) = recorded_dao(|_| {}); + let mut empty = RecordedAnswers::new(dao.root.join("round-1/nowhere"), "fake-model".into()); + let mut arm = Lowerer::new(reader); + let r = relower(&dao, &mut empty, &mut arm, None); + assert!(matches!(r, Err(RunError::Unrecorded))); + assert!(arm.seen.is_empty(), "nothing is asked of the arm before the round's answer exists"); +} + +#[test] +fn relower_floor_arm_with_an_unanswered_prompt_is_refused() { + use weksa::model::RecordedAnswers; + let (dao, _) = recorded_dao(|_| {}); + let mut floor = RecordedAnswers::new(dao.root.join("relower-1-floor"), "fake-model".into()); + let r = relower(&dao, &mut Lowerer::new(reader), &mut Lowerer::new(reader), Some(&mut floor)); + assert!(matches!(r, Err(RunError::FloorUnanswered))); + assert!(!dao.root.join("relower-1-floor").exists(), "a floor arm is never written to"); +} + +#[test] +fn relower_checks_the_embedder_not_the_model() { + struct Named(Lowerer); + impl ModelPort for Named { + fn model(&self) -> &str { + "another-model" + } + fn answer(&mut self, p: &Prompt) -> Result { + self.0.answer(p) + } + } + let (dao, _) = recorded_dao(|_| {}); + let mut other = Named(Lowerer::new(blind)); + relowered(relower(&dao, &mut Lowerer::new(reader), &mut other, None)); + let mut arm = Lowerer::new(reader); + let r = dao.run.relower(1, "arm", &mut Lowerer::new(reader), &mut arm, None, &Scorer::new(OtherEmbedder)); + assert!(matches!(r, Err(RunError::Drift(DriftOf::Embedder)))); + assert!(arm.seen.is_empty()); +} + +/// Every file under `dir` with its bytes, by relative path. +fn tree(dir: &Path) -> std::collections::BTreeMap> { + let mut out = std::collections::BTreeMap::new(); + let mut stack = vec![dir.to_path_buf()]; + while let Some(d) = stack.pop() { + for e in std::fs::read_dir(d).unwrap() { + let p = e.unwrap().path(); + if p.is_dir() { + stack.push(p); + } else { + out.insert( + p.strip_prefix(dir).unwrap().to_string_lossy().into_owned(), + std::fs::read(&p).unwrap(), + ); + } + } + } + out +} + +#[test] +fn relower_records_nothing() { + use weksa::model::RecordedAnswers; + let (dao, _) = recorded_dao(|_| {}); + let before = tree(&dao.root); + let mut recorded = RecordedAnswers::new(dao.root.join("round-1"), "fake-model".into()); + // An unanswered arm writes its prompts in its own directory only. + let mut files = dao.run.answer_files(dao.run.relower_dir(1, "arm")).unwrap(); + // The recorded answers are files here, as the relay leaves them. + let r = relower(&dao, &mut recorded, &mut files, None); + assert!(matches!(r, Err(RunError::Unrecorded)), "no round answer files yet"); + let after = tree(&dao.root); + for (path, bytes) in &before { + assert_eq!(after.get(path), Some(bytes), "{path} changed"); + } + assert!( + after.keys().all(|p| before.contains_key(p) || p.starts_with("relower-1-arm/")), + "only relower-1-arm gains files" + ); + + // With the round's answers present, the arm is asked and its prompts land in relower-1-arm. + let (dao, r) = recorded_dao(|_| {}); + for call in &r.calls()[4..] { + std::fs::write( + dao.root.join(format!("round-1/{}.answer", call.prompt)), + r#"{"text":"The highest excellence is like the water."}"#, + ) + .unwrap(); + } + let before = tree(&dao.root); + let mut recorded = RecordedAnswers::new(dao.root.join("round-1"), "fake-model".into()); + let mut files = dao.run.answer_files(dao.run.relower_dir(1, "arm")).unwrap(); + let RelowerOutcome::Unanswered { prompts } = relower(&dao, &mut recorded, &mut files, None).unwrap() else { + panic!("the arm is unanswered") + }; + assert_eq!(prompts.len(), 3); + let after = tree(&dao.root); + for (path, bytes) in &before { + assert_eq!(after.get(path), Some(bytes), "{path} changed"); + } + let added: Vec<&String> = after.keys().filter(|p| !before.contains_key(*p)).collect(); + assert!(added.iter().all(|p| p.starts_with("relower-1-arm/")), "{added:?}"); + assert_eq!(added.iter().filter(|p| p.ends_with(".prompt")).count(), 3); +} + +#[test] +fn relower_arm_prompt_holding_test_text_is_withheld() { + // A Test sentence that a Validation lowering prompt holds: a line of the + // lowering instructions, which every lowering prompt carries. + let line = LOWER_INSTRUCTIONS.lines().find(|l| l.trim().chars().count() > 30).unwrap().trim().to_string(); + let (dao, _) = recorded_dao_in( + |corpus, entities| { + let mut held = corpus.get("item:daodejing:4").unwrap().unwrap(); + held.id = "item:held:9".into(); + held.text = line.clone(); + corpus.add(held, entities).unwrap(); + }, + |_| {}, + ); + let mut files = dao.run.answer_files(dao.run.relower_dir(1, "arm")).unwrap(); + let r = relower(&dao, &mut Lowerer::new(reader), &mut files, None); + assert!(matches!(r, Err(RunError::Withheld)), "{:?}", r.map(|_| ())); + let dir = dao.run.relower_dir(1, "arm"); + let prompts = std::fs::read_dir(&dir) + .map(|d| d.filter(|e| e.as_ref().unwrap().file_name().to_string_lossy().ends_with(".prompt")).count()) + .unwrap_or(0); + assert_eq!(prompts, 0, "no prompt file is written"); +} diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 1aa7299..4a78abe 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -355,6 +355,31 @@ checkpoint the test-split composite is within the noise floor of validation. The test split is read only at checkpoints. Self then records the freeze, and the training set is the whole corpus re-encoded under the frozen revision. +## Distil measure (`distil-slice`) + +A lowerer is compared on round n's Validation packets with the prompts +byte-identical to the round's, by `weksa loop relower ROOT --round N --arm ARM +--lowerer LABEL [--floor ARM2]`. Every arm is answered through the one prompt +door (`RunStores::answer_files`, the Test-text guard); a tuned model is a script +that reads `.prompt` and writes `.answer` in +`/relower--`, not a new adapter. The recorded reading is the +round's own lowering, read over the same items as the arm. An item the round +refused before admission (no packet) or at the lowering prompt has no prompt to +ask: it is skipped, so the readings exclude it, and it is counted as `skipped=` +on the line; `n` is the items read, so `recorded` can differ from the round's +own Validation reading, which scores those items 0. An arm answer that is not a +usable lowering scores 0 and counts in `unusable=`; the same for the floor arm, +counted after `floor ... unusable=`. The floor is the spread of the recorded +lowering and a second prompted arm (lowering-only noise, no encoder noise). The +verdict is `decide_panel` over (recorded, arm, floor): Accepted is the rule's +name for a gain past the floor, Rejected(Regressed) for a loss past it, +Rejected(NoGain) for neither. None is yet a finding about the arm. The rule is calibration-pending: a one-pair floor is noisy +(a simulated prompted repeat reads NoGain only about 7-15% of the time, Regressed +44-60%, Accepted 33-41%), so NoGain is not yet evidence of non-inferiority and +is not yet the bar for cut 13; the calibration arms (a prompted repeat, a +control) decide what the floor must be before the verdict is read as a ruling. +Training pairs are Train only, at a frozen revision; the export is 12b. + ## Lowerer first cut (`golf-loop-lowerer-cut`) Authority map for the smallest lowerer, written before its code (Imagination @@ -436,5 +461,5 @@ One line per cut; specs exist for the cuts about to run (2d and 4, siblings on ` 9. `ja-panel`: ja-JP into the loop's panel, in two cuts on `e9cbd9b`. 9a `ja-panel-loop` (spec r1): language is the item's and the panel the run's: `Prompt::lower` takes the item's language and prints it on a language line (no prompt names English), entity forms are read in that language, `RunRecord` pins the panel (`weksa loop open --panel`), offers, mint and recall read only panel items, round means, floors and readings are per panel language, the accept rule runs `decide` per language, the recall split works on unspaced scripts, and `weksa corpus add` has no default language. 9b `ja-panel-corpus` (spec r1, after 9a): the ja-JP slice of `data/original`, model-written Japanese Train and Validation passages and eight Aozora Bunko public-domain Test sentences with full provenance, and their entities. Cut 5 is not a prerequisite: neither language lowers with ontology entries until cut 5 gives both theirs. 10. `conlang-lexicon`: the first conlang's lexicon placed at concept embeddings, its morphology data, its ontology entries and the gloss reader. Waits on 3, 5 and question `first-conlang`. 11. `convergence-freeze`: the convergence test, `Checkpoint`'s one door (built from the convergence window over the persisted round records), the freeze, the re-encode into training pairs. Waits on 7, 9 and 10. -12. `distil`: fine-tune a lowerer on the frozen training set. Waits on 11 and a host question asked then. +12. `distil`, three cuts. 12a `distil-slice` (relower and the prompted baseline; spec r1). 12b `distil-export`: Train-only prompt/completion JSONL at a frozen revision, Test-text guarded (follow-up `distil-export-at-freeze`); waits on 11. 12c `distil-train`: waits on 12b and the host question (follow-up `distil-host-question-owed`). 13. `live-lowering`: agents emit packets with stance; Weksa reads speaker state from persona state (follow-up `persona-state-speech-community`) and lowers for the reader. Waits on 11. From 353dd2138f33ee21a8a396072ecfbfc04e09d4de Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 03:01:52 +0000 Subject: [PATCH 09/34] relower: paired per-item noise floor over >=3 prompted repeats; panel gain clears the noise of the sum (#70) --- crates/weksa/src/main.rs | 23 ++- crates/weksa/src/run.rs | 200 +++++++++++++++++------ crates/weksa/tests/cli.rs | 18 +- crates/weksa/tests/golf_loop.rs | 82 ++++++---- docs/targets/interlingua-lowering-map.md | 34 ++-- 5 files changed, 251 insertions(+), 106 deletions(-) diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 9ba2135..fc57236 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -154,8 +154,9 @@ enum LoopCommand { ollama_url: String, }, /// Re-lower round ROUND's Validation packets through arm ARM, score them - /// against the items, and, with --floor ARM, decide against that second arm - /// by the panel accept rule. Prompts and answers live in + /// against the items, and, with --floor ARM given two or more times (prompted + /// repeats of the lowering), decide against the paired noise floor of those + /// repeats by the panel accept rule. Prompts and answers live in /// `/relower--`; the command exits 3 until the arm is /// answered. --lowerer names the arm's model in the printed line. A /// diagnostic, never a gate; records nothing; Test and Train are never @@ -169,7 +170,7 @@ enum LoopCommand { #[arg(long)] lowerer: String, #[arg(long)] - floor: Option, + floor: Vec, #[arg(long, default_value = "http://127.0.0.1:11434")] ollama_url: String, }, @@ -1147,7 +1148,10 @@ fn main() -> Result<()> { (1..=32).contains(&l.len()) && l.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') }; - if !label(&arm) || floor.as_deref().is_some_and(|f| !label(f) || f == arm) { + if !label(&arm) + || floor.iter().any(|f| !label(f) || *f == arm) + || floor.iter().enumerate().any(|(i, f)| floor[..i].contains(f)) + { anyhow::bail!("arm and floor labels must match [a-z0-9-]{{1,32}} and differ"); } let run = RunStores::open(&root)?; @@ -1156,16 +1160,19 @@ fn main() -> Result<()> { run.record().model.clone(), ); let mut files = run.answer_files(run.relower_dir(round, &arm))?; - let mut floor_port = floor.as_deref().map(|f| { - RecordedAnswers::new(run.relower_dir(round, f), run.record().model.clone()) - }); + let mut floor_ports: Vec = floor + .iter() + .map(|f| RecordedAnswers::new(run.relower_dir(round, f), run.record().model.clone())) + .collect(); + let mut floor_dyn: Vec<&mut dyn ModelPort> = + floor_ports.iter_mut().map(|p| p as &mut dyn ModelPort).collect(); let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder)); match run.relower( round, &arm, &mut recorded, &mut files, - floor_port.as_mut().map(|p| p as &mut dyn ModelPort), + &mut floor_dyn, &scorer, )? { RelowerOutcome::Unanswered { prompts } => { diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 1dc4780..dd6bc4a 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -498,6 +498,8 @@ pub enum RunError { /// The floor arm of a relower holds no answer to a prompt; it is a /// reference, never written to, and must be answered whole first. FloorUnanswered, + /// A relower was given fewer floor arms than [`FLOOR_ARMS_MIN`]. + FloorArms, } impl fmt::Display for RunError { @@ -526,6 +528,7 @@ impl fmt::Display for RunError { RunError::SpeakerState => { f.write_str("a scene does not read against the run's inventory") } + RunError::FloorArms => f.write_str("a floor needs three or more floor arms; fewer measure no noise"), RunError::FloorUnanswered => { f.write_str("the floor arm holds no answer to a lowering prompt; answer it whole first") } @@ -1586,13 +1589,14 @@ pub struct RelowerReading { pub skipped: u32, /// Arm answers that were not a usable lowering; each scored 0. pub unusable: u32, - /// Floor-arm answers that were not a usable lowering; each scored 0, which - /// widens `floor`. Zero without a floor arm. + /// Floor-arm answers that were not a usable lowering, over every floor arm; + /// each scored 0, which widens `floor`. Zero without floor arms. pub floor_unusable: u32, /// The round's own lowering, read against the items. pub recorded: Reading, pub arm: Reading, - /// `spread(recorded, floor arm)`, when a floor arm was given. + /// [`paired_floor`] of the recorded lowering against the floor arms, when + /// floor arms were given. pub floor: Option, } @@ -1637,9 +1641,10 @@ impl RunStores { /// distil decision, whether a tuned lowerer is worth keeping against the /// prompted one (calibration-pending: see the map). Asks `arm` the byte-identical lowering prompts of round `n`'s live /// Validation items, reads the arm and the round's own lowering - /// (`recorded`) against the items, and, given a `floor` arm (a second - /// prompted lowering, so the floor is lowering-only noise), decides by - /// [`decide_panel`] over (recorded, arm, spread(recorded, floor)). Only + /// (`recorded`) against the items, and, given `floors` (three or more + /// prompted repeats of the recorded lowering, so the floor is lowering-only + /// noise), decides by [`decide_panel`] over (recorded, arm, + /// [`paired_floor`]). Only /// the model of `arm` is unpinned; the embedder is checked. An arm answer /// that is not a usable lowering scores 0 and is counted in `unusable`. /// Records nothing; `arm_label` names only where the scratch packet store @@ -1650,7 +1655,7 @@ impl RunStores { arm_label: &str, recorded: &mut dyn ModelPort, arm: &mut dyn ModelPort, - mut floor: Option<&mut dyn ModelPort>, + floors: &mut [&mut dyn ModelPort], scorer: &Scorer, ) -> Result { let store = |_| RunError::Store; @@ -1666,9 +1671,12 @@ impl RunStores { } let mut scratch = PacketStore::open(&dir.join("base.cc")).map_err(store)?; - // Per language, per item: the item text, then the recorded, arm and floor - // lowerings (`None` when unusable). - type Row = (String, Option, Option, Option); + if (1..FLOOR_ARMS_MIN).contains(&floors.len()) { + return Err(RunError::FloorArms); + } + // Per language, per item: the item text, then the recorded and arm + // lowerings and one per floor arm (`None` when unusable). + type Row = (String, Option, Option, Vec>); let mut waiting: Vec = Vec::new(); let mut floor_waiting = false; let mut rows: Vec> = Vec::new(); @@ -1690,16 +1698,16 @@ impl RunStores { Asked::Waiting => return Err(RunError::Unrecorded), other => text(other), }; - let floor_text = match floor.as_deref_mut() { - Some(f) => match asked(f, &prompt)? { + let mut floor_texts = Vec::new(); + for f in floors.iter_mut() { + floor_texts.push(match asked(&mut **f, &prompt)? { Asked::Waiting => { floor_waiting = true; None } other => text(other), - }, - None => None, - }; + }); + } let arm_text = match asked(arm, &prompt)? { Asked::Waiting => { let d = prompt.digest(); @@ -1710,7 +1718,7 @@ impl RunStores { } other => text(other), }; - language_rows.push((item.text.clone(), recorded_text, arm_text, floor_text)); + language_rows.push((item.text.clone(), recorded_text, arm_text, floor_texts)); } rows.push(language_rows); } @@ -1723,8 +1731,8 @@ impl RunStores { // One score call: each usable lowering against its item, in row order. let mut inputs: Vec = Vec::new(); - for (reference, recorded, arm, floor_text) in rows.iter().flatten() { - for output in [recorded, arm, floor_text].into_iter().flatten() { + for (reference, recorded, arm, floor_texts) in rows.iter().flatten() { + for output in [recorded, arm].into_iter().chain(floor_texts).flatten() { inputs.push(ScoreInput { reference, output }); } } @@ -1751,7 +1759,8 @@ impl RunStores { let mut languages = Vec::new(); let live = self.corpus.all().map_err(store)?; for ((language, _, _), language_rows) in held.iter().zip(&rows) { - let (mut rec, mut arm_scores, mut floor_scores) = (Vec::new(), Vec::new(), Vec::new()); + let (mut rec, mut arm_scores) = (Vec::new(), Vec::new()); + let mut floor_scores: Vec> = vec![Vec::new(); floors.len()]; let (mut unusable, mut floor_unusable) = (0u32, 0u32); for (_, r, a, f) in language_rows { rec.push(next(r)?); @@ -1759,9 +1768,9 @@ impl RunStores { if a.is_none() { unusable += 1; } - if floor.is_some() { - floor_scores.push(next(f)?); - if f.is_none() { + for (scores, t) in floor_scores.iter_mut().zip(f) { + scores.push(next(t)?); + if t.is_none() { floor_unusable += 1; } } @@ -1781,14 +1790,10 @@ impl RunStores { floor_unusable, recorded: recorded_reading, arm: reading_of(&arm_scores)?, - floor: if floor.is_some() { - Some(spread(&recorded_reading, &reading_of(&floor_scores)?)) - } else { - None - }, + floor: if floors.is_empty() { None } else { Some(paired_floor(&rec, &floor_scores)) }, }); } - let verdict = if floor.is_some() { + let verdict = if !floors.is_empty() { let panel: Vec<(Reading, Reading, Reading)> = languages .iter() .filter_map(|l| Some((l.recorded, l.arm, l.floor?))) @@ -2261,7 +2266,11 @@ pub fn decide(base: &Reading, candidate: &Reading, floor: &Reading) -> Verdict { /// decides whether a language regressed (any regression rejects; the first in /// panel order names the metric), and the gain test is [`decide`]'s for one /// language and, for two or more, the panel's: the sum over languages of the -/// composite deltas (billionths) must exceed one [`LADDER_STEP`]. A language +/// composite deltas (billionths) must exceed the larger of one [`LADDER_STEP`] +/// and the noise of that sum, the root of the languages' composite floors +/// squared (independent languages; a one-language panel's noise is its floor, +/// as in [`decide`]). Without that noise term a prompted repeat of the base +/// lowering read Accepted in 40% of the calibration trials on `distil-base` at `FLOOR_Z` 3. A language /// may lose up to its own noise floor if the panel nets more than a step; the /// rule gives up strict Pareto non-regression, which on noisy per-language /// readings ratchets every base to a favourable draw and strangles the loop @@ -2289,7 +2298,10 @@ pub fn decide_panel(rows: &[(Reading, Reading, Reading)]) -> Verdict { .iter() .map(|(base, candidate, _)| nano(candidate.composite - base.composite)) .sum(); - if net > nano(LADDER_STEP) { + // The noise of a sum of independent languages' differences: the root of the + // sum of their floors squared. + let noise = rows.iter().map(|(_, _, floor)| floor.composite.powi(2)).sum::().sqrt(); + if net > nano(noise.max(LADDER_STEP)) { Verdict::Accepted } else { Verdict::Rejected(Why::NoGain) @@ -2488,6 +2500,50 @@ fn floor_of( .collect() } +/// How many standard errors of a prompted-vs-prompted mean difference the +/// relower floor is wide, and the fewest floor arms (prompted repeats of the +/// recorded lowering) it is measured over. Calibrated on `runs/distil-base` +/// (eight prompted draws of 14 en-US and 14 ja-JP Validation items, 28 +/// unordered pairs): with three floor arms a prompted repeat reads NoGain in +/// 92% of the trials (bootstrap 95% over draws 80 to 100%), Regressed 8% and +/// Accepted 0%; a "Yes." arm and a mild degradation (one detail blurred per +/// utterance) read Regressed in 100%, and a subtler one in 75% (50 to 100%): at +/// 14 items a degradation under about three standard errors cannot be told from +/// noise. `FLOOR_Z` was chosen on that data, so the rates are in-sample. +pub const FLOOR_Z: f64 = 3.0; +pub const FLOOR_ARMS_MIN: usize = 3; + +/// The lowering-only noise floor of a relower: `FLOOR_Z` standard errors of the +/// mean per-item difference between the recorded lowering and a repeat of it, +/// metric by metric. Each floor arm gives one paired sample of per-item +/// differences; their variances are pooled over the arms, so the floor rests on +/// `arms * (n - 1)` degrees of freedom instead of the one coincidental +/// difference of means a two-run spread is (which reads 0 whenever the two +/// means happen to meet, and then a one-step dip regresses). Paired per item, +/// after Koehn (2004). NaN (fail closed in [`decide`]) when there are fewer +/// than two items or no arm (0/0), and when an arm is ragged. +pub fn paired_floor(recorded: &[ItemScore], arms: &[Vec]) -> Reading { + let n = recorded.len(); + if arms.iter().any(|a| a.len() != n) { + return Reading { chrf: f64::NAN, meaning: f64::NAN, composite: f64::NAN }; + } + let pooled = |of: &dyn Fn(&ItemScore) -> f64| { + let variance = |arm: &Vec| { + let d: Vec = recorded.iter().zip(arm).map(|(r, a)| of(r) - of(a)).collect(); + let mean = d.iter().sum::() / n as f64; + d.iter().map(|x| (x - mean).powi(2)).sum::() / (n - 1) as f64 + }; + let v = arms.iter().map(variance).sum::() / arms.len() as f64; + FLOOR_Z * (v / n as f64).sqrt() + }; + let meaning = |s: &ItemScore| s.meaning.unwrap_or(0.0); + Reading { + chrf: pooled(&|s| s.chrf), + meaning: pooled(&meaning), + composite: pooled(&|s| (s.chrf / 100.0 + meaning(s)) / 2.0), + } +} + /// The absolute difference of two readings, metric by metric. fn spread(a: &Reading, b: &Reading) -> Reading { Reading { @@ -2869,6 +2925,52 @@ mod tests { assert_eq!(decide(&base, &gain(0.506), &floor), rejected(Why::NoGain)); } + #[test] + fn paired_floor_is_z_standard_errors_of_the_paired_difference() { + let item = |chrf: f64, meaning: f64| ItemScore { chrf, meaning: Some(meaning) }; + let recorded = [item(60.0, 0.8), item(40.0, 0.6), item(50.0, 0.7), item(30.0, 0.5)]; + // An arm that differs by a constant has no spread of differences: a + // shifted mean is the thing the floor is read against, not noise. + let shifted: Vec = recorded.iter().map(|s| item(s.chrf - 5.0, s.meaning.unwrap() - 0.1)).collect(); + let f = paired_floor(&recorded, &[shifted.clone(), shifted.clone(), shifted]); + assert!(f.chrf.abs() < 1e-9 && f.meaning.abs() < 1e-9 && f.composite.abs() < 1e-9, "{f:?}"); + // Differences 2, -2, 2, -2 on chrF: sample variance 16/3, standard error + // sqrt(16/3/4), FLOOR_Z of it; arms pool, so a second equal arm changes nothing. + let wobble_arm = |recorded: &[ItemScore]| -> Vec { + recorded + .iter() + .enumerate() + .map(|(i, s)| item(s.chrf - if i % 2 == 0 { 2.0 } else { -2.0 }, s.meaning.unwrap())) + .collect() + }; + let wobble = wobble_arm(&recorded); + let want = FLOOR_Z * (16.0_f64 / 3.0 / 4.0).sqrt(); + let one = paired_floor(&recorded, std::slice::from_ref(&wobble)); + let two = paired_floor(&recorded, &[wobble.clone(), wobble.clone()]); + assert!((one.chrf - want).abs() < 1e-9 && (two.chrf - want).abs() < 1e-9, "{one:?} {two:?}"); + assert!(one.meaning.abs() < 1e-9); + // Arms pool by the mean of their variances: one wobbling arm and one + // exact repeat give the root of half the variance. + let exact = recorded.to_vec(); + let mixed = paired_floor(&recorded, &[wobble.clone(), exact]); + assert!((mixed.chrf - FLOOR_Z * (16.0_f64 / 3.0 / 2.0 / 4.0).sqrt()).abs() < 1e-9, "{mixed:?}"); + // The calibrated constants, pinned apart from the formula. + assert_eq!((FLOOR_Z, FLOOR_ARMS_MIN), (3.0, 3)); + // Under two items, no arms or a ragged arm there is no floor: NaN fails + // closed in `decide`. + for f in [ + paired_floor(&recorded[..1], &[vec![item(1.0, 1.0)]]), + paired_floor(&recorded, &[]), + paired_floor(&recorded, &[vec![item(1.0, 1.0)]]), + ] { + assert!(f.composite.is_nan()); + assert_eq!( + decide(&reading(50.0, 0.5, 0.5), &reading(50.0, 0.5, 0.9), &f), + Verdict::Rejected(Why::Regressed(Metric::Composite)) + ); + } + } + #[test] fn decide_panel_table() { let zero = reading(0.0, 0.0, 0.0); @@ -2922,9 +3024,8 @@ mod tests { assert_eq!(net(LADDER_STEP + 6e-10), Verdict::Accepted); assert_eq!(net(LADDER_STEP + 2e-9), Verdict::Accepted); // Liveness: no language need pass its own threshold. Two languages each - // gaining 0.6 of a step at zero floor accept; so does one gaining two - // steps under a 0.02 floor (its own threshold) with the others flat; so - // does a gain with another language falling inside its own floor. + // gaining 0.6 of a step at zero floor accept; so does a gain past the + // noise of the sum with another language falling inside its own floor. let each = 0.6 * LADDER_STEP; assert_eq!( decide_panel(&[ @@ -2933,27 +3034,32 @@ mod tests { ]), Verdict::Accepted ); + // The sum must also clear its own noise: the root of the floors squared + // (0.02 and 0.02 give 0.0283), not just a step. A repeat of the base + // lowering gains that much by chance. let big_floor = reading(0.0, 0.0, 0.02); - assert_eq!( - decide_panel(&[(base, reading(50.0, 0.5, 0.5 + 2.0 * LADDER_STEP), big_floor), (base, flat, big_floor)]), - Verdict::Accepted - ); + let sum_of = |gain: f64| { + decide_panel(&[(base, reading(50.0, 0.5, 0.5 + gain), big_floor), (base, flat, big_floor)]) + }; + assert_eq!(sum_of(0.02), rejected(Why::NoGain), "past a step, under the noise of the sum"); + assert_eq!(sum_of(0.028), rejected(Why::NoGain)); + assert_eq!(sum_of(0.029), Verdict::Accepted); let own_floor = reading(0.0, 0.0, 0.004); + assert_eq!( + decide_panel(&[ + (base, reading(50.0, 0.5, 0.55), reading(0.0, 0.0, 0.03)), + (base, reading(50.0, 0.5, 0.497), own_floor) + ]), + Verdict::Accepted, + "a fall inside the language's floor, net 0.047 over a noise of 0.0303" + ); assert_eq!( decide_panel(&[ (base, reading(50.0, 0.5, 0.51), reading(0.0, 0.0, 0.03)), (base, reading(50.0, 0.5, 0.497), own_floor) ]), - Verdict::Accepted, - "a fall inside the language's floor, net 0.007" - ); - // The same fall with the net exactly a step rejects. - assert_eq!( - decide_panel(&[ - (base, reading(50.0, 0.5, 0.508), reading(0.0, 0.0, 0.03)), - (base, reading(50.0, 0.5, 0.497), own_floor) - ]), - rejected(Why::NoGain) + rejected(Why::NoGain), + "net 0.007 is under the noise" ); // The sum is over the deltas, with each language's fall counted. assert_eq!( diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 95a147a..e3abfc7 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -1833,12 +1833,19 @@ fn cli_loop_relower_exits_3_until_answered_and_refuses_bad_labels() { assert_eq!(o.status.code(), Some(1), "{}", out(&o)); assert!(!root.join("relower-1-second").exists()); - // Answered floor: a verdict line. - std::fs::create_dir(root.join("relower-1-second")).unwrap(); - for digest in &digests { - std::fs::write(root.join(format!("relower-1-second/{digest}.answer")), said).unwrap(); - } + // Fewer than three floor arms is refused: one pair measures no noise. let o = relower(&["--arm", "tuned", "--floor", "second"]); + assert_eq!(o.status.code(), Some(1), "{}", out(&o)); + assert!(out(&o).contains("three or more floor arms"), "{}", out(&o)); + + // Answered floors: a verdict line. + for label in ["second", "third", "fourth"] { + std::fs::create_dir_all(root.join(format!("relower-1-{label}"))).unwrap(); + for digest in &digests { + std::fs::write(root.join(format!("relower-1-{label}/{digest}.answer")), said).unwrap(); + } + } + let o = relower(&["--arm", "tuned", "--floor", "second", "--floor", "third", "--floor", "fourth"]); assert!(o.status.success(), "{}", out(&o)); let text = out(&o); assert!(text.contains("| floor chrf=0.00") && text.trim_end().ends_with("verdict Rejected(NoGain)"), "{text}"); @@ -1852,6 +1859,7 @@ fn cli_loop_relower_exits_3_until_answered_and_refuses_bad_labels() { vec!["--arm", &long], vec!["--arm", "../x"], vec!["--arm", "tuned", "--floor", "Bad"], + vec!["--arm", "tuned", "--floor", "b", "--floor", "b", "--floor", "c"], ] { let o = relower(&args); assert_eq!(o.status.code(), Some(1), "{args:?}: {}", out(&o)); diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 179d0ea..2d0320a 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -5335,14 +5335,14 @@ fn the_cycles_candidate_round_stops_on_a_withheld_lowering_prompt() { use weksa::run::{RelowerOutcome, RelowerReading}; -/// `relower` of round 1 with the Bag scorer; `floor` is an optional second arm. +/// `relower` of round 1 with the Bag scorer; `floors` are the floor arms (none, or three or more). fn relower( dao: &Dao, recorded: &mut dyn ModelPort, arm: &mut dyn ModelPort, - floor: Option<&mut dyn ModelPort>, + floors: &mut [&mut dyn ModelPort], ) -> Result { - dao.run.relower(1, "arm", recorded, arm, floor, &Scorer::new(Bag)) + dao.run.relower(1, "arm", recorded, arm, floors, &Scorer::new(Bag)) } fn relowered(outcome: Result) -> (Vec, Option) { @@ -5360,7 +5360,7 @@ fn relower_asks_only_validation_lowering_prompts_of_the_round() { // which digest is which item's. let (mut every, mut rec, mut arm) = (Lowerer::new(reader), Lowerer::new(reader), Lowerer::new(reader)); read(ablate(&dao, &mut every, &mut Lowerer::new(reader)).unwrap()); - relowered(relower(&dao, &mut rec, &mut arm, None)); + relowered(relower(&dao, &mut rec, &mut arm, &mut [])); let of = |keys: &[&str]| -> Vec { every .seen @@ -5400,7 +5400,7 @@ fn relower_reads_recorded_and_arm_against_the_item() { &dao, &mut Lowerer::new(reader), &mut Lowerer::new(blind), - None, + &mut [], )); let l = &langs[0]; assert_eq!((l.language.as_str(), l.n, l.unusable), ("en-US", 3, 0)); @@ -5408,7 +5408,7 @@ fn relower_reads_recorded_and_arm_against_the_item() { assert!(l.arm.composite > l.recorded.composite); assert!(l.floor.is_none() && verdict.is_none(), "no floor arm, no verdict"); // An arm equal to the recorded answers reads equal. - let (langs, _) = relowered(relower(&dao, &mut Lowerer::new(reader), &mut Lowerer::new(reader), None)); + let (langs, _) = relowered(relower(&dao, &mut Lowerer::new(reader), &mut Lowerer::new(reader), &mut [])); assert_eq!(langs[0].recorded, langs[0].arm); } @@ -5432,11 +5432,11 @@ impl ModelPort for Breaks { fn relower_counts_an_unusable_arm_answer_as_zero() { let (dao, _) = recorded_dao(|_| {}); let keys = ["item:daodejing:2", "item:daodejing:3", WRITTEN]; - let (whole, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), None)); + let (whole, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), &mut [])); let whole = whole[0].arm; let (mut chrf, mut meaning) = (0.0, 0.0); for key in keys { - let (langs, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Breaks(key), None)); + let (langs, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Breaks(key), &mut [])); let l = &langs[0]; assert_eq!((l.n, l.unusable), (3, 1), "{}", l.line()); assert!(l.arm.chrf < whole.chrf && l.arm.meaning < whole.meaning, "{}", l.line()); @@ -5452,8 +5452,8 @@ fn relower_counts_an_unusable_arm_answer_as_zero() { #[test] fn relower_scores_an_unusable_recorded_answer_as_zero_without_counting_it_against_the_arm() { let (dao, _) = recorded_dao(|_| {}); - let (whole, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), None)); - let (langs, _) = relowered(relower(&dao, &mut Breaks("item:daodejing:3"), &mut Lowerer::new(blind), None)); + let (whole, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), &mut [])); + let (langs, _) = relowered(relower(&dao, &mut Breaks("item:daodejing:3"), &mut Lowerer::new(blind), &mut [])); let l = &langs[0]; assert_eq!(l.unusable, 0, "{}", l.line()); assert_eq!(l.arm, whole[0].arm); @@ -5465,16 +5465,16 @@ fn relower_decides_by_decide_panel_against_the_floor_arm() { use weksa::run::decide_panel; let (dao, _) = recorded_dao(|_| {}); let run = |recorded: fn(&str, &str) -> String, arm: fn(&str, &str) -> String| { - let mut floor = Lowerer::new(recorded); + let (mut f1, mut f2, mut f3) = (Lowerer::new(recorded), Lowerer::new(recorded), Lowerer::new(recorded)); let (langs, verdict) = relowered(relower( &dao, &mut Lowerer::new(recorded), &mut Lowerer::new(arm), - Some(&mut floor), + &mut [&mut f1, &mut f2, &mut f3], )); let l = &langs[0]; let floor_reading = l.floor.expect("a floor arm gives a floor"); - // A floor arm that equals the recorded answers has no spread. + // Floor arms that equal the recorded answers have no spread. assert_eq!((floor_reading.chrf, floor_reading.meaning, floor_reading.composite), (0.0, 0.0, 0.0)); assert_eq!(verdict, Some(decide_panel(&[(l.recorded, l.arm, floor_reading)]))); verdict.unwrap() @@ -5485,29 +5485,34 @@ fn relower_decides_by_decide_panel_against_the_floor_arm() { } #[test] -fn relower_floor_is_the_spread_of_recorded_and_the_floor_arm() { +fn relower_floor_is_the_paired_noise_of_recorded_against_the_floor_arms() { let (dao, _) = recorded_dao(|_| {}); let (langs, _) = relowered(relower( &dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), - Some(&mut Breaks("item:daodejing:3")), + &mut [&mut Lowerer::new(blind), &mut Lowerer::new(blind), &mut Breaks("item:daodejing:3")], )); let l = &langs[0]; let f = l.floor.unwrap(); - // The floor arm read as an arm against the same recorded answers. - let (as_arm, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Breaks("item:daodejing:3"), None)); - let a = as_arm[0].arm; - assert!(f.chrf > 0.0 && f.meaning > 0.0 && f.composite > 0.0); - assert!((f.chrf - (l.recorded.chrf - a.chrf).abs()).abs() < 1e-9, "{}", l.line()); - assert!((f.composite - (l.recorded.composite - a.composite).abs()).abs() < 1e-9); + assert!(f.chrf > 0.0 && f.meaning > 0.0 && f.composite > 0.0, "{}", l.line()); assert!(l.line().contains(&format!("| floor chrf={:.2}", f.chrf)), "{}", l.line()); assert_eq!(l.unusable, 0, "unusable counts the arm, not the floor"); - assert_eq!(l.floor_unusable, 1, "the floor arm's refusal is counted apart"); + assert_eq!(l.floor_unusable, 1, "a floor arm's refusal is counted apart"); assert!(l.line().contains(" unusable=0 recorded "), "{}", l.line()); assert!(l.line().ends_with(" unusable=1"), "the floor's refusal is on the line: {}", l.line()); - // Without a floor arm there is none to count. - assert_eq!(as_arm[0].floor_unusable, 0); + // A floor needs three arms: fewer measure no noise, and nothing is asked of the arm. + for arms in [1usize, 2] { + let mut fs: Vec = (0..arms).map(|_| Lowerer::new(blind)).collect(); + let mut dyns: Vec<&mut dyn ModelPort> = fs.iter_mut().map(|f| f as &mut dyn ModelPort).collect(); + let mut arm = Lowerer::new(blind); + let r = relower(&dao, &mut Lowerer::new(blind), &mut arm, &mut dyns); + assert!(matches!(r, Err(RunError::FloorArms))); + assert!(arm.seen.is_empty()); + } + // Without floor arms there is none to count. + let (none, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), &mut [])); + assert_eq!(none[0].floor_unusable, 0); } #[test] @@ -5517,7 +5522,7 @@ fn relower_skips_and_counts_an_item_the_round_refused_before_admission() { answers.insert("item:daodejing:3".into(), "not a draft".into()); }); let (mut rec, mut arm) = (Lowerer::new(blind), Lowerer::new(blind)); - let (langs, _) = relowered(relower(&dao, &mut rec, &mut arm, None)); + let (langs, _) = relowered(relower(&dao, &mut rec, &mut arm, &mut [])); let l = &langs[0]; assert_eq!((l.n, l.skipped, l.unusable), (2, 1, 0), "{}", l.line()); assert!(l.line().starts_with("relower en-US n=2 skipped=1 unusable=0 "), "{}", l.line()); @@ -5536,7 +5541,7 @@ fn relower_skips_and_counts_an_item_whose_lowering_prompt_the_round_refused() { let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); assert_eq!(r.language("ja-JP").unwrap().validation_refused.lower, 1); let (mut rec, mut arm) = (Lowerer::new(bi_blind), Lowerer::new(bi_blind)); - let (langs, _) = relowered(relower(&dao, &mut rec, &mut arm, None)); + let (langs, _) = relowered(relower(&dao, &mut rec, &mut arm, &mut [])); let ja = langs.iter().find(|l| l.language == "ja-JP").unwrap(); assert_eq!((ja.n, ja.skipped), (1, 1), "{}", ja.line()); assert_eq!(ja.recorded, ja.arm); @@ -5579,7 +5584,7 @@ fn relower_decides_over_the_whole_panel_not_its_first_language() { &dao, &mut Lowerer::new(bi_reader), &mut Lowerer::new(bi_ja_lost), - Some(&mut Lowerer::new(bi_reader)), + &mut [&mut Lowerer::new(bi_reader), &mut Lowerer::new(bi_reader), &mut Lowerer::new(bi_reader)], )); assert_eq!( langs.iter().map(|l| (l.language.as_str(), l.n)).collect::>(), @@ -5603,16 +5608,16 @@ fn relower_refuses_a_cycle_round_and_a_missing_recorded_answer() { run_cycle(&mut dao, &mut cycle_fake(ONE_CONCEPT)); let candidate = dao.run.rounds().unwrap().into_iter().find(|r| r.cycle().is_some()).unwrap().n(); let mut arm = Lowerer::new(reader); - let r = dao.run.relower(candidate, "arm", &mut Lowerer::new(reader), &mut arm, None, &Scorer::new(Bag)); + let r = dao.run.relower(candidate, "arm", &mut Lowerer::new(reader), &mut arm, &mut [], &Scorer::new(Bag)); assert!(matches!(r, Err(RunError::Malformed("round")))); - let r = dao.run.relower(99, "arm", &mut Lowerer::new(reader), &mut arm, None, &Scorer::new(Bag)); + let r = dao.run.relower(99, "arm", &mut Lowerer::new(reader), &mut arm, &mut [], &Scorer::new(Bag)); assert!(matches!(r, Err(RunError::Malformed("round")))); assert!(arm.seen.is_empty()); let (dao, _) = recorded_dao(|_| {}); let mut empty = RecordedAnswers::new(dao.root.join("round-1/nowhere"), "fake-model".into()); let mut arm = Lowerer::new(reader); - let r = relower(&dao, &mut empty, &mut arm, None); + let r = relower(&dao, &mut empty, &mut arm, &mut []); assert!(matches!(r, Err(RunError::Unrecorded))); assert!(arm.seen.is_empty(), "nothing is asked of the arm before the round's answer exists"); } @@ -5622,7 +5627,12 @@ fn relower_floor_arm_with_an_unanswered_prompt_is_refused() { use weksa::model::RecordedAnswers; let (dao, _) = recorded_dao(|_| {}); let mut floor = RecordedAnswers::new(dao.root.join("relower-1-floor"), "fake-model".into()); - let r = relower(&dao, &mut Lowerer::new(reader), &mut Lowerer::new(reader), Some(&mut floor)); + let r = relower( + &dao, + &mut Lowerer::new(reader), + &mut Lowerer::new(reader), + &mut [&mut floor, &mut Lowerer::new(reader), &mut Lowerer::new(reader)], + ); assert!(matches!(r, Err(RunError::FloorUnanswered))); assert!(!dao.root.join("relower-1-floor").exists(), "a floor arm is never written to"); } @@ -5640,9 +5650,9 @@ fn relower_checks_the_embedder_not_the_model() { } let (dao, _) = recorded_dao(|_| {}); let mut other = Named(Lowerer::new(blind)); - relowered(relower(&dao, &mut Lowerer::new(reader), &mut other, None)); + relowered(relower(&dao, &mut Lowerer::new(reader), &mut other, &mut [])); let mut arm = Lowerer::new(reader); - let r = dao.run.relower(1, "arm", &mut Lowerer::new(reader), &mut arm, None, &Scorer::new(OtherEmbedder)); + let r = dao.run.relower(1, "arm", &mut Lowerer::new(reader), &mut arm, &mut [], &Scorer::new(OtherEmbedder)); assert!(matches!(r, Err(RunError::Drift(DriftOf::Embedder)))); assert!(arm.seen.is_empty()); } @@ -5676,7 +5686,7 @@ fn relower_records_nothing() { // An unanswered arm writes its prompts in its own directory only. let mut files = dao.run.answer_files(dao.run.relower_dir(1, "arm")).unwrap(); // The recorded answers are files here, as the relay leaves them. - let r = relower(&dao, &mut recorded, &mut files, None); + let r = relower(&dao, &mut recorded, &mut files, &mut []); assert!(matches!(r, Err(RunError::Unrecorded)), "no round answer files yet"); let after = tree(&dao.root); for (path, bytes) in &before { @@ -5699,7 +5709,7 @@ fn relower_records_nothing() { let before = tree(&dao.root); let mut recorded = RecordedAnswers::new(dao.root.join("round-1"), "fake-model".into()); let mut files = dao.run.answer_files(dao.run.relower_dir(1, "arm")).unwrap(); - let RelowerOutcome::Unanswered { prompts } = relower(&dao, &mut recorded, &mut files, None).unwrap() else { + let RelowerOutcome::Unanswered { prompts } = relower(&dao, &mut recorded, &mut files, &mut []).unwrap() else { panic!("the arm is unanswered") }; assert_eq!(prompts.len(), 3); @@ -5727,7 +5737,7 @@ fn relower_arm_prompt_holding_test_text_is_withheld() { |_| {}, ); let mut files = dao.run.answer_files(dao.run.relower_dir(1, "arm")).unwrap(); - let r = relower(&dao, &mut Lowerer::new(reader), &mut files, None); + let r = relower(&dao, &mut Lowerer::new(reader), &mut files, &mut []); assert!(matches!(r, Err(RunError::Withheld)), "{:?}", r.map(|_| ())); let dir = dao.run.relower_dir(1, "arm"); let prompts = std::fs::read_dir(&dir) diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 4a78abe..31018f6 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -359,7 +359,7 @@ the training set is the whole corpus re-encoded under the frozen revision. A lowerer is compared on round n's Validation packets with the prompts byte-identical to the round's, by `weksa loop relower ROOT --round N --arm ARM ---lowerer LABEL [--floor ARM2]`. Every arm is answered through the one prompt +--lowerer LABEL [--floor ARM2 --floor ARM3 --floor ARM4]`. Every arm is answered through the one prompt door (`RunStores::answer_files`, the Test-text guard); a tuned model is a script that reads `.prompt` and writes `.answer` in `/relower--`, not a new adapter. The recorded reading is the @@ -369,15 +369,29 @@ ask: it is skipped, so the readings exclude it, and it is counted as `skipped=` on the line; `n` is the items read, so `recorded` can differ from the round's own Validation reading, which scores those items 0. An arm answer that is not a usable lowering scores 0 and counts in `unusable=`; the same for the floor arm, -counted after `floor ... unusable=`. The floor is the spread of the recorded -lowering and a second prompted arm (lowering-only noise, no encoder noise). The -verdict is `decide_panel` over (recorded, arm, floor): Accepted is the rule's -name for a gain past the floor, Rejected(Regressed) for a loss past it, -Rejected(NoGain) for neither. None is yet a finding about the arm. The rule is calibration-pending: a one-pair floor is noisy -(a simulated prompted repeat reads NoGain only about 7-15% of the time, Regressed -44-60%, Accepted 33-41%), so NoGain is not yet evidence of non-inferiority and -is not yet the bar for cut 13; the calibration arms (a prompted repeat, a -control) decide what the floor must be before the verdict is read as a ruling. +counted (over all floor arms) after `floor ... unusable=`. The floor needs three +or more floor arms, each a prompted repeat of the recorded lowering on the same +prompts: it is `FLOOR_Z` (3) standard errors of the mean per-item difference +between the recorded lowering and a repeat, per metric, with the variances +pooled over the arms (`paired_floor`; lowering-only noise, no encoder noise). The +verdict is `decide_panel` over (recorded, arm, floor): Accepted is a composite +gain past the noise of the panel's sum (the root of the languages' composite +floors squared, at least one step), Rejected(Regressed) a loss past a language's +floor, Rejected(NoGain) neither. The cycle's own floor is still the spread of two +round replicates (`floor_of`); it has no per-item scores to pair. +Calibration (`runs/distil-base`, 8 prompted draws, each its own subagent, 14 +en-US and 14 ja-JP Validation items; every base, candidate and 3-arm floor +combination, 1120 trials from 28 unordered pairs; intervals bootstrap over +draws, each degraded arm one fixed draw): a prompted repeat reads NoGain 92% +(80 to 100), Regressed 8% (0 to 20), Accepted 0%, which with 28 pairs bounds +false-accept only to about 10%; an all-"Yes." arm reads Regressed 100%; a mild +degradation (one detail blurred per utterance) 100%; a subtler one 75% (50 to +100). At 14 items a degradation under about three standard errors (a composite +near 0.06) cannot be told from noise, and no arm known to be better exists, so +the rule's power to accept a real gain is unmeasured. `FLOOR_Z` was chosen on +this data (2.5 gives NoGain 84% and 93% on the subtler arm), so the rates are +in-sample. Without the noise term in the gain test a prompted repeat reads +Accepted in 40% of the trials. Training pairs are Train only, at a frozen revision; the export is 12b. ## Lowerer first cut (`golf-loop-lowerer-cut`) From 98ab798ca539e0577a79b7c30cf645a533eaa087 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 03:39:17 +0000 Subject: [PATCH 10/34] paired floor: one composite formula and its pin; docs state the power measure (#72) --- crates/weksa/src/run.rs | 20 +++++++++++++++----- docs/targets/interlingua-lowering-map.md | 24 +++++++++++++++++------- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index dd6bc4a..a9062a1 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -2157,6 +2157,12 @@ fn score_round( Ok(RoundOutcome::Recorded(record)) } +/// The composite of one chrF (0 to 100) and one meaning: their mean, chrF as a +/// fraction. The one formula for a split mean's [`Reading`] and a per-item score. +fn composite_of(chrf: f64, meaning: f64) -> f64 { + (chrf / 100.0 + meaning) / 2.0 +} + /// One validation reading: chrF 0 to 100, meaning 0 to 1, and their composite. #[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct Reading { @@ -2177,7 +2183,7 @@ impl Reading { Some(Self { chrf, meaning, - composite: (chrf / 100.0 + meaning) / 2.0, + composite: composite_of(chrf, meaning), }) } } @@ -2507,9 +2513,10 @@ fn floor_of( /// unordered pairs): with three floor arms a prompted repeat reads NoGain in /// 92% of the trials (bootstrap 95% over draws 80 to 100%), Regressed 8% and /// Accepted 0%; a "Yes." arm and a mild degradation (one detail blurred per -/// utterance) read Regressed in 100%, and a subtler one in 75% (50 to 100%): at -/// 14 items a degradation under about three standard errors cannot be told from -/// noise. `FLOOR_Z` was chosen on that data, so the rates are in-sample. +/// utterance) read Regressed in 100% (each a single fixed draw), and a subtler +/// one in 75% (50 to 100%): at 14 items a mean composite drop near 0.04 against +/// a floor near 0.06 is flagged only some of the time. `FLOOR_Z` was chosen on +/// that data, so the rates are in-sample. pub const FLOOR_Z: f64 = 3.0; pub const FLOOR_ARMS_MIN: usize = 3; @@ -2540,7 +2547,7 @@ pub fn paired_floor(recorded: &[ItemScore], arms: &[Vec]) -> Reading Reading { chrf: pooled(&|s| s.chrf), meaning: pooled(&meaning), - composite: pooled(&|s| (s.chrf / 100.0 + meaning(s)) / 2.0), + composite: pooled(&|s| composite_of(s.chrf, meaning(s))), } } @@ -2949,6 +2956,9 @@ mod tests { let two = paired_floor(&recorded, &[wobble.clone(), wobble.clone()]); assert!((one.chrf - want).abs() < 1e-9 && (two.chrf - want).abs() < 1e-9, "{one:?} {two:?}"); assert!(one.meaning.abs() < 1e-9); + // Composite per item is (chrF / 100 + meaning) / 2: the wobble moves chrF + // by 2 and meaning by 0, so the composite wobbles by 0.01. + assert!((one.composite - FLOOR_Z * (0.01_f64 * 0.01 * 4.0 / 3.0 / 4.0).sqrt()).abs() < 1e-9, "{one:?}"); // Arms pool by the mean of their variances: one wobbling arm and one // exact repeat give the root of half the variance. let exact = recorded.to_vec(); diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 31018f6..fbf2a50 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -379,19 +379,29 @@ gain past the noise of the panel's sum (the root of the languages' composite floors squared, at least one step), Rejected(Regressed) a loss past a language's floor, Rejected(NoGain) neither. The cycle's own floor is still the spread of two round replicates (`floor_of`); it has no per-item scores to pair. -Calibration (`runs/distil-base`, 8 prompted draws, each its own subagent, 14 +Calibration (`runs/distil-base`, 8 prompted draws by one model family on +identical prompts, no draw sharing more than 7 of 28 answers with another; 14 en-US and 14 ja-JP Validation items; every base, candidate and 3-arm floor combination, 1120 trials from 28 unordered pairs; intervals bootstrap over draws, each degraded arm one fixed draw): a prompted repeat reads NoGain 92% (80 to 100), Regressed 8% (0 to 20), Accepted 0%, which with 28 pairs bounds false-accept only to about 10%; an all-"Yes." arm reads Regressed 100%; a mild degradation (one detail blurred per utterance) 100%; a subtler one 75% (50 to -100). At 14 items a degradation under about three standard errors (a composite -near 0.06) cannot be told from noise, and no arm known to be better exists, so -the rule's power to accept a real gain is unmeasured. `FLOOR_Z` was chosen on -this data (2.5 gives NoGain 84% and 93% on the subtler arm), so the rates are -in-sample. Without the noise term in the gain test a prompted repeat reads -Accepted in 40% of the trials. +100), whose mean composite drop (0.044) sits under the mean floor (0.060). +`FLOOR_Z` was chosen on this data (2.5 gives NoGain 84% and 93% on the subtler +arm), so the rates are in-sample. Without the noise term in the gain test a +prompted repeat reads Accepted in 40% of the trials. +Power (can the rule Accept a real gain?): the only better arm that could be +built without Test or Validation answers is a lowerer shown the Train sentences +of the corpus (no packets, no Validation or Test text) as a register guide. +Four such draws beat the prompted draws' mean composite in both languages (en-US +0.742 to 0.759 to 0.795; ja-JP 0.559 to 0.578 to 0.620), so the premise that it is +better is measured on the very Validation items the rule reads, not independent. +Against the prompted bases with 3-arm floors the rule reads it Accepted in 29% +of 1120 trials, NoGain 71%, Regressed 0%: a net gain of 0.079 against a mean +noise term of 0.079. At 14 items per language the rule is therefore a low-power +filter for a gain of this size; by 1/sqrt(n), about 35 or more items per +language would bring the noise term near 0.05 (an extrapolation, not measured). Training pairs are Train only, at a frozen revision; the export is 12b. ## Lowerer first cut (`golf-loop-lowerer-cut`) From 516eb4b885c9a1ccda5412cd0c3a0ed1a734dcc5 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 03:47:53 +0000 Subject: [PATCH 11/34] round: per-item Validation scores in the round record (cycle-floor-scores) (#71) --- crates/weksa/src/run.rs | 30 ++- crates/weksa/src/score/mod.rs | 46 +++- crates/weksa/tests/golf_loop.rs | 231 +++++++++++++++++++ docs/targets/interlingua-lowering-map.md | 2 +- schemas/cultnet/weksa.loop_round.schema.json | 50 +++- 5 files changed, 349 insertions(+), 10 deletions(-) diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index a9062a1..62a8815 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -46,7 +46,7 @@ use crate::encode::{ use crate::form::{is_attestation, is_language_tag, is_slug}; use crate::model::{AnswerFiles, ModelPort, PortError}; use crate::packet::{Packet, Stance}; -use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate}; +use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate, validation_items}; use crate::split::{Split, SplitScheme, fnv1a64, fnv1a64_of, hex16}; use crate::store::{ CorpusStore, EntityStore, InventoryStore, OntologyStore, PacketStore, SpeakerStateStore, @@ -246,9 +246,18 @@ pub struct LanguageRound { pub validation_refused: RefusalCounts, } +/// One panel language's Validation scores in a round, in corpus order: numbers +/// only, no item key and no text. Only `run.rs` reads them (the paired floor). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct LanguageScores { + pub language: String, + pub scores: Vec, +} + /// One scored round. It holds no text and no Test reading: per-item rows for -/// Train only, Validation only as means and refusal counts by stage. Only -/// `run::round` builds one. +/// Train only; Validation as means and refusal counts by stage, and as +/// per-item scores (numbers only, private to this module). Only `run::round` +/// builds one. /// /// ``` /// fn f(r: &weksa::run::RoundRecord) -> u32 { r.n() } @@ -277,6 +286,7 @@ pub struct LanguageRound { /// parse_reasons: vec![], /// own_family: 0, /// languages: vec![], +/// validation_scores: vec![], /// } /// } /// ``` @@ -336,6 +346,12 @@ pub struct RoundRecord { // Key 20 is `own_family`; the per-language readings take 21. #[cultcache(key = 21)] languages: Vec, + /// Each panel language's live Validation scores in corpus order, a refused + /// item as 0. Private with no getter: only `run.rs` (the floor) reads them, + /// so a leak outside it is a compile error. A run from before has none. + #[cultcache(key = 22, default)] + #[schemars(default)] + validation_scores: Vec, } impl RoundRecord { @@ -2069,6 +2085,7 @@ fn score_round( let mut train_rows = Vec::new(); let mut languages: Vec = Vec::new(); + let mut validation_scores: Vec = Vec::new(); for tag in &run.record.panel { let of: Vec<(&CorpusItem, ItemScore)> = scored .iter() @@ -2076,6 +2093,10 @@ fn score_round( .cloned() .collect(); let means = aggregate(&of); + validation_scores.push(LanguageScores { + language: tag.clone(), + scores: validation_items(&of), + }); languages.push(LanguageRound { language: tag.clone(), train_mean: means.train, @@ -2153,6 +2174,7 @@ fn score_round( .collect(), own_family, languages, + validation_scores, }; Ok(RoundOutcome::Recorded(record)) } @@ -2771,6 +2793,7 @@ mod tests { parse_reasons: vec![], own_family: 0, languages: vec![], + validation_scores: vec![], }; assert_eq!(r.n(), 1); } @@ -3286,6 +3309,7 @@ mod tests { validation_scored: 2, validation_refused: RefusalCounts::default(), }], + validation_scores: vec![], } } diff --git a/crates/weksa/src/score/mod.rs b/crates/weksa/src/score/mod.rs index c1033a4..e411d7c 100644 --- a/crates/weksa/src/score/mod.rs +++ b/crates/weksa/src/score/mod.rs @@ -28,7 +28,7 @@ pub struct ScoreInput<'a> { pub output: &'a str, } -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct ItemScore { /// chrF of the output against the reference. pub chrf: f64, @@ -131,13 +131,18 @@ pub struct Aggregate { pub validation: SplitMean, } -/// The mean of the scores of the items read in split `want`. -fn split_mean(scored: &[(&CorpusItem, ItemScore)], want: Split) -> SplitMean { - let of: Vec<&ItemScore> = scored +/// The scores of the items read in split `want`, in list order. +fn scores_in<'a>(scored: &'a [(&CorpusItem, ItemScore)], want: Split) -> Vec<&'a ItemScore> { + scored .iter() .filter(|(item, _)| SplitScheme::V1.live(item) == Some(want)) .map(|(_, score)| score) - .collect(); + .collect() +} + +/// The mean of the scores of the items read in split `want`. +fn split_mean(scored: &[(&CorpusItem, ItemScore)], want: Split) -> SplitMean { + let of = scores_in(scored, want); let average = |values: Vec| { (!values.is_empty()).then(|| values.iter().sum::() / values.len() as f64) }; @@ -157,6 +162,12 @@ pub fn aggregate(scored: &[(&CorpusItem, ItemScore)]) -> Aggregate { } } +/// The Validation items' scores in list order, numbers only: no key, no text. +/// A refused item arrives scored 0 and keeps its position. +pub fn validation_items(scored: &[(&CorpusItem, ItemScore)]) -> Vec { + scores_in(scored, Split::Validation).into_iter().cloned().collect() +} + /// The Test split's readings, scalars only: no item key, no minted string, no /// pair. The checkpoint decision (convergence-test) needs numbers; a key beside /// a minted string is the handle an edit step would act on. The only producer @@ -249,6 +260,31 @@ mod tests { item(key, text, &group_in(split), model) } + #[test] + fn validation_items_keeps_validation_only_in_order() { + let train = in_split("item:t:1", UNRELATED, Split::Train, false); + let first = in_split("item:v:1", "first words", Split::Validation, false); + let refused = in_split("item:v:2", "second words", Split::Validation, false); + let last = in_split("item:v:3", "third words", Split::Validation, false); + let mut gone = in_split("item:v:4", "withdrawn words", Split::Validation, false); + gone.withdrawn = true; + let test = in_split("item:c:1", HELD_OUT, Split::Test, false); + let scored = [ + (&train, score(10.0, Some(0.1))), + (&first, score(40.0, Some(0.4))), + (&test, score(60.0, Some(0.6))), + (&refused, score(0.0, Some(0.0))), + (&gone, score(90.0, Some(0.9))), + (&last, score(80.0, None)), + ]; + let got = validation_items(&scored); + assert_eq!( + got, + [score(40.0, Some(0.4)), score(0.0, Some(0.0)), score(80.0, None)] + ); + assert_eq!(got.len(), aggregate(&scored).validation.count); + } + #[test] fn test_readings_at_checkpoint() { let train_item = in_split("item:t:1", UNRELATED, Split::Train, false); diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 2d0320a..635065e 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -2645,6 +2645,9 @@ fn source_tripwires() { "Aggregate", "validation_mean", "validation_refused", + "ItemScore", + "LanguageScores", + "validation_scores", ] { assert!(!context.contains(banned), "context.rs names {banned}"); } @@ -3364,6 +3367,234 @@ fn a_run_from_before_does_not_open() { assert!(reopened.rounds().is_err()); } +/// A round as the file holds it, with key 22 the lib keeps private. +#[derive(Clone, Debug, schemars::JsonSchema, cultcache_rs::DatabaseEntry)] +#[cultcache(type = "weksa.loop_round")] +struct ScoredRound { + #[cultcache(key = 0)] + n: u32, + #[cultcache(key = 1)] + inventory_rev: u32, + #[cultcache(key = 2)] + prompt_digest: String, + #[cultcache(key = 4)] + corpus_digest: String, + #[cultcache(key = 5)] + entity_digest: String, + #[cultcache(key = 6)] + model: String, + #[cultcache(key = 7)] + embedder: String, + #[cultcache(key = 8)] + chrf_version: String, + #[cultcache(key = 9)] + calls: Vec, + #[cultcache(key = 10)] + train: Vec, + #[cultcache(key = 16)] + inventory_digest: String, + #[cultcache(key = 17)] + cycle: Option, + #[cultcache(key = 18)] + admit_rules: Vec, + #[cultcache(key = 19)] + parse_reasons: Vec, + #[cultcache(key = 20)] + own_family: u32, + #[cultcache(key = 21)] + languages: Vec, + #[cultcache(key = 22)] + validation_scores: Vec, +} + +/// A round as written before key 22 existed. +#[derive(Clone, Debug, schemars::JsonSchema, cultcache_rs::DatabaseEntry)] +#[cultcache(type = "weksa.loop_round")] +struct PreScoresRound { + #[cultcache(key = 0)] + n: u32, + #[cultcache(key = 1)] + inventory_rev: u32, + #[cultcache(key = 2)] + prompt_digest: String, + #[cultcache(key = 4)] + corpus_digest: String, + #[cultcache(key = 5)] + entity_digest: String, + #[cultcache(key = 6)] + model: String, + #[cultcache(key = 7)] + embedder: String, + #[cultcache(key = 8)] + chrf_version: String, + #[cultcache(key = 9)] + calls: Vec, + #[cultcache(key = 10)] + train: Vec, + #[cultcache(key = 16)] + inventory_digest: String, + #[cultcache(key = 17)] + cycle: Option, + #[cultcache(key = 18)] + admit_rules: Vec, + #[cultcache(key = 19)] + parse_reasons: Vec, + #[cultcache(key = 20)] + own_family: u32, + #[cultcache(key = 21)] + languages: Vec, +} + +/// The scores a round put in `run.cc`, read through the mirror (the lib's field +/// is private). The run must be dropped first. +fn stored_scores(root: &Path) -> Vec { + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache + .add_generic_backing_store(SingleFileMessagePackBackingStore::new(root.join("run.cc"))) + .unwrap(); + cache.pull_all_backing_stores().unwrap(); + cache.get_all::().unwrap().remove(0).validation_scores +} + +/// The stored scores agree with the round's readings: one entry per language +/// in the record's order, as many scores as Validation items (scored plus +/// refused), the same means, and at least the refused items as exact zeros. +fn assert_scores_match_readings(r: &RoundRecord, stored: &[weksa::run::LanguageScores]) { + let tags: Vec<&str> = r.languages().iter().map(|l| l.language.as_str()).collect(); + let got: Vec<&str> = stored.iter().map(|l| l.language.as_str()).collect(); + assert_eq!(got, tags, "one entry per panel language, in panel order"); + for (reading, entry) in r.languages().iter().zip(stored) { + let m = &reading.validation_mean; + let v = reading.validation_refused; + let refused = (v.parse + v.admit + v.lower) as usize; + assert_eq!(entry.scores.len(), m.count, "{}", reading.language); + assert_eq!(entry.scores.len(), reading.validation_scored as usize + refused); + let n = entry.scores.len() as f64; + let chrf = entry.scores.iter().map(|s| s.chrf).sum::() / n; + assert!((chrf - m.chrf.unwrap()).abs() < 1e-9, "{}", reading.language); + let zeros = entry + .scores + .iter() + .filter(|s| s.chrf == 0.0 && s.meaning == Some(0.0)) + .count(); + assert!(zeros >= refused, "{}: refused items are stored as zeros", reading.language); + } +} + +#[test] +fn a_round_stores_one_score_per_validation_item() { + let mut dao = dao(); + let mut model = fake(&good_answers()); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + let root = dao.root.clone(); + drop(dao.run); + let stored = stored_scores(&root); + assert_eq!(stored.len(), 1); + assert_eq!(stored[0].language, "en-US"); + assert_scores_match_readings(&r, &stored); +} + +#[test] +fn a_two_language_round_stores_each_languages_scores_in_panel_order() { + let mut dao = bilingual(LAOZI_JA); + let mut model = bilingual_fake(); + // The ja-JP Validation item 13 is not an encodable draft: it scores 0. + model.answers.insert("item:daodejing:13".into(), "not a draft".into()); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + let root = dao.root.clone(); + drop(dao.run); + let stored = stored_scores(&root); + assert_scores_match_readings(&r, &stored); + let tags: Vec<&str> = stored.iter().map(|l| l.language.as_str()).collect(); + assert_eq!(tags, ["en-US", "ja-JP"]); + // ja-JP: item 12 reads perfectly, then item 13 (the refused one), in + // stored item order; none of en-US's items is among them. + let ja: Vec<(f64, Option)> = stored[1].scores.iter().map(|s| (s.chrf, s.meaning)).collect(); + assert_eq!(ja, [(100.0, Some(1.0)), (0.0, Some(0.0))]); + assert_eq!(stored[0].scores.len(), 2); + assert!(stored[0].scores.iter().all(|s| s.chrf < 100.0), "en-US holds its own items"); +} + +#[test] +fn refused_validation_items_are_stored_as_zeros_at_every_stage() { + // Lower: the ja-JP items cite an entity with no ja-JP form. + let mut dao = bilingual(None); + let mut model = bilingual_fake(); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + let root = dao.root.clone(); + drop(dao.run); + let stored = stored_scores(&root); + assert_eq!(r.language("ja-JP").unwrap().validation_refused.lower, 2); + assert_scores_match_readings(&r, &stored); + assert!(stored[1].scores.iter().all(|s| s.chrf == 0.0 && s.meaning == Some(0.0))); + + // Parse (item 3 is Validation) and Admit (item 2), one run each. + for (parse, admit) in [(true, false), (false, true)] { + let mut dao = self::dao(); + let mut answers = good_answers(); + if parse { + answers[2].1 = "not json at all".into(); + } + if admit { + answers[1].1 = edited(2, unknown_concept); + } + let mut model = fake(&answers); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + let root = dao.root.clone(); + drop(dao.run); + let stored = stored_scores(&root); + let v = r.language("en-US").unwrap().validation_refused; + assert_eq!((v.parse, v.admit), (u32::from(parse), u32::from(admit))); + assert_scores_match_readings(&r, &stored); + } +} + +#[test] +fn an_old_run_without_scores_still_opens() { + let mut dao = dao(); + let mut model = fake(&good_answers()); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + let rec = dao.run.record().clone(); + let root = dao.root.clone(); + drop(dao.run); + let old = PreScoresRound { + n: r.n(), + inventory_rev: r.inventory_rev(), + prompt_digest: r.prompt_digest().into(), + corpus_digest: r.corpus_digest().into(), + entity_digest: r.entity_digest().into(), + model: r.model().into(), + embedder: r.embedder().into(), + chrf_version: r.chrf_version().into(), + calls: r.calls().to_vec(), + train: r.train().to_vec(), + inventory_digest: r.inventory_digest().into(), + cycle: r.cycle(), + admit_rules: vec![], + parse_reasons: vec![], + own_family: 0, + languages: vec![r.language("en-US").unwrap().clone()], + }; + let _ = std::fs::remove_file(root.join("run.cc")); + { + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache + .add_generic_backing_store(SingleFileMessagePackBackingStore::new(root.join("run.cc"))) + .unwrap(); + cache.pull_all_backing_stores().unwrap(); + cache.put("run:dao".to_string(), &rec).unwrap(); + cache.put("run:dao:round:1".to_string(), &old).unwrap(); + } + let reopened = RunStores::open(&root).unwrap(); + let rounds = reopened.rounds().unwrap(); + assert_eq!(rounds.len(), 1); + assert_eq!(rounds[0].language("en-US"), r.language("en-US")); +} + #[test] fn the_published_run_and_round_schemas_require_none_of_the_new_fields() { let published = weksa::schema::published(); diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index fbf2a50..9766d35 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -40,7 +40,7 @@ means no store holds it and nothing writes it. | Scene (cut `speaker-scene`) | Key `scene:@`, the item's split group; type `weksa.scene`: participants (speaker slugs) and, per ordered pair, relation features of slot `Scene` (relative status, intimacy, in-group, audience and whatever the loop adds), at an inventory revision (provenance; a run reads the features at its own head), and an `Annotator` | As Speaker. Keyed by the group, so a scene never straddles the split. `data/original/speaker-state.cc` holds Model-annotated keeper scenes, one per Train or Validation group of both languages (`weksa scene put|dump`); a Test group has none and reads `NoScene` | As Speaker; a model-annotated scene is never Test evidence | | Corpus item | Key `item::`, `n` canonical decimal >= 1 (no sign, no leading zero, from cut 2b); type `weksa.corpus_item` with text, language, speaker, split group, prior turns, source. Source `ModelWritten` carries a `ModelFamily` enum, the generator string as the fixture wrote it, and a prompt digest (cut 2b). The item carries no speaker state of its own: its speaker slug and group name the Speaker and Scene documents | Admitted once; the whole item is immutable (re-adding an identical item is a no-op, any difference is refused; a correction is a new item that `supersedes` the old); text already held under another key is refused unless the new item supersedes that key (cut 2b); withdrawn, never deleted, so old scores stay explainable | Importers and `weksa corpus add`, under `corpus-cast-plus-public-domain`; the source enum has no variant for Discord or any training-forbidden source (`corpus-clean`) | | Split assignment | Derived: `split(scheme, item.group)` | Fixed for a scheme; a new scheme is a new constant, never a reassignment. A model-written item whose group derives test is in no split; a superseding item keeps its predecessor's group | The hash function. Nobody assigns by hand, so there is no writer to forbid | -| Loop round | Key `run: