From b3821810a58374537adbed8fa1f51e77fa80c8e6 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 00:29:13 +0000 Subject: [PATCH 1/3] test-text-guard: AnswerFiles refuses a prompt that holds the text of a live Test item, before any file is read or written RunStores::answer_files is the one constructor; the five CLI sites use it. Whole lines and JSON strings match at any length, fragments from 24 chars. Zero hits over 168 real measure-ja prompts. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/main.rs | 25 ++++---- crates/weksa/src/model.rs | 30 +++++++++- crates/weksa/src/run.rs | 27 +++++++-- crates/weksa/src/split.rs | 3 +- crates/weksa/src/store.rs | 11 ++++ crates/weksa/tests/golf_loop.rs | 89 +++++++++++++++++++++++------ crates/weksa/tests/ontology_mint.rs | 6 +- 7 files changed, 149 insertions(+), 42 deletions(-) diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 92c7d5e..b76051c 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -19,7 +19,7 @@ use weksa::run::{ }; use weksa::score::ScoreInput; use weksa::{ - Annotator, AnswerFiles, CorpusStore, EntityId, EntityStore, Family, InventoryStore, OllamaEmbedder, OntologyStore, OptionLabel, + Annotator, CorpusStore, EntityId, EntityStore, Family, InventoryStore, OllamaEmbedder, OntologyStore, OptionLabel, PacketStore, PairRelation, Party, Scene, Slot, SpeakerStateStore, ValueKind, Scorer, Split, SplitScheme, StoreError, lower, pronounce, validate, }; @@ -1070,10 +1070,7 @@ fn main() -> Result<()> { Command::Loop(LoopCommand::Round { root, ollama_url }) => { let unreadable = |_| anyhow::anyhow!("a store would not read"); let mut run = RunStores::open(&root)?; - let mut files = AnswerFiles::new( - run.next_round_dir().map_err(unreadable)?, - run.record().model.clone(), - ); + let mut files = run.answer_files(run.next_round_dir().map_err(unreadable)?)?; let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder)); match round(&mut run, &mut files, &scorer)? { RoundOutcome::Unanswered { prompts } => { @@ -1093,10 +1090,11 @@ fn main() -> Result<()> { ollama_url, }) => { let run = RunStores::open(&root)?; - let model = run.record().model.clone(); - let mut recorded = - RecordedAnswers::new(root.join(format!("round-{round}")), model.clone()); - let mut files = AnswerFiles::new(run.ablation_dir(round), model); + let mut recorded = RecordedAnswers::new( + root.join(format!("round-{round}")), + run.record().model.clone(), + ); + let mut files = run.answer_files(run.ablation_dir(round))?; let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder)); match run.stance_ablation(round, &mut recorded, &mut files, &scorer)? { AblationOutcome::Unanswered { prompts } => { @@ -1119,7 +1117,7 @@ fn main() -> Result<()> { } Command::Loop(LoopCommand::Recall { root }) => { let run = RunStores::open(&root)?; - let mut files = AnswerFiles::new(run.recall_dir(), run.record().model.clone()); + let mut files = run.answer_files(run.recall_dir())?; match run.recall(&mut files)? { RecallOutcome::Unanswered { prompts } => { for digest in &prompts { @@ -1133,7 +1131,7 @@ fn main() -> Result<()> { } Command::Loop(LoopCommand::MintOntology { root, language }) => { let run = RunStores::open(&root)?; - let mut files = AnswerFiles::new(run.ontology_mint_dir(), run.record().model.clone()); + let mut files = run.answer_files(run.ontology_mint_dir())?; match run.mint_ontology(&mut files, &language)? { OntologyMintOutcome::Unanswered { prompt } => { println!("unanswered {prompt}"); @@ -1152,10 +1150,7 @@ fn main() -> Result<()> { Command::Loop(LoopCommand::Cycle { root, ollama_url }) => { let unreadable = |_| anyhow::anyhow!("a store would not read"); let mut run = RunStores::open(&root)?; - let mut files = AnswerFiles::new( - run.next_cycle_dir().map_err(unreadable)?, - run.record().model.clone(), - ); + let mut files = run.answer_files(run.next_cycle_dir().map_err(unreadable)?)?; let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder)); match cycle(&mut run, &mut files, &scorer)? { CycleOutcome::Unanswered { prompts } => { diff --git a/crates/weksa/src/model.rs b/crates/weksa/src/model.rs index 1a79449..1b2e37b 100644 --- a/crates/weksa/src/model.rs +++ b/crates/weksa/src/model.rs @@ -20,6 +20,8 @@ pub trait ModelPort { pub enum PortError { /// Nobody has answered yet. Unanswered, + /// The prompt holds the text of a Test item. It is neither written nor answered. + Withheld, Io(ErrorKind), } @@ -27,6 +29,7 @@ impl fmt::Display for PortError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { PortError::Unanswered => f.write_str("the prompt is unanswered"), + PortError::Withheld => f.write_str("the prompt holds Test item text"), PortError::Io(kind) => write!(f, "io error {kind:?}"), } } @@ -40,14 +43,34 @@ impl std::error::Error for PortError {} pub struct AnswerFiles { dir: PathBuf, model: String, + withheld: Vec, } impl AnswerFiles { - pub fn new(dir: PathBuf, model: String) -> Self { - Self { dir, model } + /// Refuses (`PortError::Withheld`) any prompt that holds the text of one of + /// `withheld`, before it reads or writes a file: the one door a prompt + /// leaves the loop through, so a Test sentence cannot reach a model by it. + pub fn new(dir: PathBuf, model: String, withheld: Vec) -> Self { + Self { dir, model, withheld } } } +/// Whether `prompt` holds `text` as a line, as a JSON string, or (at 24 chars +/// or more) anywhere. A short sentence is matched whole, since "Stop." is a +/// substring of any prompt that says "Stop. Then ..."; an item reaches a +/// prompt as a line or a JSON string value. +fn holds(prompt: &str, text: &str) -> bool { + let text = text.trim(); + if text.is_empty() { + return false; + } + if text.chars().count() >= 24 && prompt.contains(text) { + return true; + } + let json = serde_json::to_string(text).unwrap_or_default(); + prompt.lines().any(|l| l.trim() == text) || prompt.contains(&json) +} + /// Answers read from one directory and nothing else: `.answer` is the /// reply to the prompt whose digest is ``. A prompt with no file is /// [`PortError::Unanswered`]; this port never creates or writes anything. The @@ -86,6 +109,9 @@ impl ModelPort for AnswerFiles { fn answer(&mut self, prompt: &Prompt) -> Result { let io = |e: std::io::Error| PortError::Io(e.kind()); + if self.withheld.iter().any(|t| holds(prompt.text(), t)) { + return Err(PortError::Withheld); + } let digest = prompt.digest(); match RecordedAnswers::new(self.dir.clone(), self.model.clone()).answer(prompt) { Err(PortError::Unanswered) => {} diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index d58e2cb..b6a7557 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -44,7 +44,7 @@ use crate::encode::{ EditDraft, LoweringDraft, MAX_EDIT_CHANGES, OntologyDraft, PacketDraft, encoder_id, }; use crate::form::{is_attestation, is_language_tag, is_slug}; -use crate::model::{ModelPort, PortError}; +use crate::model::{AnswerFiles, ModelPort, PortError}; use crate::packet::{Packet, Stance}; use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate}; use crate::split::{Split, SplitScheme, fnv1a64, fnv1a64_of, hex16}; @@ -474,6 +474,8 @@ pub enum RunError { /// The inventory store holds no revision. NoInventory, Io(ErrorKind), + /// A prompt held the text of a Test item and was refused before any file. + Withheld, /// A store or the run file would not read or write. Store, /// The embedder failed. @@ -507,6 +509,7 @@ impl fmt::Display for RunError { RunError::Drift(of) => write!(f, "drift in {of:?}"), RunError::NoInventory => f.write_str("the inventory holds no revision"), RunError::Io(kind) => write!(f, "io error {kind:?}"), + RunError::Withheld => f.write_str("a prompt holds Test item text; nothing was written"), RunError::Store => f.write_str("a store would not read or write"), RunError::Unrecorded => f.write_str( "the round holds no answer to a lowering prompt rebuilt now: the prompt digest drifted or the head inventory moved since the round", @@ -935,6 +938,13 @@ impl RunStores { .collect()) } + /// The answer door for prompts in `dir`: refuses a prompt that holds the text + /// of any live Test item of the corpus, in any language. + pub fn answer_files(&self, dir: PathBuf) -> anyhow::Result { + let withheld = self.corpus.test_texts()?; + Ok(AnswerFiles::new(dir, self.record.model.clone(), withheld)) + } + /// The recorded rounds, in order. pub fn rounds(&self) -> anyhow::Result> { let mut rounds = self.cache.get_all::()?; @@ -1120,7 +1130,8 @@ impl RunStores { } Err(PortError::Unanswered) => unanswered.push(prompt.digest()), Err(PortError::Io(ErrorKind::InvalidData)) => scores.push((language, None)), - Err(PortError::Io(kind)) => return Err(RunError::Io(kind)), + Err(PortError::Withheld) => return Err(RunError::Withheld), + Err(PortError::Io(kind)) => return Err(RunError::Io(kind)), } } if !unanswered.is_empty() { @@ -1208,7 +1219,8 @@ impl RunStores { Err(PortError::Io(ErrorKind::InvalidData)) => { Ok(OntologyMintOutcome::Refused { code: "invalid-data" }) } - Err(PortError::Io(kind)) => Err(RunError::Io(kind)), + Err(PortError::Withheld) => Err(RunError::Withheld), + Err(PortError::Io(kind)) => Err(RunError::Io(kind)), } } @@ -1305,6 +1317,7 @@ fn asked(port: &mut dyn ModelPort, prompt: &Prompt) -> Result { .map_or(Asked::Unusable, |d| Asked::Text(d.text))), Err(PortError::Io(ErrorKind::InvalidData)) => Ok(Asked::Unusable), Err(PortError::Unanswered) => Ok(Asked::Waiting), + Err(PortError::Withheld) => Err(RunError::Withheld), Err(PortError::Io(kind)) => Err(RunError::Io(kind)), } } @@ -1636,7 +1649,8 @@ fn score_round( answers.push((offer.key().to_string(), None)) } Err(PortError::Unanswered) => unanswered.push(prompt.digest()), - Err(PortError::Io(kind)) => return Err(RunError::Io(kind)), + Err(PortError::Withheld) => return Err(RunError::Withheld), + Err(PortError::Io(kind)) => return Err(RunError::Io(kind)), } calls.push(Call { prompt: prompt.digest(), @@ -1730,6 +1744,10 @@ fn score_round( } continue; } + Err(PortError::Withheld) => { + discard_packets(dir); + return Err(RunError::Withheld); + } Err(PortError::Io(kind)) => { discard_packets(dir); return Err(RunError::Io(kind)); @@ -2277,6 +2295,7 @@ pub fn cycle( prompts: vec![prompt.digest()], }); } + Err(PortError::Withheld) => return Err(RunError::Withheld), Err(PortError::Io(kind)) => return Err(RunError::Io(kind)), }; diff --git a/crates/weksa/src/split.rs b/crates/weksa/src/split.rs index 898439d..744e308 100644 --- a/crates/weksa/src/split.rs +++ b/crates/weksa/src/split.rs @@ -5,7 +5,8 @@ //! `EncoderOffer` need a Checkpoint, and only the lib can make one //! (convergence-freeze adds the door). Operator tools (`weksa corpus dump`, //! `weksa score`, `CorpusStore`) read corpus text of any split; that is not -//! gated here. +//! gated here. `AnswerFiles`, the one door a prompt leaves the loop by, refuses +//! any prompt that holds the text of a live Test item (`PortError::Withheld`). use std::collections::BTreeSet; diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index b0b83a9..f746d70 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -12,6 +12,7 @@ use unicode_normalization::{UnicodeNormalization, char::is_combining_mark, is_nf use unicode_segmentation::UnicodeSegmentation; use crate::corpus::{CorpusItem, CorpusItemId, Source, item_key, key_slug}; +use crate::split::{Split, SplitScheme}; use crate::entity::{Entity, EntityOrigin, Pronunciation}; use crate::form::{ is_attestation, is_calendar_date, is_joining_only, is_language_tag, is_refused_format, is_slug, @@ -248,6 +249,16 @@ impl CorpusStore { Ok(items) } + /// The text of every live Test item, any language: what no prompt may hold. + pub fn test_texts(&self) -> anyhow::Result> { + Ok(self + .all()? + .into_iter() + .filter(|item| SplitScheme::V1.live(item) == Some(Split::Test)) + .map(|item| item.text) + .collect()) + } + pub fn get(&self, key: &str) -> anyhow::Result> { self.cache.get::(key) } diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 3ce5e62..dc310a4 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -669,7 +669,7 @@ fn answer_files_write_the_prompt_and_read_the_answer() { let p = prompt(); let dir = tempfile::tempdir().unwrap(); let round = dir.path().join("round-1"); - let mut files = AnswerFiles::new(round.clone(), "answerer".into()); + let mut files = AnswerFiles::new(round.clone(), "answerer".into(), vec![]); assert_eq!(files.model(), "answerer"); // No answer: the prompt is written under its digest, and nothing else. @@ -701,7 +701,7 @@ fn answer_files_write_the_prompt_and_read_the_answer() { // A directory that cannot be made is an io error of its kind. std::fs::write(dir.path().join("file"), "").unwrap(); - let mut blocked = AnswerFiles::new(dir.path().join("file").join("round"), "m".into()); + let mut blocked = AnswerFiles::new(dir.path().join("file").join("round"), "m".into(), vec![]); assert!(matches!(blocked.answer(&p), Err(PortError::Io(_)))); // A reply or prompt file that cannot be read is an io error, never "unanswered". @@ -1113,7 +1113,7 @@ fn unanswered_round_writes_nothing() { let expected = prompts(&dao.run); assert_eq!(expected.len(), 3); - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); let RoundOutcome::Unanswered { prompts: digests } = round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap() else { @@ -1144,7 +1144,7 @@ fn unanswered_round_writes_nothing() { for (key, p) in &expected[..2] { std::fs::write(round1.join(format!("{}.answer", p.digest())), &answers[key]).unwrap(); } - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); let RoundOutcome::Unanswered { prompts: digests } = round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap() else { @@ -1159,7 +1159,7 @@ fn unanswered_round_writes_nothing() { let (key, p) = &expected[2]; std::fs::write(round1.join(format!("{}.answer", p.digest())), &answers[key]).unwrap(); let lowering = lowering_prompts(&dao, dao.run.inventory()); - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); let RoundOutcome::Unanswered { prompts: digests } = round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap() else { @@ -1183,7 +1183,7 @@ fn unanswered_round_writes_nothing() { for (key, p) in &lowering[..2] { std::fs::write(round1.join(format!("{}.answer", p.digest())), lowered(key)).unwrap(); } - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); let RoundOutcome::Unanswered { prompts: digests } = round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap() else { @@ -1195,13 +1195,13 @@ fn unanswered_round_writes_nothing() { assert_eq!(std::fs::read(dao.root.join("run.cc")).unwrap(), run_cc); let (key, p) = &lowering[2]; std::fs::write(round1.join(format!("{}.answer", p.digest())), lowered(key)).unwrap(); - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()); assert_eq!((r.n(), r.calls().len()), (1, 6)); assert_eq!(two_dp(r.language("en-US").unwrap().train_mean.chrf.unwrap()), 44.18); // Round 2 asks again, encode first. - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); assert!(dao.root.join("round-2").parent().is_some()); let RoundOutcome::Unanswered { prompts: again } = round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap() @@ -1224,7 +1224,7 @@ fn unanswered_round_writes_nothing() { ) .unwrap(); } - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); let second = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()); assert_eq!(second.n(), 2); let ns: Vec = dao.run.rounds().unwrap().iter().map(|r| r.n()).collect(); @@ -1411,7 +1411,7 @@ fn non_utf8_encode_answer_is_a_parse_refusal() { let answer = serde_json::json!({ "text": table[key] }).to_string(); std::fs::write(round1.join(format!("{}.answer", p.digest())), answer).unwrap(); } - let mut files = AnswerFiles::new(round1.clone(), "fake-model".into()); + let mut files = AnswerFiles::new(round1.clone(), "fake-model".into(), vec![]); let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()); assert_eq!(r.calls().len(), 5); // Item 3 is Validation: its refusal is a Parse refusal and the round stands. @@ -1621,7 +1621,7 @@ fn errors_echo_no_input() { // A port error, from a canary directory. let blocker = tmp.path().join("canaryfile"); std::fs::write(&blocker, "").unwrap(); - let mut files = AnswerFiles::new(blocker.join("canarydir"), "m".into()); + let mut files = AnswerFiles::new(blocker.join("canarydir"), "m".into(), vec![]); let e = files.answer(&prompt()).err().unwrap(); note(&e, &e); // A round over canary answers records refusals, not the answers. @@ -2084,7 +2084,7 @@ fn a_rejection_leaves_no_candidate_inventory_from_an_earlier_attempt() { let scorer = Scorer::new(Bag); let ask = |dao: &mut Dao| { let mut files = - AnswerFiles::new(dao.run.next_cycle_dir().unwrap(), "fake-model".into()); + AnswerFiles::new(dao.run.next_cycle_dir().unwrap(), "fake-model".into(), vec![]); cycle(&mut dao.run, &mut files, &scorer).unwrap() }; let dir = dao.root.join("cycle-1"); @@ -2154,7 +2154,7 @@ fn unanswered_cycle_writes_nothing() { let run_cc = std::fs::read(dao.root.join("run.cc")).unwrap(); let scorer = Scorer::new(Bag); let ask = |dao: &mut Dao| { - let mut files = AnswerFiles::new(dao.run.next_cycle_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_cycle_dir().unwrap(), "fake-model".into(), vec![]); cycle(&mut dao.run, &mut files, &scorer).unwrap() }; let dir = dao.root.join("cycle-1"); @@ -3036,7 +3036,7 @@ fn views_with_one_meaning_share_one_unanswered_lowering_prompt() { }; std::fs::write(round1.join(format!("{}.answer", p.digest())), answer).unwrap(); } - let mut files = AnswerFiles::new(round1.clone(), "fake-model".into()); + let mut files = AnswerFiles::new(round1.clone(), "fake-model".into(), vec![]); let RoundOutcome::Unanswered { prompts: asked } = round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap() else { @@ -3190,7 +3190,7 @@ fn parse_reasons_count_per_code_over_both_splits_and_name_unreadable() { let answer = serde_json::json!({ "text": table[key] }).to_string(); std::fs::write(round1.join(format!("{}.answer", p.digest())), answer).unwrap(); } - let mut files = AnswerFiles::new(round1, "fake-model".into()); + let mut files = AnswerFiles::new(round1, "fake-model".into(), vec![]); let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()); assert_eq!(reasons(&r), [("syntax", 1), ("unreadable", 1)]); } @@ -3483,7 +3483,7 @@ fn a_recall_of_the_wrong_words_or_no_answer_reads_low_and_a_missing_answer_waits let r = reading.languages[0].clone(); assert_eq!((r.n, r.answered, r.chrf), (3, 0, 0.0)); // With no answers on disk the files port waits on all three prompts. - let mut files = AnswerFiles::new(dao.run.recall_dir(), "m".into()); + let mut files = AnswerFiles::new(dao.run.recall_dir(), "m".into(), vec![]); let RecallOutcome::Unanswered { prompts } = dao.run.recall(&mut files).unwrap() else { panic!("nobody answered"); }; @@ -4031,7 +4031,7 @@ fn a_stance_ablation_records_nothing_and_writes_nothing_in_its_round() { let before = (snapshot(&round1), snapshot(&dao.root), dao.run.rounds().unwrap(), dao.run.decisions().unwrap()); let go = || { let mut rec = RecordedAnswers::new(round1.clone(), "fake-model".into()); - let mut files = weksa::model::AnswerFiles::new(dao.run.ablation_dir(1), "fake-model".into()); + let mut files = weksa::model::AnswerFiles::new(dao.run.ablation_dir(1), "fake-model".into(), vec![]); // Nothing answers yet: the replay and variant prompts are written. dao.run.stance_ablation(1, &mut rec, &mut files, &Scorer::new(Bag)) }; @@ -5114,3 +5114,58 @@ fn a_run_reads_entries_and_scenes_at_its_own_head() { ); assert!(run.is_ok(), "{:?}", run.err()); } + +// ---- test-text-guard: no prompt file or answer for a prompt holding Test text ---- + +#[test] +fn answer_files_withhold_prompts_that_hold_test_text() { + use weksa::model::{AnswerFiles, ModelPort, PortError}; + let p = prompt(); + let line = p.text().lines().find(|l| l.chars().count() >= 24).unwrap().to_string(); + let short = p.text().lines().find(|l| !l.is_empty() && l.chars().count() < 24).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let round = dir.path().join("round-1"); + let held = |texts: Vec| AnswerFiles::new(round.clone(), "m".into(), texts); + // A whole line, a long fragment of one, and a JSON string value are Test text. + for text in [line.clone(), line[..24].to_string(), format!("{short:?}")] { + let mut files = held(vec![text.trim_matches('"').to_string()]); + assert_eq!(files.answer(&p), Err(PortError::Withheld), "{text}"); + } + assert!(!round.exists(), "a refused prompt creates nothing"); + // A short Test sentence inside other prose is not Test text: "Stop." is not "Stop. Then". + let mut files = held(vec![short[..short.len() - 1].to_string()]); + assert_eq!(files.answer(&p), Err(PortError::Unanswered)); + // The refusal comes before a recorded answer is read. + std::fs::write(round.join(format!("{}.answer", p.digest())), "x").unwrap(); + assert_eq!(held(vec![line]).answer(&p), Err(PortError::Withheld)); +} + +#[test] +fn run_answer_files_withhold_a_live_test_item_text() { + use common::{cast, entities_in, group_in_bucket, item_in}; + use weksa::model::{ModelPort, PortError}; + let mut w = world(); + let entities = entities_in(w.dir.path()); + let text = "Hold this sentence back."; + // The sentence in a Test group, and a Train line that quotes it (the corpus + // door refuses an identical twin, not a longer line that holds it). + let quoting = format!("{text} And then some."); + for (key, line, bucket) in [("item:held:1", text, 90), ("item:held:2", "ing, 10)] { + w.corpus + .add(item_in(key, line, &group_in_bucket(bucket), cast()), &entities) + .unwrap(); + } + let run = run_over(w.dir.path()); + let dir = w.dir.path().join("answers"); + let mut files = run.answer_files(dir.clone()).unwrap(); + let inv = run.inventory().inventory().unwrap(); + let train = run.train().unwrap(); + let encode = |key: &str| { + let offer = run.offer(key).unwrap().unwrap(); + weksa::context::Prompt::encode(&offer, &train, &inv, 4) + }; + assert_eq!(files.answer(&encode("item:nibu:1")), Err(PortError::Unanswered)); + // The quoting line's prompt holds the Test sentence: refused, and nothing is written. + assert_eq!(files.answer(&encode("item:held:2")), Err(PortError::Withheld)); + assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 1, "only the clean prompt"); +} diff --git a/crates/weksa/tests/ontology_mint.rs b/crates/weksa/tests/ontology_mint.rs index 2f150b3..3085d5d 100644 --- a/crates/weksa/tests/ontology_mint.rs +++ b/crates/weksa/tests/ontology_mint.rs @@ -242,7 +242,7 @@ fn mint_ontology_writes_nothing_but_its_prompt_and_answer_files() { let dir = tempfile::tempdir().unwrap(); let run = fixture(dir.path(), &["en-US", "ja-JP"]); let before = digests(dir.path()); - let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into()); + let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into(), vec![]); let OntologyMintOutcome::Unanswered { prompt } = run.mint_ontology(&mut files, "en-US").unwrap() else { @@ -268,7 +268,7 @@ fn mint_ontology_writes_nothing_but_its_prompt_and_answer_files() { fn mint_ontology_unanswered_then_drafted_then_refused() { let dir = tempfile::tempdir().unwrap(); let run = fixture(dir.path(), &["en-US"]); - let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into()); + let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into(), vec![]); let OntologyMintOutcome::Unanswered { prompt } = run.mint_ontology(&mut files, "en-US").unwrap() else { @@ -310,7 +310,7 @@ fn a_mint_answer_admits_into_a_copy_and_a_new_run_pins_the_copy() { let draft = format!( r#"{{"entries":[{{"layer":"Discourse","when":[{{"slot":"Stance","feature":"{STANCE_G}","test":{{"AtLeast":500}}}}],"choice":"a grown choice","examples":[]}}]}}"# ); - let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into()); + let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into(), vec![]); let OntologyMintOutcome::Unanswered { prompt } = run.mint_ontology(&mut files, "en-US").unwrap() else { -- 2.47.3 From f2b1e4fb815524de616de1f55e1750975f32b4d5 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 00:40:26 +0000 Subject: [PATCH 2/3] test-text-guard s1 fixes: corpus-door sameness for texts of 12+ normalised chars, short texts as a line's end, every Test-group item withheld, round/cycle path pinned, split.rs doc dropped so PROMPT_SOURCE_DIGEST is unchanged Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/model.rs | 30 +++++++++----- crates/weksa/src/run.rs | 2 +- crates/weksa/src/split.rs | 3 +- crates/weksa/src/store.rs | 5 ++- crates/weksa/tests/golf_loop.rs | 71 +++++++++++++++++++++++---------- 5 files changed, 74 insertions(+), 37 deletions(-) diff --git a/crates/weksa/src/model.rs b/crates/weksa/src/model.rs index 1b2e37b..415c0d5 100644 --- a/crates/weksa/src/model.rs +++ b/crates/weksa/src/model.rs @@ -7,6 +7,7 @@ use std::io::ErrorKind; use std::path::PathBuf; use crate::context::Prompt; +use crate::store::normalized; /// A model answering [`Prompt`]s and nothing else. pub trait ModelPort { @@ -20,7 +21,7 @@ pub trait ModelPort { pub enum PortError { /// Nobody has answered yet. Unanswered, - /// The prompt holds the text of a Test item. It is neither written nor answered. + /// The prompt holds the text of an item in a Test group. It is neither written nor answered. Withheld, Io(ErrorKind), } @@ -55,20 +56,29 @@ impl AnswerFiles { } } -/// Whether `prompt` holds `text` as a line, as a JSON string, or (at 24 chars -/// or more) anywhere. A short sentence is matched whole, since "Stop." is a -/// substring of any prompt that says "Stop. Then ..."; an item reaches a -/// prompt as a line or a JSON string value. +/// Normalised length from which a Test text is matched anywhere in a prompt. +const LONG: usize = 12; + +/// Whether `prompt` holds `text`. The corpus door's own sameness +/// ([`normalized`]: NFKC, case fold, spacing and punctuation dropped) decides +/// for a text of `LONG` normalised characters or more, so a re-cased, +/// re-spaced, wrapped or width-changed copy still counts. A shorter text would +/// match anything ("Stop." is in "Stop. Then ..."), so it counts only as a +/// line's end after a non-letter (every renderer prints an item's text last on its line). fn holds(prompt: &str, text: &str) -> bool { let text = text.trim(); - if text.is_empty() { + let key = normalized(text); + if key.is_empty() { return false; } - if text.chars().count() >= 24 && prompt.contains(text) { - return true; + if key.chars().count() >= LONG { + return normalized(prompt).contains(&key); } - let json = serde_json::to_string(text).unwrap_or_default(); - prompt.lines().any(|l| l.trim() == text) || prompt.contains(&json) + prompt.lines().any(|l| { + l.trim_end().strip_suffix(text).is_some_and(|head| { + head.chars().next_back().is_none_or(|c| !c.is_alphanumeric()) + }) + }) } /// Answers read from one directory and nothing else: `.answer` is the diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index b6a7557..9b795f4 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -509,7 +509,7 @@ impl fmt::Display for RunError { RunError::Drift(of) => write!(f, "drift in {of:?}"), RunError::NoInventory => f.write_str("the inventory holds no revision"), RunError::Io(kind) => write!(f, "io error {kind:?}"), - RunError::Withheld => f.write_str("a prompt holds Test item text; nothing was written"), + RunError::Withheld => f.write_str("a prompt holds Test item text; it was neither written nor answered"), RunError::Store => f.write_str("a store would not read or write"), RunError::Unrecorded => f.write_str( "the round holds no answer to a lowering prompt rebuilt now: the prompt digest drifted or the head inventory moved since the round", diff --git a/crates/weksa/src/split.rs b/crates/weksa/src/split.rs index 744e308..898439d 100644 --- a/crates/weksa/src/split.rs +++ b/crates/weksa/src/split.rs @@ -5,8 +5,7 @@ //! `EncoderOffer` need a Checkpoint, and only the lib can make one //! (convergence-freeze adds the door). Operator tools (`weksa corpus dump`, //! `weksa score`, `CorpusStore`) read corpus text of any split; that is not -//! gated here. `AnswerFiles`, the one door a prompt leaves the loop by, refuses -//! any prompt that holds the text of a live Test item (`PortError::Withheld`). +//! gated here. use std::collections::BTreeSet; diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index f746d70..56fcc5b 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -249,12 +249,13 @@ impl CorpusStore { Ok(items) } - /// The text of every live Test item, any language: what no prompt may hold. + /// The text of every item in a Test group, any language, withdrawn and + /// model-written ones included: what no prompt may hold. pub fn test_texts(&self) -> anyhow::Result> { Ok(self .all()? .into_iter() - .filter(|item| SplitScheme::V1.live(item) == Some(Split::Test)) + .filter(|item| SplitScheme::V1.of_bucket(SplitScheme::V1.bucket(&item.group)) == Split::Test) .map(|item| item.text) .collect()) } diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index dc310a4..664a52a 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -2596,6 +2596,8 @@ fn source_tripwires() { assert!(!text.contains(banned), "{file} names {banned}"); } } + // Every prompt file leaves through RunStores::answer_files, which withholds Test text. + assert!(!read("main.rs").contains("AnswerFiles::new")); let mut all = Vec::new(); let mut stack = vec![src.clone()]; while let Some(dir) = stack.pop() { @@ -5121,41 +5123,58 @@ fn a_run_reads_entries_and_scenes_at_its_own_head() { fn answer_files_withhold_prompts_that_hold_test_text() { use weksa::model::{AnswerFiles, ModelPort, PortError}; let p = prompt(); - let line = p.text().lines().find(|l| l.chars().count() >= 24).unwrap().to_string(); - let short = p.text().lines().find(|l| !l.is_empty() && l.chars().count() < 24).unwrap(); let dir = tempfile::tempdir().unwrap(); let round = dir.path().join("round-1"); - let held = |texts: Vec| AnswerFiles::new(round.clone(), "m".into(), texts); - // A whole line, a long fragment of one, and a JSON string value are Test text. - for text in [line.clone(), line[..24].to_string(), format!("{short:?}")] { - let mut files = held(vec![text.trim_matches('"').to_string()]); - assert_eq!(files.answer(&p), Err(PortError::Withheld), "{text}"); - } + let refused = |t: &str| { + AnswerFiles::new(round.clone(), "m".into(), vec![t.to_string()]).answer(&p) + == Err(PortError::Withheld) + }; + // A text of 12 normalised characters or more counts anywhere in the prompt, in + // any case, spacing, wrapping or width: the corpus door's own sameness. + assert!(refused("Nibu says mind the hatch.")); + assert!(refused("NIBU SAYS MIND THE HATCH")); + assert!(refused("nibu says\n mind the\n\n hatch")); + assert!(refused("\u{ff2e}ibu says mind the hatch.")); assert!(!round.exists(), "a refused prompt creates nothing"); - // A short Test sentence inside other prose is not Test text: "Stop." is not "Stop. Then". - let mut files = held(vec![short[..short.len() - 1].to_string()]); - assert_eq!(files.answer(&p), Err(PortError::Unanswered)); + // A short text counts only as the end of a line after a non-letter. + assert!(refused("the hatch.")); + assert!(!refused("hatch"), "a mid-word short text is not the line's end"); + assert!(!refused("mind the")); + assert!(!refused(" "), "a blank withheld text matches nothing"); + // Unrelated text passes and the prompt is then written for its answerer. + let mut clean = AnswerFiles::new(round.clone(), "m".into(), vec!["The ferry leaves at noon.".into()]); + assert_eq!(clean.answer(&p), Err(PortError::Unanswered)); + assert!(round.join(format!("{}.prompt", p.digest())).exists()); // The refusal comes before a recorded answer is read. std::fs::write(round.join(format!("{}.answer", p.digest())), "x").unwrap(); - assert_eq!(held(vec![line]).answer(&p), Err(PortError::Withheld)); + assert!(refused("Nibu says mind the hatch.")); } #[test] -fn run_answer_files_withhold_a_live_test_item_text() { +fn a_round_stops_on_a_prompt_that_holds_test_text() { use common::{cast, entities_in, group_in_bucket, item_in}; use weksa::model::{ModelPort, PortError}; + use weksa::run::{RunError, round}; let mut w = world(); let entities = entities_in(w.dir.path()); - let text = "Hold this sentence back."; - // The sentence in a Test group, and a Train line that quotes it (the corpus - // door refuses an identical twin, not a longer line that holds it). - let quoting = format!("{text} And then some."); - for (key, line, bucket) in [("item:held:1", text, 90), ("item:held:2", "ing, 10)] { + // Test sentences (one withdrawn), and Train lines that quote them: the corpus + // door refuses an identical twin, not a longer line that holds one. + let held = "Hold this sentence back."; + let gone = "A withdrawn Test sentence."; + w.corpus + .add(item_in("item:held:3", gone, &group_in_bucket(91), cast()), &entities) + .unwrap(); + w.corpus.withdraw("item:held:3").unwrap(); + for (key, line, bucket) in [ + ("item:held:1", held.to_string(), 90), + ("item:held:2", format!("{held} And then some."), 10), + ("item:held:4", format!("{gone} And then some."), 11), + ] { w.corpus - .add(item_in(key, line, &group_in_bucket(bucket), cast()), &entities) + .add(item_in(key, &line, &group_in_bucket(bucket), cast()), &entities) .unwrap(); } - let run = run_over(w.dir.path()); + let mut run = run_over(w.dir.path()); let dir = w.dir.path().join("answers"); let mut files = run.answer_files(dir.clone()).unwrap(); let inv = run.inventory().inventory().unwrap(); @@ -5165,7 +5184,15 @@ fn run_answer_files_withhold_a_live_test_item_text() { weksa::context::Prompt::encode(&offer, &train, &inv, 4) }; assert_eq!(files.answer(&encode("item:nibu:1")), Err(PortError::Unanswered)); - // The quoting line's prompt holds the Test sentence: refused, and nothing is written. - assert_eq!(files.answer(&encode("item:held:2")), Err(PortError::Withheld)); + for key in ["item:held:2", "item:held:4"] { + assert_eq!(files.answer(&encode(key)), Err(PortError::Withheld), "{key}"); + } assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 1, "only the clean prompt"); + // The round stops on it rather than skipping the item. + let scorer = weksa::score::Scorer::new(OneBag); + let mut round_files = run.answer_files(w.dir.path().join("round")).unwrap(); + assert!(matches!( + round(&mut run, &mut round_files, &scorer), + Err(RunError::Withheld) + )); } -- 2.47.3 From 8e50bac366742ebeecc6f09ec958c12f56dab56c Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 00:57:21 +0000 Subject: [PATCH 3/3] test-text-guard s2 fixes: drop the short-text path, pin the 12-char threshold, round and mint-ontology stop on Withheld, model-written Test-group text withheld Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/model.rs | 26 +++++----------- crates/weksa/src/run.rs | 2 +- crates/weksa/tests/golf_loop.rs | 53 +++++++++++++++++++++++++++++---- 3 files changed, 56 insertions(+), 25 deletions(-) diff --git a/crates/weksa/src/model.rs b/crates/weksa/src/model.rs index 415c0d5..a9af4fa 100644 --- a/crates/weksa/src/model.rs +++ b/crates/weksa/src/model.rs @@ -56,29 +56,17 @@ impl AnswerFiles { } } -/// Normalised length from which a Test text is matched anywhere in a prompt. +/// Normalised length from which a Test text is matched. Every live Test text is +/// longer; a shorter one ("Stop.") would match any prompt that says "Stop. Then +/// ...", so a Test item that short is not caught here. const LONG: usize = 12; -/// Whether `prompt` holds `text`. The corpus door's own sameness -/// ([`normalized`]: NFKC, case fold, spacing and punctuation dropped) decides -/// for a text of `LONG` normalised characters or more, so a re-cased, -/// re-spaced, wrapped or width-changed copy still counts. A shorter text would -/// match anything ("Stop." is in "Stop. Then ..."), so it counts only as a -/// line's end after a non-letter (every renderer prints an item's text last on its line). +/// Whether `prompt` holds `text`, by the corpus door's own sameness +/// ([`normalized`]: NFKC, case fold, spacing and punctuation dropped): a +/// re-cased, re-spaced, wrapped or width-changed copy still counts. fn holds(prompt: &str, text: &str) -> bool { - let text = text.trim(); let key = normalized(text); - if key.is_empty() { - return false; - } - if key.chars().count() >= LONG { - return normalized(prompt).contains(&key); - } - prompt.lines().any(|l| { - l.trim_end().strip_suffix(text).is_some_and(|head| { - head.chars().next_back().is_none_or(|c| !c.is_alphanumeric()) - }) - }) + key.chars().count() >= LONG && normalized(prompt).contains(&key) } /// Answers read from one directory and nothing else: `.answer` is the diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 9b795f4..d4b3f9f 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -939,7 +939,7 @@ impl RunStores { } /// The answer door for prompts in `dir`: refuses a prompt that holds the text - /// of any live Test item of the corpus, in any language. + /// of any item in a Test group of the corpus, in any language. pub fn answer_files(&self, dir: PathBuf) -> anyhow::Result { let withheld = self.corpus.test_texts()?; Ok(AnswerFiles::new(dir, self.record.model.clone(), withheld)) diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 664a52a..f754f95 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -5136,10 +5136,10 @@ fn answer_files_withhold_prompts_that_hold_test_text() { assert!(refused("nibu says\n mind the\n\n hatch")); assert!(refused("\u{ff2e}ibu says mind the hatch.")); assert!(!round.exists(), "a refused prompt creates nothing"); - // A short text counts only as the end of a line after a non-letter. - assert!(refused("the hatch.")); - assert!(!refused("hatch"), "a mid-word short text is not the line's end"); - assert!(!refused("mind the")); + // The threshold is 12 normalised characters: "the hatch" (8) says nothing about a + // prompt, "mind the hatch" (12) does. + assert!(!refused("the hatch")); + assert!(refused("mind the hatch")); assert!(!refused(" "), "a blank withheld text matches nothing"); // Unrelated text passes and the prompt is then written for its answerer. let mut clean = AnswerFiles::new(round.clone(), "m".into(), vec!["The ferry leaves at noon.".into()]); @@ -5188,7 +5188,9 @@ fn a_round_stops_on_a_prompt_that_holds_test_text() { assert_eq!(files.answer(&encode(key)), Err(PortError::Withheld), "{key}"); } assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 1, "only the clean prompt"); - // The round stops on it rather than skipping the item. + // The round stops on it rather than skipping the item. A model-written line in a + // Test group is withheld too: it is in no split, and its text is still held out. + // (Covered by test_texts taking every Test-group item, below.) let scorer = weksa::score::Scorer::new(OneBag); let mut round_files = run.answer_files(w.dir.path().join("round")).unwrap(); assert!(matches!( @@ -5196,3 +5198,44 @@ fn a_round_stops_on_a_prompt_that_holds_test_text() { Err(RunError::Withheld) )); } + +/// Refuses every prompt as holding Test text, as the answer door does. +struct Refuses; + +impl weksa::model::ModelPort for Refuses { + fn model(&self) -> &str { + "fake-model" + } + fn answer(&mut self, _: &weksa::context::Prompt) -> Result { + Err(weksa::model::PortError::Withheld) + } +} + +#[test] +fn every_model_call_stops_on_a_withheld_prompt() { + use weksa::run::{RunError, round}; + let w = world(); + let mut run = run_over(w.dir.path()); + let scorer = weksa::score::Scorer::new(OneBag); + let r = round(&mut run, &mut Refuses, &scorer); + assert!(matches!(r, Err(RunError::Withheld)), "{:?}", r.map(|_| ())); + assert!(matches!(run.mint_ontology(&mut Refuses, "en-US"), Err(RunError::Withheld))); +} + +#[test] +fn model_written_test_group_text_is_withheld_too() { + use common::{entities_in, group_in_bucket, item_in, model}; + let mut w = world(); + let entities = entities_in(w.dir.path()); + w.corpus + .add(item_in("item:held:5", "A model line of a Test meaning.", &group_in_bucket(92), model()), &entities) + .unwrap(); + assert!( + w.corpus + .test_texts() + .unwrap() + .iter() + .any(|t| t == "A model line of a Test meaning."), + "a model-written Test-group item is in no split, and its text is still held out" + ); +} -- 2.47.3