diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 92c7d5e..b76051c 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -19,7 +19,7 @@ use weksa::run::{ }; use weksa::score::ScoreInput; use weksa::{ - Annotator, AnswerFiles, CorpusStore, EntityId, EntityStore, Family, InventoryStore, OllamaEmbedder, OntologyStore, OptionLabel, + Annotator, CorpusStore, EntityId, EntityStore, Family, InventoryStore, OllamaEmbedder, OntologyStore, OptionLabel, PacketStore, PairRelation, Party, Scene, Slot, SpeakerStateStore, ValueKind, Scorer, Split, SplitScheme, StoreError, lower, pronounce, validate, }; @@ -1070,10 +1070,7 @@ fn main() -> Result<()> { Command::Loop(LoopCommand::Round { root, ollama_url }) => { let unreadable = |_| anyhow::anyhow!("a store would not read"); let mut run = RunStores::open(&root)?; - let mut files = AnswerFiles::new( - run.next_round_dir().map_err(unreadable)?, - run.record().model.clone(), - ); + let mut files = run.answer_files(run.next_round_dir().map_err(unreadable)?)?; let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder)); match round(&mut run, &mut files, &scorer)? { RoundOutcome::Unanswered { prompts } => { @@ -1093,10 +1090,11 @@ fn main() -> Result<()> { ollama_url, }) => { let run = RunStores::open(&root)?; - let model = run.record().model.clone(); - let mut recorded = - RecordedAnswers::new(root.join(format!("round-{round}")), model.clone()); - let mut files = AnswerFiles::new(run.ablation_dir(round), model); + let mut recorded = RecordedAnswers::new( + root.join(format!("round-{round}")), + run.record().model.clone(), + ); + let mut files = run.answer_files(run.ablation_dir(round))?; let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder)); match run.stance_ablation(round, &mut recorded, &mut files, &scorer)? { AblationOutcome::Unanswered { prompts } => { @@ -1119,7 +1117,7 @@ fn main() -> Result<()> { } Command::Loop(LoopCommand::Recall { root }) => { let run = RunStores::open(&root)?; - let mut files = AnswerFiles::new(run.recall_dir(), run.record().model.clone()); + let mut files = run.answer_files(run.recall_dir())?; match run.recall(&mut files)? { RecallOutcome::Unanswered { prompts } => { for digest in &prompts { @@ -1133,7 +1131,7 @@ fn main() -> Result<()> { } Command::Loop(LoopCommand::MintOntology { root, language }) => { let run = RunStores::open(&root)?; - let mut files = AnswerFiles::new(run.ontology_mint_dir(), run.record().model.clone()); + let mut files = run.answer_files(run.ontology_mint_dir())?; match run.mint_ontology(&mut files, &language)? { OntologyMintOutcome::Unanswered { prompt } => { println!("unanswered {prompt}"); @@ -1152,10 +1150,7 @@ fn main() -> Result<()> { Command::Loop(LoopCommand::Cycle { root, ollama_url }) => { let unreadable = |_| anyhow::anyhow!("a store would not read"); let mut run = RunStores::open(&root)?; - let mut files = AnswerFiles::new( - run.next_cycle_dir().map_err(unreadable)?, - run.record().model.clone(), - ); + let mut files = run.answer_files(run.next_cycle_dir().map_err(unreadable)?)?; let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder)); match cycle(&mut run, &mut files, &scorer)? { CycleOutcome::Unanswered { prompts } => { diff --git a/crates/weksa/src/model.rs b/crates/weksa/src/model.rs index 1a79449..a9af4fa 100644 --- a/crates/weksa/src/model.rs +++ b/crates/weksa/src/model.rs @@ -7,6 +7,7 @@ use std::io::ErrorKind; use std::path::PathBuf; use crate::context::Prompt; +use crate::store::normalized; /// A model answering [`Prompt`]s and nothing else. pub trait ModelPort { @@ -20,6 +21,8 @@ pub trait ModelPort { pub enum PortError { /// Nobody has answered yet. Unanswered, + /// The prompt holds the text of an item in a Test group. It is neither written nor answered. + Withheld, Io(ErrorKind), } @@ -27,6 +30,7 @@ impl fmt::Display for PortError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { PortError::Unanswered => f.write_str("the prompt is unanswered"), + PortError::Withheld => f.write_str("the prompt holds Test item text"), PortError::Io(kind) => write!(f, "io error {kind:?}"), } } @@ -40,14 +44,31 @@ impl std::error::Error for PortError {} pub struct AnswerFiles { dir: PathBuf, model: String, + withheld: Vec, } impl AnswerFiles { - pub fn new(dir: PathBuf, model: String) -> Self { - Self { dir, model } + /// Refuses (`PortError::Withheld`) any prompt that holds the text of one of + /// `withheld`, before it reads or writes a file: the one door a prompt + /// leaves the loop through, so a Test sentence cannot reach a model by it. + pub fn new(dir: PathBuf, model: String, withheld: Vec) -> Self { + Self { dir, model, withheld } } } +/// Normalised length from which a Test text is matched. Every live Test text is +/// longer; a shorter one ("Stop.") would match any prompt that says "Stop. Then +/// ...", so a Test item that short is not caught here. +const LONG: usize = 12; + +/// Whether `prompt` holds `text`, by the corpus door's own sameness +/// ([`normalized`]: NFKC, case fold, spacing and punctuation dropped): a +/// re-cased, re-spaced, wrapped or width-changed copy still counts. +fn holds(prompt: &str, text: &str) -> bool { + let key = normalized(text); + key.chars().count() >= LONG && normalized(prompt).contains(&key) +} + /// Answers read from one directory and nothing else: `.answer` is the /// reply to the prompt whose digest is ``. A prompt with no file is /// [`PortError::Unanswered`]; this port never creates or writes anything. The @@ -86,6 +107,9 @@ impl ModelPort for AnswerFiles { fn answer(&mut self, prompt: &Prompt) -> Result { let io = |e: std::io::Error| PortError::Io(e.kind()); + if self.withheld.iter().any(|t| holds(prompt.text(), t)) { + return Err(PortError::Withheld); + } let digest = prompt.digest(); match RecordedAnswers::new(self.dir.clone(), self.model.clone()).answer(prompt) { Err(PortError::Unanswered) => {} diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index d58e2cb..d4b3f9f 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -44,7 +44,7 @@ use crate::encode::{ EditDraft, LoweringDraft, MAX_EDIT_CHANGES, OntologyDraft, PacketDraft, encoder_id, }; use crate::form::{is_attestation, is_language_tag, is_slug}; -use crate::model::{ModelPort, PortError}; +use crate::model::{AnswerFiles, ModelPort, PortError}; use crate::packet::{Packet, Stance}; use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate}; use crate::split::{Split, SplitScheme, fnv1a64, fnv1a64_of, hex16}; @@ -474,6 +474,8 @@ pub enum RunError { /// The inventory store holds no revision. NoInventory, Io(ErrorKind), + /// A prompt held the text of a Test item and was refused before any file. + Withheld, /// A store or the run file would not read or write. Store, /// The embedder failed. @@ -507,6 +509,7 @@ impl fmt::Display for RunError { RunError::Drift(of) => write!(f, "drift in {of:?}"), RunError::NoInventory => f.write_str("the inventory holds no revision"), RunError::Io(kind) => write!(f, "io error {kind:?}"), + RunError::Withheld => f.write_str("a prompt holds Test item text; it was neither written nor answered"), RunError::Store => f.write_str("a store would not read or write"), RunError::Unrecorded => f.write_str( "the round holds no answer to a lowering prompt rebuilt now: the prompt digest drifted or the head inventory moved since the round", @@ -935,6 +938,13 @@ impl RunStores { .collect()) } + /// The answer door for prompts in `dir`: refuses a prompt that holds the text + /// of any item in a Test group of the corpus, in any language. + pub fn answer_files(&self, dir: PathBuf) -> anyhow::Result { + let withheld = self.corpus.test_texts()?; + Ok(AnswerFiles::new(dir, self.record.model.clone(), withheld)) + } + /// The recorded rounds, in order. pub fn rounds(&self) -> anyhow::Result> { let mut rounds = self.cache.get_all::()?; @@ -1120,7 +1130,8 @@ impl RunStores { } Err(PortError::Unanswered) => unanswered.push(prompt.digest()), Err(PortError::Io(ErrorKind::InvalidData)) => scores.push((language, None)), - Err(PortError::Io(kind)) => return Err(RunError::Io(kind)), + Err(PortError::Withheld) => return Err(RunError::Withheld), + Err(PortError::Io(kind)) => return Err(RunError::Io(kind)), } } if !unanswered.is_empty() { @@ -1208,7 +1219,8 @@ impl RunStores { Err(PortError::Io(ErrorKind::InvalidData)) => { Ok(OntologyMintOutcome::Refused { code: "invalid-data" }) } - Err(PortError::Io(kind)) => Err(RunError::Io(kind)), + Err(PortError::Withheld) => Err(RunError::Withheld), + Err(PortError::Io(kind)) => Err(RunError::Io(kind)), } } @@ -1305,6 +1317,7 @@ fn asked(port: &mut dyn ModelPort, prompt: &Prompt) -> Result { .map_or(Asked::Unusable, |d| Asked::Text(d.text))), Err(PortError::Io(ErrorKind::InvalidData)) => Ok(Asked::Unusable), Err(PortError::Unanswered) => Ok(Asked::Waiting), + Err(PortError::Withheld) => Err(RunError::Withheld), Err(PortError::Io(kind)) => Err(RunError::Io(kind)), } } @@ -1636,7 +1649,8 @@ fn score_round( answers.push((offer.key().to_string(), None)) } Err(PortError::Unanswered) => unanswered.push(prompt.digest()), - Err(PortError::Io(kind)) => return Err(RunError::Io(kind)), + Err(PortError::Withheld) => return Err(RunError::Withheld), + Err(PortError::Io(kind)) => return Err(RunError::Io(kind)), } calls.push(Call { prompt: prompt.digest(), @@ -1730,6 +1744,10 @@ fn score_round( } continue; } + Err(PortError::Withheld) => { + discard_packets(dir); + return Err(RunError::Withheld); + } Err(PortError::Io(kind)) => { discard_packets(dir); return Err(RunError::Io(kind)); @@ -2277,6 +2295,7 @@ pub fn cycle( prompts: vec![prompt.digest()], }); } + Err(PortError::Withheld) => return Err(RunError::Withheld), Err(PortError::Io(kind)) => return Err(RunError::Io(kind)), }; diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index b0b83a9..56fcc5b 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -12,6 +12,7 @@ use unicode_normalization::{UnicodeNormalization, char::is_combining_mark, is_nf use unicode_segmentation::UnicodeSegmentation; use crate::corpus::{CorpusItem, CorpusItemId, Source, item_key, key_slug}; +use crate::split::{Split, SplitScheme}; use crate::entity::{Entity, EntityOrigin, Pronunciation}; use crate::form::{ is_attestation, is_calendar_date, is_joining_only, is_language_tag, is_refused_format, is_slug, @@ -248,6 +249,17 @@ impl CorpusStore { Ok(items) } + /// The text of every item in a Test group, any language, withdrawn and + /// model-written ones included: what no prompt may hold. + pub fn test_texts(&self) -> anyhow::Result> { + Ok(self + .all()? + .into_iter() + .filter(|item| SplitScheme::V1.of_bucket(SplitScheme::V1.bucket(&item.group)) == Split::Test) + .map(|item| item.text) + .collect()) + } + pub fn get(&self, key: &str) -> anyhow::Result> { self.cache.get::(key) } diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 3ce5e62..f754f95 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -669,7 +669,7 @@ fn answer_files_write_the_prompt_and_read_the_answer() { let p = prompt(); let dir = tempfile::tempdir().unwrap(); let round = dir.path().join("round-1"); - let mut files = AnswerFiles::new(round.clone(), "answerer".into()); + let mut files = AnswerFiles::new(round.clone(), "answerer".into(), vec![]); assert_eq!(files.model(), "answerer"); // No answer: the prompt is written under its digest, and nothing else. @@ -701,7 +701,7 @@ fn answer_files_write_the_prompt_and_read_the_answer() { // A directory that cannot be made is an io error of its kind. std::fs::write(dir.path().join("file"), "").unwrap(); - let mut blocked = AnswerFiles::new(dir.path().join("file").join("round"), "m".into()); + let mut blocked = AnswerFiles::new(dir.path().join("file").join("round"), "m".into(), vec![]); assert!(matches!(blocked.answer(&p), Err(PortError::Io(_)))); // A reply or prompt file that cannot be read is an io error, never "unanswered". @@ -1113,7 +1113,7 @@ fn unanswered_round_writes_nothing() { let expected = prompts(&dao.run); assert_eq!(expected.len(), 3); - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); let RoundOutcome::Unanswered { prompts: digests } = round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap() else { @@ -1144,7 +1144,7 @@ fn unanswered_round_writes_nothing() { for (key, p) in &expected[..2] { std::fs::write(round1.join(format!("{}.answer", p.digest())), &answers[key]).unwrap(); } - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); let RoundOutcome::Unanswered { prompts: digests } = round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap() else { @@ -1159,7 +1159,7 @@ fn unanswered_round_writes_nothing() { let (key, p) = &expected[2]; std::fs::write(round1.join(format!("{}.answer", p.digest())), &answers[key]).unwrap(); let lowering = lowering_prompts(&dao, dao.run.inventory()); - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); let RoundOutcome::Unanswered { prompts: digests } = round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap() else { @@ -1183,7 +1183,7 @@ fn unanswered_round_writes_nothing() { for (key, p) in &lowering[..2] { std::fs::write(round1.join(format!("{}.answer", p.digest())), lowered(key)).unwrap(); } - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); let RoundOutcome::Unanswered { prompts: digests } = round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap() else { @@ -1195,13 +1195,13 @@ fn unanswered_round_writes_nothing() { assert_eq!(std::fs::read(dao.root.join("run.cc")).unwrap(), run_cc); let (key, p) = &lowering[2]; std::fs::write(round1.join(format!("{}.answer", p.digest())), lowered(key)).unwrap(); - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()); assert_eq!((r.n(), r.calls().len()), (1, 6)); assert_eq!(two_dp(r.language("en-US").unwrap().train_mean.chrf.unwrap()), 44.18); // Round 2 asks again, encode first. - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); assert!(dao.root.join("round-2").parent().is_some()); let RoundOutcome::Unanswered { prompts: again } = round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap() @@ -1224,7 +1224,7 @@ fn unanswered_round_writes_nothing() { ) .unwrap(); } - let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]); let second = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()); assert_eq!(second.n(), 2); let ns: Vec = dao.run.rounds().unwrap().iter().map(|r| r.n()).collect(); @@ -1411,7 +1411,7 @@ fn non_utf8_encode_answer_is_a_parse_refusal() { let answer = serde_json::json!({ "text": table[key] }).to_string(); std::fs::write(round1.join(format!("{}.answer", p.digest())), answer).unwrap(); } - let mut files = AnswerFiles::new(round1.clone(), "fake-model".into()); + let mut files = AnswerFiles::new(round1.clone(), "fake-model".into(), vec![]); let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()); assert_eq!(r.calls().len(), 5); // Item 3 is Validation: its refusal is a Parse refusal and the round stands. @@ -1621,7 +1621,7 @@ fn errors_echo_no_input() { // A port error, from a canary directory. let blocker = tmp.path().join("canaryfile"); std::fs::write(&blocker, "").unwrap(); - let mut files = AnswerFiles::new(blocker.join("canarydir"), "m".into()); + let mut files = AnswerFiles::new(blocker.join("canarydir"), "m".into(), vec![]); let e = files.answer(&prompt()).err().unwrap(); note(&e, &e); // A round over canary answers records refusals, not the answers. @@ -2084,7 +2084,7 @@ fn a_rejection_leaves_no_candidate_inventory_from_an_earlier_attempt() { let scorer = Scorer::new(Bag); let ask = |dao: &mut Dao| { let mut files = - AnswerFiles::new(dao.run.next_cycle_dir().unwrap(), "fake-model".into()); + AnswerFiles::new(dao.run.next_cycle_dir().unwrap(), "fake-model".into(), vec![]); cycle(&mut dao.run, &mut files, &scorer).unwrap() }; let dir = dao.root.join("cycle-1"); @@ -2154,7 +2154,7 @@ fn unanswered_cycle_writes_nothing() { let run_cc = std::fs::read(dao.root.join("run.cc")).unwrap(); let scorer = Scorer::new(Bag); let ask = |dao: &mut Dao| { - let mut files = AnswerFiles::new(dao.run.next_cycle_dir().unwrap(), "fake-model".into()); + let mut files = AnswerFiles::new(dao.run.next_cycle_dir().unwrap(), "fake-model".into(), vec![]); cycle(&mut dao.run, &mut files, &scorer).unwrap() }; let dir = dao.root.join("cycle-1"); @@ -2596,6 +2596,8 @@ fn source_tripwires() { assert!(!text.contains(banned), "{file} names {banned}"); } } + // Every prompt file leaves through RunStores::answer_files, which withholds Test text. + assert!(!read("main.rs").contains("AnswerFiles::new")); let mut all = Vec::new(); let mut stack = vec![src.clone()]; while let Some(dir) = stack.pop() { @@ -3036,7 +3038,7 @@ fn views_with_one_meaning_share_one_unanswered_lowering_prompt() { }; std::fs::write(round1.join(format!("{}.answer", p.digest())), answer).unwrap(); } - let mut files = AnswerFiles::new(round1.clone(), "fake-model".into()); + let mut files = AnswerFiles::new(round1.clone(), "fake-model".into(), vec![]); let RoundOutcome::Unanswered { prompts: asked } = round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap() else { @@ -3190,7 +3192,7 @@ fn parse_reasons_count_per_code_over_both_splits_and_name_unreadable() { let answer = serde_json::json!({ "text": table[key] }).to_string(); std::fs::write(round1.join(format!("{}.answer", p.digest())), answer).unwrap(); } - let mut files = AnswerFiles::new(round1, "fake-model".into()); + let mut files = AnswerFiles::new(round1, "fake-model".into(), vec![]); let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()); assert_eq!(reasons(&r), [("syntax", 1), ("unreadable", 1)]); } @@ -3483,7 +3485,7 @@ fn a_recall_of_the_wrong_words_or_no_answer_reads_low_and_a_missing_answer_waits let r = reading.languages[0].clone(); assert_eq!((r.n, r.answered, r.chrf), (3, 0, 0.0)); // With no answers on disk the files port waits on all three prompts. - let mut files = AnswerFiles::new(dao.run.recall_dir(), "m".into()); + let mut files = AnswerFiles::new(dao.run.recall_dir(), "m".into(), vec![]); let RecallOutcome::Unanswered { prompts } = dao.run.recall(&mut files).unwrap() else { panic!("nobody answered"); }; @@ -4031,7 +4033,7 @@ fn a_stance_ablation_records_nothing_and_writes_nothing_in_its_round() { let before = (snapshot(&round1), snapshot(&dao.root), dao.run.rounds().unwrap(), dao.run.decisions().unwrap()); let go = || { let mut rec = RecordedAnswers::new(round1.clone(), "fake-model".into()); - let mut files = weksa::model::AnswerFiles::new(dao.run.ablation_dir(1), "fake-model".into()); + let mut files = weksa::model::AnswerFiles::new(dao.run.ablation_dir(1), "fake-model".into(), vec![]); // Nothing answers yet: the replay and variant prompts are written. dao.run.stance_ablation(1, &mut rec, &mut files, &Scorer::new(Bag)) }; @@ -5114,3 +5116,126 @@ fn a_run_reads_entries_and_scenes_at_its_own_head() { ); assert!(run.is_ok(), "{:?}", run.err()); } + +// ---- test-text-guard: no prompt file or answer for a prompt holding Test text ---- + +#[test] +fn answer_files_withhold_prompts_that_hold_test_text() { + use weksa::model::{AnswerFiles, ModelPort, PortError}; + let p = prompt(); + let dir = tempfile::tempdir().unwrap(); + let round = dir.path().join("round-1"); + let refused = |t: &str| { + AnswerFiles::new(round.clone(), "m".into(), vec![t.to_string()]).answer(&p) + == Err(PortError::Withheld) + }; + // A text of 12 normalised characters or more counts anywhere in the prompt, in + // any case, spacing, wrapping or width: the corpus door's own sameness. + assert!(refused("Nibu says mind the hatch.")); + assert!(refused("NIBU SAYS MIND THE HATCH")); + assert!(refused("nibu says\n mind the\n\n hatch")); + assert!(refused("\u{ff2e}ibu says mind the hatch.")); + assert!(!round.exists(), "a refused prompt creates nothing"); + // The threshold is 12 normalised characters: "the hatch" (8) says nothing about a + // prompt, "mind the hatch" (12) does. + assert!(!refused("the hatch")); + assert!(refused("mind the hatch")); + assert!(!refused(" "), "a blank withheld text matches nothing"); + // Unrelated text passes and the prompt is then written for its answerer. + let mut clean = AnswerFiles::new(round.clone(), "m".into(), vec!["The ferry leaves at noon.".into()]); + assert_eq!(clean.answer(&p), Err(PortError::Unanswered)); + assert!(round.join(format!("{}.prompt", p.digest())).exists()); + // The refusal comes before a recorded answer is read. + std::fs::write(round.join(format!("{}.answer", p.digest())), "x").unwrap(); + assert!(refused("Nibu says mind the hatch.")); +} + +#[test] +fn a_round_stops_on_a_prompt_that_holds_test_text() { + use common::{cast, entities_in, group_in_bucket, item_in}; + use weksa::model::{ModelPort, PortError}; + use weksa::run::{RunError, round}; + let mut w = world(); + let entities = entities_in(w.dir.path()); + // Test sentences (one withdrawn), and Train lines that quote them: the corpus + // door refuses an identical twin, not a longer line that holds one. + let held = "Hold this sentence back."; + let gone = "A withdrawn Test sentence."; + w.corpus + .add(item_in("item:held:3", gone, &group_in_bucket(91), cast()), &entities) + .unwrap(); + w.corpus.withdraw("item:held:3").unwrap(); + for (key, line, bucket) in [ + ("item:held:1", held.to_string(), 90), + ("item:held:2", format!("{held} And then some."), 10), + ("item:held:4", format!("{gone} And then some."), 11), + ] { + w.corpus + .add(item_in(key, &line, &group_in_bucket(bucket), cast()), &entities) + .unwrap(); + } + let mut run = run_over(w.dir.path()); + let dir = w.dir.path().join("answers"); + let mut files = run.answer_files(dir.clone()).unwrap(); + let inv = run.inventory().inventory().unwrap(); + let train = run.train().unwrap(); + let encode = |key: &str| { + let offer = run.offer(key).unwrap().unwrap(); + weksa::context::Prompt::encode(&offer, &train, &inv, 4) + }; + assert_eq!(files.answer(&encode("item:nibu:1")), Err(PortError::Unanswered)); + for key in ["item:held:2", "item:held:4"] { + assert_eq!(files.answer(&encode(key)), Err(PortError::Withheld), "{key}"); + } + assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 1, "only the clean prompt"); + // The round stops on it rather than skipping the item. A model-written line in a + // Test group is withheld too: it is in no split, and its text is still held out. + // (Covered by test_texts taking every Test-group item, below.) + let scorer = weksa::score::Scorer::new(OneBag); + let mut round_files = run.answer_files(w.dir.path().join("round")).unwrap(); + assert!(matches!( + round(&mut run, &mut round_files, &scorer), + Err(RunError::Withheld) + )); +} + +/// Refuses every prompt as holding Test text, as the answer door does. +struct Refuses; + +impl weksa::model::ModelPort for Refuses { + fn model(&self) -> &str { + "fake-model" + } + fn answer(&mut self, _: &weksa::context::Prompt) -> Result { + Err(weksa::model::PortError::Withheld) + } +} + +#[test] +fn every_model_call_stops_on_a_withheld_prompt() { + use weksa::run::{RunError, round}; + let w = world(); + let mut run = run_over(w.dir.path()); + let scorer = weksa::score::Scorer::new(OneBag); + let r = round(&mut run, &mut Refuses, &scorer); + assert!(matches!(r, Err(RunError::Withheld)), "{:?}", r.map(|_| ())); + assert!(matches!(run.mint_ontology(&mut Refuses, "en-US"), Err(RunError::Withheld))); +} + +#[test] +fn model_written_test_group_text_is_withheld_too() { + use common::{entities_in, group_in_bucket, item_in, model}; + let mut w = world(); + let entities = entities_in(w.dir.path()); + w.corpus + .add(item_in("item:held:5", "A model line of a Test meaning.", &group_in_bucket(92), model()), &entities) + .unwrap(); + assert!( + w.corpus + .test_texts() + .unwrap() + .iter() + .any(|t| t == "A model line of a Test meaning."), + "a model-written Test-group item is in no split, and its text is still held out" + ); +} diff --git a/crates/weksa/tests/ontology_mint.rs b/crates/weksa/tests/ontology_mint.rs index 2f150b3..3085d5d 100644 --- a/crates/weksa/tests/ontology_mint.rs +++ b/crates/weksa/tests/ontology_mint.rs @@ -242,7 +242,7 @@ fn mint_ontology_writes_nothing_but_its_prompt_and_answer_files() { let dir = tempfile::tempdir().unwrap(); let run = fixture(dir.path(), &["en-US", "ja-JP"]); let before = digests(dir.path()); - let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into()); + let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into(), vec![]); let OntologyMintOutcome::Unanswered { prompt } = run.mint_ontology(&mut files, "en-US").unwrap() else { @@ -268,7 +268,7 @@ fn mint_ontology_writes_nothing_but_its_prompt_and_answer_files() { fn mint_ontology_unanswered_then_drafted_then_refused() { let dir = tempfile::tempdir().unwrap(); let run = fixture(dir.path(), &["en-US"]); - let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into()); + let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into(), vec![]); let OntologyMintOutcome::Unanswered { prompt } = run.mint_ontology(&mut files, "en-US").unwrap() else { @@ -310,7 +310,7 @@ fn a_mint_answer_admits_into_a_copy_and_a_new_run_pins_the_copy() { let draft = format!( r#"{{"entries":[{{"layer":"Discourse","when":[{{"slot":"Stance","feature":"{STANCE_G}","test":{{"AtLeast":500}}}}],"choice":"a grown choice","examples":[]}}]}}"# ); - let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into()); + let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into(), vec![]); let OntologyMintOutcome::Unanswered { prompt } = run.mint_ontology(&mut files, "en-US").unwrap() else {