GameCult/Weksa
Watch
9
0
Fork
You've already forked Weksa
 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

test-text-guard: AnswerFiles refuses a prompt that holds the text of an item in a Test group #65

Merged
weksa merged 3 commits from weksa-lowering/test-text-guard into main 2026-11-13 00:57:33 +00:00
Conversation 0 Commits 3 Files changed 6 +216 -41
6 changed files with 216 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

25
crates/weksa/src/main.rs
View file

@ -19,7 +19,7 @@ use weksa::run::{
};
use weksa::score::ScoreInput;
use weksa::{
Annotator, AnswerFiles, CorpusStore, EntityId, EntityStore, Family, InventoryStore, OllamaEmbedder, OntologyStore, OptionLabel,
Annotator, CorpusStore, EntityId, EntityStore, Family, InventoryStore, OllamaEmbedder, OntologyStore, OptionLabel,
PacketStore, PairRelation, Party, Scene, Slot, SpeakerStateStore, ValueKind, Scorer, Split, SplitScheme, StoreError, lower, pronounce, validate,
};
@ -1070,10 +1070,7 @@ fn main() -> Result<()> {
Command::Loop(LoopCommand::Round { root, ollama_url }) => {
let unreadable = |_| anyhow::anyhow!("a store would not read");
let mut run = RunStores::open(&root)?;
let mut files = AnswerFiles::new(
run.next_round_dir().map_err(unreadable)?,
run.record().model.clone(),
);
let mut files = run.answer_files(run.next_round_dir().map_err(unreadable)?)?;
let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder));
match round(&mut run, &mut files, &scorer)? {
RoundOutcome::Unanswered { prompts } => {
@ -1093,10 +1090,11 @@ fn main() -> Result<()> {
ollama_url,
}) => {
let run = RunStores::open(&root)?;
let model = run.record().model.clone();
let mut recorded =
RecordedAnswers::new(root.join(format!("round-{round}")), model.clone());
let mut files = AnswerFiles::new(run.ablation_dir(round), model);
let mut recorded = RecordedAnswers::new(
root.join(format!("round-{round}")),
run.record().model.clone(),
);
let mut files = run.answer_files(run.ablation_dir(round))?;
let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder));
match run.stance_ablation(round, &mut recorded, &mut files, &scorer)? {
AblationOutcome::Unanswered { prompts } => {
@ -1119,7 +1117,7 @@ fn main() -> Result<()> {
}
Command::Loop(LoopCommand::Recall { root }) => {
let run = RunStores::open(&root)?;
let mut files = AnswerFiles::new(run.recall_dir(), run.record().model.clone());
let mut files = run.answer_files(run.recall_dir())?;
match run.recall(&mut files)? {
RecallOutcome::Unanswered { prompts } => {
for digest in &prompts {
@ -1133,7 +1131,7 @@ fn main() -> Result<()> {
}
Command::Loop(LoopCommand::MintOntology { root, language }) => {
let run = RunStores::open(&root)?;
let mut files = AnswerFiles::new(run.ontology_mint_dir(), run.record().model.clone());
let mut files = run.answer_files(run.ontology_mint_dir())?;
match run.mint_ontology(&mut files, &language)? {
OntologyMintOutcome::Unanswered { prompt } => {
println!("unanswered {prompt}");
@ -1152,10 +1150,7 @@ fn main() -> Result<()> {
Command::Loop(LoopCommand::Cycle { root, ollama_url }) => {
let unreadable = |_| anyhow::anyhow!("a store would not read");
let mut run = RunStores::open(&root)?;
let mut files = AnswerFiles::new(
run.next_cycle_dir().map_err(unreadable)?,
run.record().model.clone(),
);
let mut files = run.answer_files(run.next_cycle_dir().map_err(unreadable)?)?;
let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder));
match cycle(&mut run, &mut files, &scorer)? {
CycleOutcome::Unanswered { prompts } => {

28
crates/weksa/src/model.rs
View file

@ -7,6 +7,7 @@ use std::io::ErrorKind;
use std::path::PathBuf;
use crate::context::Prompt;
use crate::store::normalized;
/// A model answering [`Prompt`]s and nothing else.
pub trait ModelPort {
@ -20,6 +21,8 @@ pub trait ModelPort {
pub enum PortError {
/// Nobody has answered yet.
Unanswered,
/// The prompt holds the text of an item in a Test group. It is neither written nor answered.
Withheld,
Io(ErrorKind),
}
@ -27,6 +30,7 @@ impl fmt::Display for PortError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
PortError::Unanswered => f.write_str("the prompt is unanswered"),
PortError::Withheld => f.write_str("the prompt holds Test item text"),
PortError::Io(kind) => write!(f, "io error {kind:?}"),
}
}
@ -40,14 +44,31 @@ impl std::error::Error for PortError {}
pub struct AnswerFiles {
dir: PathBuf,
model: String,
withheld: Vec<String>,
}
impl AnswerFiles {
pub fn new(dir: PathBuf, model: String) -> Self {
Self { dir, model }
/// Refuses (`PortError::Withheld`) any prompt that holds the text of one of
/// `withheld`, before it reads or writes a file: the one door a prompt
/// leaves the loop through, so a Test sentence cannot reach a model by it.
pub fn new(dir: PathBuf, model: String, withheld: Vec<String>) -> Self {
Self { dir, model, withheld }
}
}
/// Normalised length from which a Test text is matched. Every live Test text is
/// longer; a shorter one ("Stop.") would match any prompt that says "Stop. Then
/// ...", so a Test item that short is not caught here.
const LONG: usize = 12;
/// Whether `prompt` holds `text`, by the corpus door's own sameness
/// ([`normalized`]: NFKC, case fold, spacing and punctuation dropped): a
/// re-cased, re-spaced, wrapped or width-changed copy still counts.
fn holds(prompt: &str, text: &str) -> bool {
let key = normalized(text);
key.chars().count() >= LONG && normalized(prompt).contains(&key)
}
/// Answers read from one directory and nothing else: `<digest>.answer` is the
/// reply to the prompt whose digest is `<digest>`. A prompt with no file is
/// [`PortError::Unanswered`]; this port never creates or writes anything. The
@ -86,6 +107,9 @@ impl ModelPort for AnswerFiles {
fn answer(&mut self, prompt: &Prompt) -> Result<String, PortError> {
let io = |e: std::io::Error| PortError::Io(e.kind());
if self.withheld.iter().any(|t| holds(prompt.text(), t)) {
return Err(PortError::Withheld);
}
let digest = prompt.digest();
match RecordedAnswers::new(self.dir.clone(), self.model.clone()).answer(prompt) {
Err(PortError::Unanswered) => {}

27
crates/weksa/src/run.rs
View file

@ -44,7 +44,7 @@ use crate::encode::{
EditDraft, LoweringDraft, MAX_EDIT_CHANGES, OntologyDraft, PacketDraft, encoder_id,
};
use crate::form::{is_attestation, is_language_tag, is_slug};
use crate::model::{ModelPort, PortError};
use crate::model::{AnswerFiles, ModelPort, PortError};
use crate::packet::{Packet, Stance};
use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate};
use crate::split::{Split, SplitScheme, fnv1a64, fnv1a64_of, hex16};
@ -474,6 +474,8 @@ pub enum RunError {
/// The inventory store holds no revision.
NoInventory,
Io(ErrorKind),
/// A prompt held the text of a Test item and was refused before any file.
Withheld,
/// A store or the run file would not read or write.
Store,
/// The embedder failed.
@ -507,6 +509,7 @@ impl fmt::Display for RunError {
RunError::Drift(of) => write!(f, "drift in {of:?}"),
RunError::NoInventory => f.write_str("the inventory holds no revision"),
RunError::Io(kind) => write!(f, "io error {kind:?}"),
RunError::Withheld => f.write_str("a prompt holds Test item text; it was neither written nor answered"),
RunError::Store => f.write_str("a store would not read or write"),
RunError::Unrecorded => f.write_str(
"the round holds no answer to a lowering prompt rebuilt now: the prompt digest drifted or the head inventory moved since the round",
@ -935,6 +938,13 @@ impl RunStores {
.collect())
}
/// The answer door for prompts in `dir`: refuses a prompt that holds the text
/// of any item in a Test group of the corpus, in any language.
pub fn answer_files(&self, dir: PathBuf) -> anyhow::Result<AnswerFiles> {
let withheld = self.corpus.test_texts()?;
Ok(AnswerFiles::new(dir, self.record.model.clone(), withheld))
}
/// The recorded rounds, in order.
pub fn rounds(&self) -> anyhow::Result<Vec<RoundRecord>> {
let mut rounds = self.cache.get_all::<RoundRecord>()?;
@ -1120,7 +1130,8 @@ impl RunStores {
}
Err(PortError::Unanswered) => unanswered.push(prompt.digest()),
Err(PortError::Io(ErrorKind::InvalidData)) => scores.push((language, None)),
Err(PortError::Io(kind)) => return Err(RunError::Io(kind)),
Err(PortError::Withheld) => return Err(RunError::Withheld),
Err(PortError::Io(kind)) => return Err(RunError::Io(kind)),
}
}
if !unanswered.is_empty() {
@ -1208,7 +1219,8 @@ impl RunStores {
Err(PortError::Io(ErrorKind::InvalidData)) => {
Ok(OntologyMintOutcome::Refused { code: "invalid-data" })
}
Err(PortError::Io(kind)) => Err(RunError::Io(kind)),
Err(PortError::Withheld) => Err(RunError::Withheld),
Err(PortError::Io(kind)) => Err(RunError::Io(kind)),
}
}
@ -1305,6 +1317,7 @@ fn asked(port: &mut dyn ModelPort, prompt: &Prompt) -> Result<Asked, RunError> {
.map_or(Asked::Unusable, |d| Asked::Text(d.text))),
Err(PortError::Io(ErrorKind::InvalidData)) => Ok(Asked::Unusable),
Err(PortError::Unanswered) => Ok(Asked::Waiting),
Err(PortError::Withheld) => Err(RunError::Withheld),
Err(PortError::Io(kind)) => Err(RunError::Io(kind)),
}
}
@ -1636,7 +1649,8 @@ fn score_round<M: ModelPort, E: Embedder>(
answers.push((offer.key().to_string(), None))
}
Err(PortError::Unanswered) => unanswered.push(prompt.digest()),
Err(PortError::Io(kind)) => return Err(RunError::Io(kind)),
Err(PortError::Withheld) => return Err(RunError::Withheld),
Err(PortError::Io(kind)) => return Err(RunError::Io(kind)),
}
calls.push(Call {
prompt: prompt.digest(),
@ -1730,6 +1744,10 @@ fn score_round<M: ModelPort, E: Embedder>(
}
continue;
}
Err(PortError::Withheld) => {
discard_packets(dir);
return Err(RunError::Withheld);
}
Err(PortError::Io(kind)) => {
discard_packets(dir);
return Err(RunError::Io(kind));
@ -2277,6 +2295,7 @@ pub fn cycle<M: ModelPort, E: Embedder>(
prompts: vec![prompt.digest()],
});
}
Err(PortError::Withheld) => return Err(RunError::Withheld),
Err(PortError::Io(kind)) => return Err(RunError::Io(kind)),
};

12
crates/weksa/src/store.rs
View file

@ -12,6 +12,7 @@ use unicode_normalization::{UnicodeNormalization, char::is_combining_mark, is_nf
use unicode_segmentation::UnicodeSegmentation;
use crate::corpus::{CorpusItem, CorpusItemId, Source, item_key, key_slug};
use crate::split::{Split, SplitScheme};
use crate::entity::{Entity, EntityOrigin, Pronunciation};
use crate::form::{
is_attestation, is_calendar_date, is_joining_only, is_language_tag, is_refused_format, is_slug,
@ -248,6 +249,17 @@ impl CorpusStore {
Ok(items)
}
/// The text of every item in a Test group, any language, withdrawn and
/// model-written ones included: what no prompt may hold.
pub fn test_texts(&self) -> anyhow::Result<Vec<String>> {
Ok(self
.all()?
.into_iter()
.filter(|item| SplitScheme::V1.of_bucket(SplitScheme::V1.bucket(&item.group)) == Split::Test)
.map(|item| item.text)
.collect())
}
pub fn get(&self, key: &str) -> anyhow::Result<Option<CorpusItem>> {
self.cache.get::<CorpusItem>(key)
}

159
crates/weksa/tests/golf_loop.rs
View file

@ -669,7 +669,7 @@ fn answer_files_write_the_prompt_and_read_the_answer() {
let p = prompt();
let dir = tempfile::tempdir().unwrap();
let round = dir.path().join("round-1");
let mut files = AnswerFiles::new(round.clone(), "answerer".into());
let mut files = AnswerFiles::new(round.clone(), "answerer".into(), vec![]);
assert_eq!(files.model(), "answerer");
// No answer: the prompt is written under its digest, and nothing else.
@ -701,7 +701,7 @@ fn answer_files_write_the_prompt_and_read_the_answer() {
// A directory that cannot be made is an io error of its kind.
std::fs::write(dir.path().join("file"), "").unwrap();
let mut blocked = AnswerFiles::new(dir.path().join("file").join("round"), "m".into());
let mut blocked = AnswerFiles::new(dir.path().join("file").join("round"), "m".into(), vec![]);
assert!(matches!(blocked.answer(&p), Err(PortError::Io(_))));
// A reply or prompt file that cannot be read is an io error, never "unanswered".
@ -1113,7 +1113,7 @@ fn unanswered_round_writes_nothing() {
let expected = prompts(&dao.run);
assert_eq!(expected.len(), 3);
let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into());
let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]);
let RoundOutcome::Unanswered { prompts: digests } =
round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()
else {
@ -1144,7 +1144,7 @@ fn unanswered_round_writes_nothing() {
for (key, p) in &expected[..2] {
std::fs::write(round1.join(format!("{}.answer", p.digest())), &answers[key]).unwrap();
}
let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into());
let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]);
let RoundOutcome::Unanswered { prompts: digests } =
round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()
else {
@ -1159,7 +1159,7 @@ fn unanswered_round_writes_nothing() {
let (key, p) = &expected[2];
std::fs::write(round1.join(format!("{}.answer", p.digest())), &answers[key]).unwrap();
let lowering = lowering_prompts(&dao, dao.run.inventory());
let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into());
let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]);
let RoundOutcome::Unanswered { prompts: digests } =
round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()
else {
@ -1183,7 +1183,7 @@ fn unanswered_round_writes_nothing() {
for (key, p) in &lowering[..2] {
std::fs::write(round1.join(format!("{}.answer", p.digest())), lowered(key)).unwrap();
}
let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into());
let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]);
let RoundOutcome::Unanswered { prompts: digests } =
round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()
else {
@ -1195,13 +1195,13 @@ fn unanswered_round_writes_nothing() {
assert_eq!(std::fs::read(dao.root.join("run.cc")).unwrap(), run_cc);
let (key, p) = &lowering[2];
std::fs::write(round1.join(format!("{}.answer", p.digest())), lowered(key)).unwrap();
let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into());
let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]);
let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap());
assert_eq!((r.n(), r.calls().len()), (1, 6));
assert_eq!(two_dp(r.language("en-US").unwrap().train_mean.chrf.unwrap()), 44.18);
// Round 2 asks again, encode first.
let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into());
let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]);
assert!(dao.root.join("round-2").parent().is_some());
let RoundOutcome::Unanswered { prompts: again } =
round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()
@ -1224,7 +1224,7 @@ fn unanswered_round_writes_nothing() {
)
.unwrap();
}
let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into());
let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into(), vec![]);
let second = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap());
assert_eq!(second.n(), 2);
let ns: Vec<u32> = dao.run.rounds().unwrap().iter().map(|r| r.n()).collect();
@ -1411,7 +1411,7 @@ fn non_utf8_encode_answer_is_a_parse_refusal() {
let answer = serde_json::json!({ "text": table[key] }).to_string();
std::fs::write(round1.join(format!("{}.answer", p.digest())), answer).unwrap();
}
let mut files = AnswerFiles::new(round1.clone(), "fake-model".into());
let mut files = AnswerFiles::new(round1.clone(), "fake-model".into(), vec![]);
let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap());
assert_eq!(r.calls().len(), 5);
// Item 3 is Validation: its refusal is a Parse refusal and the round stands.
@ -1621,7 +1621,7 @@ fn errors_echo_no_input() {
// A port error, from a canary directory.
let blocker = tmp.path().join("canaryfile");
std::fs::write(&blocker, "").unwrap();
let mut files = AnswerFiles::new(blocker.join("canarydir"), "m".into());
let mut files = AnswerFiles::new(blocker.join("canarydir"), "m".into(), vec![]);
let e = files.answer(&prompt()).err().unwrap();
note(&e, &e);
// A round over canary answers records refusals, not the answers.
@ -2084,7 +2084,7 @@ fn a_rejection_leaves_no_candidate_inventory_from_an_earlier_attempt() {
let scorer = Scorer::new(Bag);
let ask = |dao: &mut Dao| {
let mut files =
AnswerFiles::new(dao.run.next_cycle_dir().unwrap(), "fake-model".into());
AnswerFiles::new(dao.run.next_cycle_dir().unwrap(), "fake-model".into(), vec![]);
cycle(&mut dao.run, &mut files, &scorer).unwrap()
};
let dir = dao.root.join("cycle-1");
@ -2154,7 +2154,7 @@ fn unanswered_cycle_writes_nothing() {
let run_cc = std::fs::read(dao.root.join("run.cc")).unwrap();
let scorer = Scorer::new(Bag);
let ask = |dao: &mut Dao| {
let mut files = AnswerFiles::new(dao.run.next_cycle_dir().unwrap(), "fake-model".into());
let mut files = AnswerFiles::new(dao.run.next_cycle_dir().unwrap(), "fake-model".into(), vec![]);
cycle(&mut dao.run, &mut files, &scorer).unwrap()
};
let dir = dao.root.join("cycle-1");
@ -2596,6 +2596,8 @@ fn source_tripwires() {
assert!(!text.contains(banned), "{file} names {banned}");
}
}
// Every prompt file leaves through RunStores::answer_files, which withholds Test text.
assert!(!read("main.rs").contains("AnswerFiles::new"));
let mut all = Vec::new();
let mut stack = vec![src.clone()];
while let Some(dir) = stack.pop() {
@ -3036,7 +3038,7 @@ fn views_with_one_meaning_share_one_unanswered_lowering_prompt() {
};
std::fs::write(round1.join(format!("{}.answer", p.digest())), answer).unwrap();
}
let mut files = AnswerFiles::new(round1.clone(), "fake-model".into());
let mut files = AnswerFiles::new(round1.clone(), "fake-model".into(), vec![]);
let RoundOutcome::Unanswered { prompts: asked } =
round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()
else {
@ -3190,7 +3192,7 @@ fn parse_reasons_count_per_code_over_both_splits_and_name_unreadable() {
let answer = serde_json::json!({ "text": table[key] }).to_string();
std::fs::write(round1.join(format!("{}.answer", p.digest())), answer).unwrap();
}
let mut files = AnswerFiles::new(round1, "fake-model".into());
let mut files = AnswerFiles::new(round1, "fake-model".into(), vec![]);
let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap());
assert_eq!(reasons(&r), [("syntax", 1), ("unreadable", 1)]);
}
@ -3483,7 +3485,7 @@ fn a_recall_of_the_wrong_words_or_no_answer_reads_low_and_a_missing_answer_waits
let r = reading.languages[0].clone();
assert_eq!((r.n, r.answered, r.chrf), (3, 0, 0.0));
// With no answers on disk the files port waits on all three prompts.
let mut files = AnswerFiles::new(dao.run.recall_dir(), "m".into());
let mut files = AnswerFiles::new(dao.run.recall_dir(), "m".into(), vec![]);
let RecallOutcome::Unanswered { prompts } = dao.run.recall(&mut files).unwrap() else {
panic!("nobody answered");
};
@ -4031,7 +4033,7 @@ fn a_stance_ablation_records_nothing_and_writes_nothing_in_its_round() {
let before = (snapshot(&round1), snapshot(&dao.root), dao.run.rounds().unwrap(), dao.run.decisions().unwrap());
let go = || {
let mut rec = RecordedAnswers::new(round1.clone(), "fake-model".into());
let mut files = weksa::model::AnswerFiles::new(dao.run.ablation_dir(1), "fake-model".into());
let mut files = weksa::model::AnswerFiles::new(dao.run.ablation_dir(1), "fake-model".into(), vec![]);
// Nothing answers yet: the replay and variant prompts are written.
dao.run.stance_ablation(1, &mut rec, &mut files, &Scorer::new(Bag))
};
@ -5114,3 +5116,126 @@ fn a_run_reads_entries_and_scenes_at_its_own_head() {
);
assert!(run.is_ok(), "{:?}", run.err());
}
// ---- test-text-guard: no prompt file or answer for a prompt holding Test text ----
#[test]
fn answer_files_withhold_prompts_that_hold_test_text() {
use weksa::model::{AnswerFiles, ModelPort, PortError};
let p = prompt();
let dir = tempfile::tempdir().unwrap();
let round = dir.path().join("round-1");
let refused = |t: &str| {
AnswerFiles::new(round.clone(), "m".into(), vec![t.to_string()]).answer(&p)
== Err(PortError::Withheld)
};
// A text of 12 normalised characters or more counts anywhere in the prompt, in
// any case, spacing, wrapping or width: the corpus door's own sameness.
assert!(refused("Nibu says mind the hatch."));
assert!(refused("NIBU SAYS MIND THE HATCH"));
assert!(refused("nibu says\n mind the\n\n hatch"));
assert!(refused("\u{ff2e}ibu says mind the hatch."));
assert!(!round.exists(), "a refused prompt creates nothing");
// The threshold is 12 normalised characters: "the hatch" (8) says nothing about a
// prompt, "mind the hatch" (12) does.
assert!(!refused("the hatch"));
assert!(refused("mind the hatch"));
assert!(!refused(" "), "a blank withheld text matches nothing");
// Unrelated text passes and the prompt is then written for its answerer.
let mut clean = AnswerFiles::new(round.clone(), "m".into(), vec!["The ferry leaves at noon.".into()]);
assert_eq!(clean.answer(&p), Err(PortError::Unanswered));
assert!(round.join(format!("{}.prompt", p.digest())).exists());
// The refusal comes before a recorded answer is read.
std::fs::write(round.join(format!("{}.answer", p.digest())), "x").unwrap();
assert!(refused("Nibu says mind the hatch."));
}
#[test]
fn a_round_stops_on_a_prompt_that_holds_test_text() {
use common::{cast, entities_in, group_in_bucket, item_in};
use weksa::model::{ModelPort, PortError};
use weksa::run::{RunError, round};
let mut w = world();
let entities = entities_in(w.dir.path());
// Test sentences (one withdrawn), and Train lines that quote them: the corpus
// door refuses an identical twin, not a longer line that holds one.
let held = "Hold this sentence back.";
let gone = "A withdrawn Test sentence.";
w.corpus
.add(item_in("item:held:3", gone, &group_in_bucket(91), cast()), &entities)
.unwrap();
w.corpus.withdraw("item:held:3").unwrap();
for (key, line, bucket) in [
("item:held:1", held.to_string(), 90),
("item:held:2", format!("{held} And then some."), 10),
("item:held:4", format!("{gone} And then some."), 11),
] {
w.corpus
.add(item_in(key, &line, &group_in_bucket(bucket), cast()), &entities)
.unwrap();
}
let mut run = run_over(w.dir.path());
let dir = w.dir.path().join("answers");
let mut files = run.answer_files(dir.clone()).unwrap();
let inv = run.inventory().inventory().unwrap();
let train = run.train().unwrap();
let encode = |key: &str| {
let offer = run.offer(key).unwrap().unwrap();
weksa::context::Prompt::encode(&offer, &train, &inv, 4)
};
assert_eq!(files.answer(&encode("item:nibu:1")), Err(PortError::Unanswered));
for key in ["item:held:2", "item:held:4"] {
assert_eq!(files.answer(&encode(key)), Err(PortError::Withheld), "{key}");
}
assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 1, "only the clean prompt");
// The round stops on it rather than skipping the item. A model-written line in a
// Test group is withheld too: it is in no split, and its text is still held out.
// (Covered by test_texts taking every Test-group item, below.)
let scorer = weksa::score::Scorer::new(OneBag);
let mut round_files = run.answer_files(w.dir.path().join("round")).unwrap();
assert!(matches!(
round(&mut run, &mut round_files, &scorer),
Err(RunError::Withheld)
));
}
/// Refuses every prompt as holding Test text, as the answer door does.
struct Refuses;
impl weksa::model::ModelPort for Refuses {
fn model(&self) -> &str {
"fake-model"
}
fn answer(&mut self, _: &weksa::context::Prompt) -> Result<String, weksa::model::PortError> {
Err(weksa::model::PortError::Withheld)
}
}
#[test]
fn every_model_call_stops_on_a_withheld_prompt() {
use weksa::run::{RunError, round};
let w = world();
let mut run = run_over(w.dir.path());
let scorer = weksa::score::Scorer::new(OneBag);
let r = round(&mut run, &mut Refuses, &scorer);
assert!(matches!(r, Err(RunError::Withheld)), "{:?}", r.map(|_| ()));
assert!(matches!(run.mint_ontology(&mut Refuses, "en-US"), Err(RunError::Withheld)));
}
#[test]
fn model_written_test_group_text_is_withheld_too() {
use common::{entities_in, group_in_bucket, item_in, model};
let mut w = world();
let entities = entities_in(w.dir.path());
w.corpus
.add(item_in("item:held:5", "A model line of a Test meaning.", &group_in_bucket(92), model()), &entities)
.unwrap();
assert!(
w.corpus
.test_texts()
.unwrap()
.iter()
.any(|t| t == "A model line of a Test meaning."),
"a model-written Test-group item is in no split, and its text is still held out"
);
}

6
crates/weksa/tests/ontology_mint.rs
View file

@ -242,7 +242,7 @@ fn mint_ontology_writes_nothing_but_its_prompt_and_answer_files() {
let dir = tempfile::tempdir().unwrap();
let run = fixture(dir.path(), &["en-US", "ja-JP"]);
let before = digests(dir.path());
let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into());
let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into(), vec![]);
let OntologyMintOutcome::Unanswered { prompt } =
run.mint_ontology(&mut files, "en-US").unwrap()
else {
@ -268,7 +268,7 @@ fn mint_ontology_writes_nothing_but_its_prompt_and_answer_files() {
fn mint_ontology_unanswered_then_drafted_then_refused() {
let dir = tempfile::tempdir().unwrap();
let run = fixture(dir.path(), &["en-US"]);
let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into());
let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into(), vec![]);
let OntologyMintOutcome::Unanswered { prompt } =
run.mint_ontology(&mut files, "en-US").unwrap()
else {
@ -310,7 +310,7 @@ fn a_mint_answer_admits_into_a_copy_and_a_new_run_pins_the_copy() {
let draft = format!(
r#"{{"entries":[{{"layer":"Discourse","when":[{{"slot":"Stance","feature":"{STANCE_G}","test":{{"AtLeast":500}}}}],"choice":"a grown choice","examples":[]}}]}}"#
);
let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into());
let mut files = AnswerFiles::new(run.ontology_mint_dir(), "m".into(), vec![]);
let OntologyMintOutcome::Unanswered { prompt } =
run.mint_ontology(&mut files, "en-US").unwrap()
else {