test-text-guard s2 fixes: drop the short-text path, pin the 12-char threshold, round and mint-ontology stop on Withheld, model-written Test-group text withheld

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Weksa 2026-11-13 00:57:21 +00:00
commit 8e50bac366
3 changed files with 56 additions and 25 deletions

View file

@ -56,29 +56,17 @@ impl AnswerFiles {
}
}
/// Normalised length from which a Test text is matched anywhere in a prompt.
/// Normalised length from which a Test text is matched. Every live Test text is
/// longer; a shorter one ("Stop.") would match any prompt that says "Stop. Then
/// ...", so a Test item that short is not caught here.
const LONG: usize = 12;
/// Whether `prompt` holds `text`. The corpus door's own sameness
/// ([`normalized`]: NFKC, case fold, spacing and punctuation dropped) decides
/// for a text of `LONG` normalised characters or more, so a re-cased,
/// re-spaced, wrapped or width-changed copy still counts. A shorter text would
/// match anything ("Stop." is in "Stop. Then ..."), so it counts only as a
/// line's end after a non-letter (every renderer prints an item's text last on its line).
/// Whether `prompt` holds `text`, by the corpus door's own sameness
/// ([`normalized`]: NFKC, case fold, spacing and punctuation dropped): a
/// re-cased, re-spaced, wrapped or width-changed copy still counts.
fn holds(prompt: &str, text: &str) -> bool {
let text = text.trim();
let key = normalized(text);
if key.is_empty() {
return false;
}
if key.chars().count() >= LONG {
return normalized(prompt).contains(&key);
}
prompt.lines().any(|l| {
l.trim_end().strip_suffix(text).is_some_and(|head| {
head.chars().next_back().is_none_or(|c| !c.is_alphanumeric())
})
})
key.chars().count() >= LONG && normalized(prompt).contains(&key)
}
/// Answers read from one directory and nothing else: `<digest>.answer` is the

View file

@ -939,7 +939,7 @@ impl RunStores {
}
/// The answer door for prompts in `dir`: refuses a prompt that holds the text
/// of any live Test item of the corpus, in any language.
/// of any item in a Test group of the corpus, in any language.
pub fn answer_files(&self, dir: PathBuf) -> anyhow::Result<AnswerFiles> {
let withheld = self.corpus.test_texts()?;
Ok(AnswerFiles::new(dir, self.record.model.clone(), withheld))

View file

@ -5136,10 +5136,10 @@ fn answer_files_withhold_prompts_that_hold_test_text() {
assert!(refused("nibu says\n mind the\n\n hatch"));
assert!(refused("\u{ff2e}ibu says mind the hatch."));
assert!(!round.exists(), "a refused prompt creates nothing");
// A short text counts only as the end of a line after a non-letter.
assert!(refused("the hatch."));
assert!(!refused("hatch"), "a mid-word short text is not the line's end");
assert!(!refused("mind the"));
// The threshold is 12 normalised characters: "the hatch" (8) says nothing about a
// prompt, "mind the hatch" (12) does.
assert!(!refused("the hatch"));
assert!(refused("mind the hatch"));
assert!(!refused(" "), "a blank withheld text matches nothing");
// Unrelated text passes and the prompt is then written for its answerer.
let mut clean = AnswerFiles::new(round.clone(), "m".into(), vec!["The ferry leaves at noon.".into()]);
@ -5188,7 +5188,9 @@ fn a_round_stops_on_a_prompt_that_holds_test_text() {
assert_eq!(files.answer(&encode(key)), Err(PortError::Withheld), "{key}");
}
assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 1, "only the clean prompt");
// The round stops on it rather than skipping the item.
// The round stops on it rather than skipping the item. A model-written line in a
// Test group is withheld too: it is in no split, and its text is still held out.
// (Covered by test_texts taking every Test-group item, below.)
let scorer = weksa::score::Scorer::new(OneBag);
let mut round_files = run.answer_files(w.dir.path().join("round")).unwrap();
assert!(matches!(
@ -5196,3 +5198,44 @@ fn a_round_stops_on_a_prompt_that_holds_test_text() {
Err(RunError::Withheld)
));
}
/// Refuses every prompt as holding Test text, as the answer door does.
struct Refuses;
impl weksa::model::ModelPort for Refuses {
fn model(&self) -> &str {
"fake-model"
}
fn answer(&mut self, _: &weksa::context::Prompt) -> Result<String, weksa::model::PortError> {
Err(weksa::model::PortError::Withheld)
}
}
#[test]
fn every_model_call_stops_on_a_withheld_prompt() {
use weksa::run::{RunError, round};
let w = world();
let mut run = run_over(w.dir.path());
let scorer = weksa::score::Scorer::new(OneBag);
let r = round(&mut run, &mut Refuses, &scorer);
assert!(matches!(r, Err(RunError::Withheld)), "{:?}", r.map(|_| ()));
assert!(matches!(run.mint_ontology(&mut Refuses, "en-US"), Err(RunError::Withheld)));
}
#[test]
fn model_written_test_group_text_is_withheld_too() {
use common::{entities_in, group_in_bucket, item_in, model};
let mut w = world();
let entities = entities_in(w.dir.path());
w.corpus
.add(item_in("item:held:5", "A model line of a Test meaning.", &group_in_bucket(92), model()), &entities)
.unwrap();
assert!(
w.corpus
.test_texts()
.unwrap()
.iter()
.any(|t| t == "A model line of a Test meaning."),
"a model-written Test-group item is in no split, and its text is still held out"
);
}