Require word boundaries at both ends of a located source span (PA.f67)
`SourceSpan::locate` in ghostlight-persona-projection was a raw substring `find`: "I can" byte-exact matched inside "I cannot go" and was accepted as the quoted span, even though it is not the word the model claims to be quoting. "Speech is a verbatim span" only holds if the span's edges are word edges. `locate` now walks every occurrence of the quote (`match_indices`) and takes the first one whose start and end each land on a word boundary: the edge of the source, or a character that is not Unicode-alphanumeric (`starts_on_word_boundary` / `ends_on_word_boundary`). A first occurrence that fails either edge is skipped in favor of a later one. The Interpreter (crates/ghostlight/src/controllers.rs) calls the same function through `SourceRange`; `cargo test -p ghostlight --lib` still passes at 603, so no existing test relied on a mid-word span. Added five tests: a mid-word quote is refused, a whole-word quote is accepted, a span equal to the whole source is accepted, a later boundary-valid occurrence is found after an invalid first one (the first occurrence fails only the start edge), and punctuation at a span's edges is handled. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
56020d4155
commit
275b0f21e6
1 changed files with 93 additions and 8 deletions
|
|
@ -340,6 +340,29 @@ impl PersonaTurn {
|
|||
}
|
||||
}
|
||||
|
||||
/// Whether `byte` is a valid left edge for a quoted span: the start of
|
||||
/// `source`, or the character immediately before it is not
|
||||
/// Unicode-alphanumeric. `byte` must itself be a valid char boundary, which
|
||||
/// every caller here gets for free from `match_indices`.
|
||||
fn starts_on_word_boundary(source: &str, byte: usize) -> bool {
|
||||
byte == 0
|
||||
|| !source[..byte]
|
||||
.chars()
|
||||
.next_back()
|
||||
.is_some_and(char::is_alphanumeric)
|
||||
}
|
||||
|
||||
/// The mirror of [`starts_on_word_boundary`] for the right edge: the end of
|
||||
/// `source`, or the character immediately after it is not
|
||||
/// Unicode-alphanumeric.
|
||||
fn ends_on_word_boundary(source: &str, byte: usize) -> bool {
|
||||
byte == source.len()
|
||||
|| !source[byte..]
|
||||
.chars()
|
||||
.next()
|
||||
.is_some_and(char::is_alphanumeric)
|
||||
}
|
||||
|
||||
/// An exact UTF-8 byte range in the preserved Persona source prose.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||
pub struct SourceSpan {
|
||||
|
|
@ -360,18 +383,30 @@ impl SourceSpan {
|
|||
})
|
||||
}
|
||||
|
||||
/// The first exact occurrence of `quote` in `source`. The model quotes
|
||||
/// words; the harness counts bytes. A model that counts bytes itself cut
|
||||
/// three of eight utterances mid-word on the first Claude road run.
|
||||
/// The first occurrence of `quote` in `source` that lands on a word
|
||||
/// boundary at both ends. The model quotes words; the harness counts
|
||||
/// bytes. A model that counts bytes itself cut three of eight utterances
|
||||
/// mid-word on the first Claude road run — and a raw substring `find`
|
||||
/// has the same defect the other way: "I can" is a byte-exact match
|
||||
/// inside "I cannot go", but it is not the word the model claims to be
|
||||
/// quoting. Each end must be the edge of `source` or sit next to a
|
||||
/// non-alphanumeric (Unicode-aware) character; a first occurrence that
|
||||
/// fails either edge is skipped in favor of a later one.
|
||||
pub fn locate(source: &str, quote: &str) -> Option<Self> {
|
||||
if quote.is_empty() {
|
||||
return None;
|
||||
}
|
||||
source.find(quote).map(|start_byte| Self {
|
||||
start_byte,
|
||||
end_byte: start_byte + quote.len(),
|
||||
verbatim: quote.to_owned(),
|
||||
})
|
||||
source
|
||||
.match_indices(quote)
|
||||
.find(|(start_byte, matched)| {
|
||||
starts_on_word_boundary(source, *start_byte)
|
||||
&& ends_on_word_boundary(source, start_byte + matched.len())
|
||||
})
|
||||
.map(|(start_byte, matched)| Self {
|
||||
start_byte,
|
||||
end_byte: start_byte + matched.len(),
|
||||
verbatim: matched.to_owned(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn whole(source: &str) -> Self {
|
||||
|
|
@ -718,6 +753,56 @@ mod tests {
|
|||
assert!(!prompt.contains("sha256:"));
|
||||
}
|
||||
|
||||
/// PA.f67: `SourceSpan::locate` used a raw substring `find`, so "I can"
|
||||
/// was accepted as a span of "I cannot go" — a byte-exact match that is
|
||||
/// not the word the model claims to quote. A located span must land on a
|
||||
/// word boundary at both ends.
|
||||
#[test]
|
||||
fn a_mid_word_quote_is_refused() {
|
||||
assert_eq!(SourceSpan::locate("I cannot go", "I can"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_whole_word_quote_is_accepted() {
|
||||
let source = "I cannot go";
|
||||
let located = SourceSpan::locate(source, "I cannot").expect("a whole-word quote locates");
|
||||
assert_eq!(located.start_byte(), 0);
|
||||
assert_eq!(located.end_byte(), "I cannot".len());
|
||||
assert_eq!(located.verbatim(), "I cannot");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_span_equal_to_the_whole_source_is_accepted() {
|
||||
let source = "I cannot go";
|
||||
let located = SourceSpan::locate(source, source).expect("the whole source locates");
|
||||
assert_eq!(located.start_byte(), 0);
|
||||
assert_eq!(located.end_byte(), source.len());
|
||||
}
|
||||
|
||||
/// The first occurrence of "cat" sits inside "subcat" — preceded by an
|
||||
/// alphanumeric character on the left, so it fails only the start-edge
|
||||
/// check. The second, standalone "cat" is boundary-valid on both edges
|
||||
/// and is what `locate` must return.
|
||||
#[test]
|
||||
fn a_later_boundary_valid_occurrence_is_found_after_an_invalid_first() {
|
||||
let source = "The subcat and the cat both purred.";
|
||||
let located = SourceSpan::locate(source, "cat").expect("the standalone word locates");
|
||||
let expected_start = source.rfind("cat").unwrap();
|
||||
assert_eq!(located.start_byte(), expected_start);
|
||||
assert_eq!(located.verbatim(), "cat");
|
||||
}
|
||||
|
||||
/// Punctuation at a span's edges is not alphanumeric, so it is a valid
|
||||
/// boundary on either side without needing whitespace.
|
||||
#[test]
|
||||
fn a_span_with_punctuation_at_its_edges_is_handled() {
|
||||
let source = "She said, \"Wait for the bell,\" and left.";
|
||||
let located =
|
||||
SourceSpan::locate(source, "Wait for the bell").expect("a comma-bounded quote locates");
|
||||
assert_eq!(located.verbatim(), "Wait for the bell");
|
||||
assert!(located.is_exact_in(source));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persisted_exact_turn_rehydrates_with_the_same_receipt() {
|
||||
let recorded = turn("I tell Mara, \"The western brace is giving way.\"");
|
||||
|
|
|
|||
Loading…
Reference in a new issue