From 4fea8fdd366a839e6ace55d73dc5734bcccc04d7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 22 Sep 2026 16:25:32 +0200 Subject: [PATCH] Use UAX #29 word boundaries in SourceSpan::locate (PA.f78) The hand-written is_alphanumeric neighbour rule in SourceSpan::locate got several boundary cases wrong: it accepted "I can" inside "I can't go" (an apostrophe is not alphanumeric, so it read as a boundary even though it binds the contraction into one word), accepted "the cafe" ending right before a combining acute mark, accepted "run" inside "run_fast", and falsely refused every CJK quote because every ideograph counts as alphanumeric and CJK prose carries no spaces. Replace the rule with real UAX #29 word segmentation from the unicode-segmentation crate: collect the boundary byte offsets split_word_bound_indices() reports for the source, plus source.len(), and accept an occurrence only when both its start and end land in that set. Delete starts_on_word_boundary/ends_on_word_boundary; the crate is the authority now, not a hand rule. unicode-segmentation is pinned to 1.x in the workspace [workspace.dependencies] table (Cargo.lock gains only that one package) and added to ghostlight-persona-projection's own Cargo.toml. Verified against UAX #29 with new tests: contractions (straight and curly apostrophe) are refused, a quote ending before a combining mark is refused, an underscore-joined word is refused, a leading-CJK two-character quote is accepted, a quote with its own leading space is accepted, a hyphen-bounded quote is accepted, and "Stop." in "Stop.Now" is refused (the crate glues a MidNumLet full stop between two letters into one word, so this is the crate's called shot, not a special case carved into the rule). Callers (Interpreter capture, Dungeon's span_is_exact) are unchanged; the lib and Dungeon bin suites both still pass in full (618 lib tests with the one known PA.f77 flake, 63 Dungeon bin tests). Co-Authored-By: Claude Sonnet 5 --- Cargo.lock | 7 + Cargo.toml | 1 + .../ghostlight-persona-projection/Cargo.toml | 1 + .../ghostlight-persona-projection/src/lib.rs | 127 +++++++++++++----- 4 files changed, 102 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d723b98..892fe21 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -724,6 +724,7 @@ version = "0.1.0" dependencies = [ "serde", "sha2", + "unicode-segmentation", ] [[package]] @@ -2136,6 +2137,12 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + [[package]] name = "universal-hash" version = "0.5.1" diff --git a/Cargo.toml b/Cargo.toml index 76d2879..7bfeb7e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,5 +36,6 @@ tokio = { version = "1", features = ["full"] } tower-http = { version = "0.6", features = ["fs", "trace"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } +unicode-segmentation = "1" uuid = { version = "1", features = ["v4", "serde"] } zeroize = "1.8" diff --git a/crates/ghostlight-persona-projection/Cargo.toml b/crates/ghostlight-persona-projection/Cargo.toml index 7918732..92790a2 100644 --- a/crates/ghostlight-persona-projection/Cargo.toml +++ b/crates/ghostlight-persona-projection/Cargo.toml @@ -7,4 +7,5 @@ license.workspace = true [dependencies] serde.workspace = true sha2.workspace = true +unicode-segmentation.workspace = true diff --git a/crates/ghostlight-persona-projection/src/lib.rs b/crates/ghostlight-persona-projection/src/lib.rs index bd7a840..1af8676 100644 --- a/crates/ghostlight-persona-projection/src/lib.rs +++ b/crates/ghostlight-persona-projection/src/lib.rs @@ -1,5 +1,6 @@ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; +use unicode_segmentation::UnicodeSegmentation; pub const MEMBRANE_SCHEMA: &str = "ghostlight.persona_projection_membrane.v1"; pub const COGNITION_CONTROLLER_SCHEMA: &str = "ghostlight.decision_controller.v1"; @@ -340,29 +341,6 @@ impl PersonaTurn { } } -/// Whether `byte` is a valid left edge for a quoted span: the start of -/// `source`, or the character immediately before it is not -/// Unicode-alphanumeric. `byte` must itself be a valid char boundary, which -/// every caller here gets for free from `match_indices`. -fn starts_on_word_boundary(source: &str, byte: usize) -> bool { - byte == 0 - || !source[..byte] - .chars() - .next_back() - .is_some_and(char::is_alphanumeric) -} - -/// The mirror of [`starts_on_word_boundary`] for the right edge: the end of -/// `source`, or the character immediately after it is not -/// Unicode-alphanumeric. -fn ends_on_word_boundary(source: &str, byte: usize) -> bool { - byte == source.len() - || !source[byte..] - .chars() - .next() - .is_some_and(char::is_alphanumeric) -} - /// An exact UTF-8 byte range in the preserved Persona source prose. #[derive(Clone, Debug, PartialEq, Eq, Serialize)] pub struct SourceSpan { @@ -383,24 +361,30 @@ impl SourceSpan { }) } - /// The first occurrence of `quote` in `source` that lands on a word - /// boundary at both ends. The model quotes words; the harness counts - /// bytes. A model that counts bytes itself cut three of eight utterances - /// mid-word on the first Claude road run — and a raw substring `find` - /// has the same defect the other way: "I can" is a byte-exact match - /// inside "I cannot go", but it is not the word the model claims to be - /// quoting. Each end must be the edge of `source` or sit next to a - /// non-alphanumeric (Unicode-aware) character; a first occurrence that - /// fails either edge is skipped in favor of a later one. + /// The first occurrence of `quote` in `source` that lands on a Unicode + /// word boundary (UAX #29) at both ends. The model quotes words; the + /// harness counts bytes. A model that counts bytes itself cut three of + /// eight utterances mid-word on the first Claude road run — and a raw + /// substring `find` has the same defect the other way: "I can" is a + /// byte-exact match inside "I cannot go", but it is not the word the + /// model claims to be quoting. Each end must be one of the boundary + /// offsets `unicode-segmentation` reports for `source`, or `source.len()`; + /// a first occurrence that fails either edge is skipped in favor of a + /// later one. pub fn locate(source: &str, quote: &str) -> Option { if quote.is_empty() { return None; } + let mut boundaries: std::collections::HashSet = source + .split_word_bound_indices() + .map(|(byte, _)| byte) + .collect(); + boundaries.insert(source.len()); source .match_indices(quote) .find(|(start_byte, matched)| { - starts_on_word_boundary(source, *start_byte) - && ends_on_word_boundary(source, start_byte + matched.len()) + boundaries.contains(start_byte) + && boundaries.contains(&(start_byte + matched.len())) }) .map(|(start_byte, matched)| Self { start_byte, @@ -803,6 +787,81 @@ mod tests { assert!(located.is_exact_in(source)); } + /// PA.f78: the hand-written `is_alphanumeric`-neighbour rule accepted "I + /// can" inside "I can't go" because an ASCII apostrophe does not read as + /// alphanumeric — but UAX #29 keeps a MidLetter apostrophe glued to the + /// letters on both sides, so "can't" is one word and "I can" must be + /// refused. + #[test] + fn a_quote_split_by_an_apostrophe_contraction_is_refused() { + assert_eq!(SourceSpan::locate("I can't go", "I can"), None); + } + + /// The same rule under the curly apostrophe (U+2019), which the hand rule + /// also treated as non-alphanumeric and therefore wrongly accepted. + #[test] + fn a_quote_split_by_a_curly_apostrophe_contraction_is_refused() { + assert_eq!(SourceSpan::locate("I can\u{2019}t go", "I can"), None); + } + + /// A combining acute accent (U+0301) after "e" is a word-internal Extend + /// character under UAX #29: it never introduces a boundary, so "the + /// cafe" ending before the mark is not a valid span even though the mark + /// itself is not alphanumeric. + #[test] + fn a_quote_ending_before_a_combining_mark_is_refused() { + let source = "the cafe\u{0301} is open"; + assert_eq!(SourceSpan::locate(source, "the cafe"), None); + } + + /// An underscore is ExtendNumLet under UAX #29, so "run_fast" is one + /// word and "run" alone does not end on a boundary. + #[test] + fn a_quote_ending_before_an_underscore_is_refused() { + assert_eq!(SourceSpan::locate("run_fast", "run"), None); + } + + /// CJK ideographs carry no spaces and are not `char::is_alphanumeric` + /// exceptions under the old hand rule's assumptions, but UAX #29 breaks + /// between adjacent ideographs, so each one is its own word and a + /// leading two-character quote must be accepted. + #[test] + fn a_cjk_quote_with_no_surrounding_whitespace_is_accepted() { + let source = "我不能去"; + let located = SourceSpan::locate(source, "我不").expect("ideographs break individually"); + assert_eq!(located.start_byte(), 0); + assert_eq!(located.verbatim(), "我不"); + } + + /// A leading space is itself a word-boundary segment, so a quote that + /// starts mid-source with its own leading space is still boundary-valid. + #[test] + fn a_quote_with_a_leading_space_is_accepted() { + let source = "I cannot go"; + let located = SourceSpan::locate(source, " cannot").expect("a leading-space quote locates"); + assert_eq!(located.verbatim(), " cannot"); + } + + /// A hyphen is not glued to its neighbours under UAX #29 (it is not one + /// of the Mid* classes), so it is itself a word boundary and "re" out of + /// "re-enter" is accepted. + #[test] + fn a_quote_ending_before_a_hyphen_is_accepted() { + let source = "please re-enter the code"; + let located = SourceSpan::locate(source, "re").expect("a hyphen-bounded quote locates"); + assert_eq!(located.verbatim(), "re"); + } + + /// "Stop." in "Stop.Now": a full stop between two letters is MidNumLet + /// under UAX #29, so the crate glues the whole run into one word and + /// this quote is refused. Recorded rather than asserted as an + /// independently-motivated rule: the crate's UAX #29 behavior is the + /// authority here, not a special case carved out for this input. + #[test] + fn a_quote_ending_before_a_period_between_letters_reports_the_crate_result() { + assert_eq!(SourceSpan::locate("Stop.Now", "Stop."), None); + } + #[test] fn persisted_exact_turn_rehydrates_with_the_same_receipt() { let recorded = turn("I tell Mara, \"The western brace is giving way.\"");