Use UAX #29 word boundaries in SourceSpan::locate (PA.f78)

The hand-written is_alphanumeric neighbour rule in SourceSpan::locate got
several boundary cases wrong: it accepted "I can" inside "I can't go" (an
apostrophe is not alphanumeric, so it read as a boundary even though it
binds the contraction into one word), accepted "the cafe" ending right
before a combining acute mark, accepted "run" inside "run_fast", and
falsely refused every CJK quote because every ideograph counts as
alphanumeric and CJK prose carries no spaces.

Replace the rule with real UAX #29 word segmentation from the
unicode-segmentation crate: collect the boundary byte offsets
split_word_bound_indices() reports for the source, plus source.len(), and
accept an occurrence only when both its start and end land in that set.
Delete starts_on_word_boundary/ends_on_word_boundary; the crate is the
authority now, not a hand rule.

unicode-segmentation is pinned to 1.x in the workspace
[workspace.dependencies] table (Cargo.lock gains only that one package)
and added to ghostlight-persona-projection's own Cargo.toml.

Verified against UAX #29 with new tests: contractions (straight and curly
apostrophe) are refused, a quote ending before a combining mark is
refused, an underscore-joined word is refused, a leading-CJK two-character
quote is accepted, a quote with its own leading space is accepted, a
hyphen-bounded quote is accepted, and "Stop." in "Stop.Now" is refused
(the crate glues a MidNumLet full stop between two letters into one word,
so this is the crate's called shot, not a special case carved into the
rule).

Callers (Interpreter capture, Dungeon's span_is_exact) are unchanged; the
lib and Dungeon bin suites both still pass in full (618 lib tests with the
one known PA.f77 flake, 63 Dungeon bin tests).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

Cargo.lock

@@ -724,6 +724,7 @@ version = "0.1.0"
 dependencies = [
  "serde",
  "sha2",
+ "unicode-segmentation",
 ]
 
 [[package]]
@@ -2136,6 +2137,12 @@ version = "1.0.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
 
+[[package]]
+name = "unicode-segmentation"
+version = "1.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
+
 [[package]]
 name = "universal-hash"
 version = "0.5.1"

Cargo.toml

@@ -36,5 +36,6 @@ tokio = { version = "1", features = ["full"] }
 tower-http = { version = "0.6", features = ["fs", "trace"] }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+unicode-segmentation = "1"
 uuid = { version = "1", features = ["v4", "serde"] }
 zeroize = "1.8"

crates/ghostlight-persona-projection/Cargo.toml

@@ -7,4 +7,5 @@ license.workspace = true
 [dependencies]
 serde.workspace = true
 sha2.workspace = true
+unicode-segmentation.workspace = true
 

crates/ghostlight-persona-projection/src/lib.rs

@@ -1,5 +1,6 @@
 use serde::{Deserialize, Serialize};
 use sha2::{Digest, Sha256};
+use unicode_segmentation::UnicodeSegmentation;
 
 pub const MEMBRANE_SCHEMA: &str = "ghostlight.persona_projection_membrane.v1";
 pub const COGNITION_CONTROLLER_SCHEMA: &str = "ghostlight.decision_controller.v1";
@@ -340,29 +341,6 @@ impl PersonaTurn {
     }
 }
 
-/// Whether `byte` is a valid left edge for a quoted span: the start of
-/// `source`, or the character immediately before it is not
-/// Unicode-alphanumeric. `byte` must itself be a valid char boundary, which
-/// every caller here gets for free from `match_indices`.
-fn starts_on_word_boundary(source: &str, byte: usize) -> bool {
-    byte == 0
-        || !source[..byte]
-            .chars()
-            .next_back()
-            .is_some_and(char::is_alphanumeric)
-}
-
-/// The mirror of [`starts_on_word_boundary`] for the right edge: the end of
-/// `source`, or the character immediately after it is not
-/// Unicode-alphanumeric.
-fn ends_on_word_boundary(source: &str, byte: usize) -> bool {
-    byte == source.len()
-        || !source[byte..]
-            .chars()
-            .next()
-            .is_some_and(char::is_alphanumeric)
-}
-
 /// An exact UTF-8 byte range in the preserved Persona source prose.
 #[derive(Clone, Debug, PartialEq, Eq, Serialize)]
 pub struct SourceSpan {
@@ -383,24 +361,30 @@ impl SourceSpan {
         })
     }
 
-    /// The first occurrence of `quote` in `source` that lands on a word
-    /// boundary at both ends. The model quotes words; the harness counts
-    /// bytes. A model that counts bytes itself cut three of eight utterances
-    /// mid-word on the first Claude road run — and a raw substring `find`
-    /// has the same defect the other way: "I can" is a byte-exact match
-    /// inside "I cannot go", but it is not the word the model claims to be
-    /// quoting. Each end must be the edge of `source` or sit next to a
-    /// non-alphanumeric (Unicode-aware) character; a first occurrence that
-    /// fails either edge is skipped in favor of a later one.
+    /// The first occurrence of `quote` in `source` that lands on a Unicode
+    /// word boundary (UAX #29) at both ends. The model quotes words; the
+    /// harness counts bytes. A model that counts bytes itself cut three of
+    /// eight utterances mid-word on the first Claude road run — and a raw
+    /// substring `find` has the same defect the other way: "I can" is a
+    /// byte-exact match inside "I cannot go", but it is not the word the
+    /// model claims to be quoting. Each end must be one of the boundary
+    /// offsets `unicode-segmentation` reports for `source`, or `source.len()`;
+    /// a first occurrence that fails either edge is skipped in favor of a
+    /// later one.
     pub fn locate(source: &str, quote: &str) -> Option<Self> {
         if quote.is_empty() {
             return None;
         }
+        let mut boundaries: std::collections::HashSet<usize> = source
+            .split_word_bound_indices()
+            .map(|(byte, _)| byte)
+            .collect();
+        boundaries.insert(source.len());
         source
             .match_indices(quote)
             .find(|(start_byte, matched)| {
-                starts_on_word_boundary(source, *start_byte)
-                    && ends_on_word_boundary(source, start_byte + matched.len())
+                boundaries.contains(start_byte)
+                    && boundaries.contains(&(start_byte + matched.len()))
             })
             .map(|(start_byte, matched)| Self {
                 start_byte,
@@ -803,6 +787,81 @@ mod tests {
         assert!(located.is_exact_in(source));
     }
 
+    /// PA.f78: the hand-written `is_alphanumeric`-neighbour rule accepted "I
+    /// can" inside "I can't go" because an ASCII apostrophe does not read as
+    /// alphanumeric — but UAX #29 keeps a MidLetter apostrophe glued to the
+    /// letters on both sides, so "can't" is one word and "I can" must be
+    /// refused.
+    #[test]
+    fn a_quote_split_by_an_apostrophe_contraction_is_refused() {
+        assert_eq!(SourceSpan::locate("I can't go", "I can"), None);
+    }
+
+    /// The same rule under the curly apostrophe (U+2019), which the hand rule
+    /// also treated as non-alphanumeric and therefore wrongly accepted.
+    #[test]
+    fn a_quote_split_by_a_curly_apostrophe_contraction_is_refused() {
+        assert_eq!(SourceSpan::locate("I can\u{2019}t go", "I can"), None);
+    }
+
+    /// A combining acute accent (U+0301) after "e" is a word-internal Extend
+    /// character under UAX #29: it never introduces a boundary, so "the
+    /// cafe" ending before the mark is not a valid span even though the mark
+    /// itself is not alphanumeric.
+    #[test]
+    fn a_quote_ending_before_a_combining_mark_is_refused() {
+        let source = "the cafe\u{0301} is open";
+        assert_eq!(SourceSpan::locate(source, "the cafe"), None);
+    }
+
+    /// An underscore is ExtendNumLet under UAX #29, so "run_fast" is one
+    /// word and "run" alone does not end on a boundary.
+    #[test]
+    fn a_quote_ending_before_an_underscore_is_refused() {
+        assert_eq!(SourceSpan::locate("run_fast", "run"), None);
+    }
+
+    /// CJK ideographs carry no spaces and are not `char::is_alphanumeric`
+    /// exceptions under the old hand rule's assumptions, but UAX #29 breaks
+    /// between adjacent ideographs, so each one is its own word and a
+    /// leading two-character quote must be accepted.
+    #[test]
+    fn a_cjk_quote_with_no_surrounding_whitespace_is_accepted() {
+        let source = "我不能去";
+        let located = SourceSpan::locate(source, "我不").expect("ideographs break individually");
+        assert_eq!(located.start_byte(), 0);
+        assert_eq!(located.verbatim(), "我不");
+    }
+
+    /// A leading space is itself a word-boundary segment, so a quote that
+    /// starts mid-source with its own leading space is still boundary-valid.
+    #[test]
+    fn a_quote_with_a_leading_space_is_accepted() {
+        let source = "I cannot go";
+        let located = SourceSpan::locate(source, " cannot").expect("a leading-space quote locates");
+        assert_eq!(located.verbatim(), " cannot");
+    }
+
+    /// A hyphen is not glued to its neighbours under UAX #29 (it is not one
+    /// of the Mid* classes), so it is itself a word boundary and "re" out of
+    /// "re-enter" is accepted.
+    #[test]
+    fn a_quote_ending_before_a_hyphen_is_accepted() {
+        let source = "please re-enter the code";
+        let located = SourceSpan::locate(source, "re").expect("a hyphen-bounded quote locates");
+        assert_eq!(located.verbatim(), "re");
+    }
+
+    /// "Stop." in "Stop.Now": a full stop between two letters is MidNumLet
+    /// under UAX #29, so the crate glues the whole run into one word and
+    /// this quote is refused. Recorded rather than asserted as an
+    /// independently-motivated rule: the crate's UAX #29 behavior is the
+    /// authority here, not a special case carved out for this input.
+    #[test]
+    fn a_quote_ending_before_a_period_between_letters_reports_the_crate_result() {
+        assert_eq!(SourceSpan::locate("Stop.Now", "Stop."), None);
+    }
+
     #[test]
     fn persisted_exact_turn_rehydrates_with_the_same_receipt() {
         let recorded = turn("I tell Mara, \"The western brace is giving way.\"");