From 81572b1a5425e76ac89fc9da20faa0d0581582f5 Mon Sep 17 00:00:00 2001 From: Weksa Date: Sat, 7 Nov 2026 22:04:59 +0000 Subject: [PATCH 1/4] Packet carries an addressee-consent fact (schema v5, V14, V15) Discourse.addressee_consent names the predication the addressee asked for or agreed to. V4 checks it resolves, V14 needs an addressee and a predication, V15 allows only Authored or GameTelemetry sources. The English lowerer renders no consent and refuses a packet carrying one. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/lower.rs | 3 + crates/weksa/src/packet.rs | 10 +++- crates/weksa/src/validate.rs | 41 ++++++++++++- crates/weksa/tests/common/mod.rs | 1 + crates/weksa/tests/entities.rs | 2 +- crates/weksa/tests/lower.rs | 2 + crates/weksa/tests/stores.rs | 4 +- crates/weksa/tests/validate.rs | 59 +++++++++++++++++++ docs/interlingua-standard.md | 16 +++-- .../weksa.interlingua_packet.schema.json | 11 ++++ 10 files changed, 139 insertions(+), 10 deletions(-) diff --git a/crates/weksa/src/lower.rs b/crates/weksa/src/lower.rs index 43a8752..8f7b856 100644 --- a/crates/weksa/src/lower.rs +++ b/crates/weksa/src/lower.rs @@ -139,6 +139,9 @@ pub fn lower( { return unrendered("features"); } + if input.discourse().addressee_consent.is_some() { + return unrendered("addressee consent"); + } if !input.relations().is_empty() { return unrendered("relations"); } diff --git a/crates/weksa/src/packet.rs b/crates/weksa/src/packet.rs index f2f06de..9d6c582 100644 --- a/crates/weksa/src/packet.rs +++ b/crates/weksa/src/packet.rs @@ -16,7 +16,7 @@ pub use crate::inventory::InvId; use crate::inventory::OptionLabel; /// Bumped by a commit that changes these types; the published schema moves with it. -pub const SCHEMA_VERSION: u32 = 4; +pub const SCHEMA_VERSION: u32 = 5; /// Id of a referent or predication within one packet: `n`, `k` a canonical /// decimal 1..=9999. @@ -261,6 +261,13 @@ pub struct Discourse { /// Speech-act id. pub speech_act: Option, pub addressee: Option, + /// The predication the addressee asked for or agreed to. A fact about what + /// the addressee did, never about how the line sounds, so it is no feature: + /// only a source that saw the request may set it (V15), it needs an + /// addressee (V14) and names a predication (V14), and a lowering that + /// speaks for the addressee's request ("you asked her to") reads it here + /// and refuses without it. + pub addressee_consent: Option, pub features: Vec, } @@ -385,6 +392,7 @@ pub fn proof_packet() -> Packet { discourse: Discourse { speech_act: Some(id("speech_act:5")), addressee: Some(node("n1")), + addressee_consent: None, features: vec![], }, referents: vec![ diff --git a/crates/weksa/src/validate.rs b/crates/weksa/src/validate.rs index 6630430..e8aaf2f 100644 --- a/crates/weksa/src/validate.rs +++ b/crates/weksa/src/validate.rs @@ -3,7 +3,7 @@ //! returns `Ok(())` or the refusals. [`LoweringInput`], the only thing a //! lowerer sees, comes from `PacketStore::admit` alone, which builds it through //! the crate-private `admissible`. The stance is checked like every feature -//! list, in slot `Stance`. The rules are V1..V8 and V10..V13: the shape of an id +//! list, in slot `Stance`. The rules are V1..V8 and V10..V15: the shape of an id //! or a grade is its type's parse, not a rule, so V9 is retired and not //! renumbered. V13 guards the one other way features reach a lowering view: //! [`LoweringInput::with_scene`]. @@ -29,7 +29,7 @@ use std::fmt; use crate::inventory::{Family, InvId, Inventory, InventoryEntry, Slot, ValueKind}; use crate::packet::{ Constraints, Context, Discourse, Extension, FeatValue, FeatureValue, NodeId, Packet, - PacketKind, Predication, Referent, Relation, SCHEMA_VERSION, Stance, + PacketKind, Predication, ProvenanceSource, Referent, Relation, SCHEMA_VERSION, Stance, }; /// A broken rule, named. @@ -71,6 +71,15 @@ pub enum Refusal { /// state the speaker's relation to the addressee, so with none they /// describe nobody. V13SceneWithoutAddressee, + /// V14: an addressee consent that no addressee can have given, or that + /// names a node that is not a predication. It is what the addressee asked + /// for or agreed to, an act, so a referent cannot be its object. + V14ConsentWithoutAddressee, + V14ConsentNotAPredication { id: NodeId }, + /// V15: an addressee consent set by a source that only read wording. + /// A decomposition of text, or imported corpus text, sees "you asked her + /// to" as words; only authoring or game telemetry saw the request. + V15ConsentSource { source: ProvenanceSource }, } impl Refusal { @@ -90,6 +99,10 @@ impl Refusal { | Refusal::V11DuplicateRole { .. } => "V11", Refusal::V12TooManyNodes { .. } => "V12", Refusal::V13SceneWithoutAddressee => "V13", + Refusal::V14ConsentWithoutAddressee | Refusal::V14ConsentNotAPredication { .. } => { + "V14" + } + Refusal::V15ConsentSource { .. } => "V15", } } } @@ -156,6 +169,7 @@ pub(crate) fn admissible( let nodes = check_nodes(packet, &mut out); check_refs(packet, &nodes, &mut out); check_repeats(packet, &mut out); + check_consent(packet, &nodes, &mut out); if inventory.has_revision(packet.inventory_rev) { Inventoried { inv: inventory, @@ -208,6 +222,9 @@ fn check_refs(packet: &Packet, nodes: &BTreeSet, out: &mut Vec) if let Some(a) = &packet.discourse.addressee { need(a, "addressee"); } + if let Some(c) = &packet.discourse.addressee_consent { + need(c, "addressee consent"); + } for (list, at) in node_lists(packet) { for id in list { need(id, at); @@ -215,6 +232,26 @@ fn check_refs(packet: &Packet, nodes: &BTreeSet, out: &mut Vec) } } +/// V14 and V15: the addressee's consent is a typed fact with one origin. A +/// dangling id is V4's, so V14 judges only a node that exists. +fn check_consent(packet: &Packet, nodes: &BTreeSet, out: &mut Vec) { + let Some(consent) = &packet.discourse.addressee_consent else { + return; + }; + if packet.discourse.addressee.is_none() { + out.push(Refusal::V14ConsentWithoutAddressee); + } + if nodes.contains(consent) && !packet.predications.iter().any(|p| &p.id == consent) { + out.push(Refusal::V14ConsentNotAPredication { + id: consent.clone(), + }); + } + match packet.provenance.source { + ProvenanceSource::Authored | ProvenanceSource::GameTelemetry => {} + source => out.push(Refusal::V15ConsentSource { source }), + } +} + fn node_lists(packet: &Packet) -> [(&Vec, &'static str); 3] { [ (&packet.context.participants, "participant"), diff --git a/crates/weksa/tests/common/mod.rs b/crates/weksa/tests/common/mod.rs index d866904..60ba544 100644 --- a/crates/weksa/tests/common/mod.rs +++ b/crates/weksa/tests/common/mod.rs @@ -266,6 +266,7 @@ pub fn good() -> Packet { discourse: Discourse { speech_act: Some(iv(WARN)), addressee: Some(nd("n1")), + addressee_consent: None, features: vec![FeatureValue { feature: iv(HEDGE), value: FeatValue::Grade(grade(300)), diff --git a/crates/weksa/tests/entities.rs b/crates/weksa/tests/entities.rs index 5493943..5f1f7cf 100644 --- a/crates/weksa/tests/entities.rs +++ b/crates/weksa/tests/entities.rs @@ -397,7 +397,7 @@ fn packet_entities_resolve() { #[test] fn packet_schema_v2() { - assert_eq!(SCHEMA_VERSION, 4); + assert_eq!(SCHEMA_VERSION, 5); let mut p = good(); p.schema_version = 2; let Err(refused) = validate(&p, &inventory()) else { diff --git a/crates/weksa/tests/lower.rs b/crates/weksa/tests/lower.rs index 4f1691c..1d8301d 100644 --- a/crates/weksa/tests/lower.rs +++ b/crates/weksa/tests/lower.rs @@ -142,6 +142,7 @@ fn a_packet_carrying_what_the_lowerer_does_not_render_is_refused() { filler, }) }; + let consent = |p: &mut Packet| p.discourse.addressee_consent = Some(nd("n5")); let lone_referent = |p: &mut Packet| { let mut r = p.referents[0].clone(); r.id = nd("n99"); @@ -150,6 +151,7 @@ fn a_packet_carrying_what_the_lowerer_does_not_render_is_refused() { for (what, edit) in [ ("features", &discourse as &dyn Fn(&mut Packet)), ("features", &predication), + ("addressee consent", &consent), ("relations", &relation), ("extensions", &extension), ("no predication", &no_predication), diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs index 07cbe0c..cf5ba4a 100644 --- a/crates/weksa/tests/stores.rs +++ b/crates/weksa/tests/stores.rs @@ -989,14 +989,14 @@ fn packet_admission_goes_through_the_validator() { let mut w = world(); let mut store = PacketStore::open(&w.dir.path().join("p.cc")).unwrap(); let mut bad = good(); - bad.schema_version = 5; + bad.schema_version = 6; match store.admit(bad, &w.inventory, &w.corpus, &w.entities) { Err(e @ StoreError::Invalid(_)) => { assert!(e.to_string().contains("V1"), "{e}"); let StoreError::Invalid(r) = e else { unreachable!() }; - assert_eq!(r, vec![Refusal::V1SchemaVersion { found: 5 }]); + assert_eq!(r, vec![Refusal::V1SchemaVersion { found: 6 }]); } other => panic!("{other:?}"), } diff --git a/crates/weksa/tests/validate.rs b/crates/weksa/tests/validate.rs index 3453376..aaa87bd 100644 --- a/crates/weksa/tests/validate.rs +++ b/crates/weksa/tests/validate.rs @@ -683,3 +683,62 @@ fn v11_role_repeat_in_one_predication_refused() { ok.predications.push(second); assert_eq!(refusals(&ok), vec![]); } + +// The addressee's consent: which predication the addressee asked for, set only +// by a source that saw the request. + +fn consenting() -> Packet { + let mut p = good(); + p.discourse.addressee_consent = Some(nd("n2")); + p +} + +#[test] +fn consent_to_a_predication_by_an_authored_or_telemetry_source_passes() { + assert_eq!(refusals(&consenting()), vec![]); + let mut p = consenting(); + p.provenance.source = ProvenanceSource::GameTelemetry; + assert_eq!(refusals(&p), vec![]); +} + +#[test] +fn consent_needs_an_addressee() { + let mut p = consenting(); + p.discourse.addressee = None; + assert_eq!(refusals(&p), vec![Refusal::V14ConsentWithoutAddressee]); +} + +#[test] +fn consent_names_a_predication_not_a_referent() { + let mut p = consenting(); + p.discourse.addressee_consent = Some(nd("n1")); + assert_eq!( + refusals(&p), + vec![Refusal::V14ConsentNotAPredication { id: nd("n1") }] + ); +} + +#[test] +fn consent_to_nothing_is_dangling_only() { + let mut p = consenting(); + p.discourse.addressee_consent = Some(nd("n99")); + assert_eq!( + refusals(&p), + vec![Refusal::V4DanglingRef { + id: nd("n99"), + at: "addressee consent" + }] + ); +} + +#[test] +fn a_source_that_only_read_wording_cannot_set_consent() { + for source in [ + ProvenanceSource::ModelDecomposition, + ProvenanceSource::ImportedCorpus, + ] { + let mut p = consenting(); + p.provenance.source = source; + assert_eq!(refusals(&p), vec![Refusal::V15ConsentSource { source }]); + } +} diff --git a/docs/interlingua-standard.md b/docs/interlingua-standard.md index 6a835dd..ca392d1 100644 --- a/docs/interlingua-standard.md +++ b/docs/interlingua-standard.md @@ -120,7 +120,7 @@ Packet id, schema_version, inventory_rev, kind provenance { source, corpus_item, encoder, confidence } context { setting, medium, participants, audience } - discourse { speech_act, addressee, features } + discourse { speech_act, addressee, addressee_consent, features } referents [ { id, kind, concept, entity, features } ] predications [ { id, concept, frame, roles: [ { role, filler } ], features } ] relations [ { relation, from, to, features } ] @@ -144,8 +144,8 @@ names the rule. - V1: `schema_version` is the version these types define. - V2: at least one of `referents`, `predications`, `relations` is present. - V3: a node id appears once across referents and predications. -- V4: every role filler, relation end, addressee, participant, audience member - and preserved node names a node. +- V4: every role filler, relation end, addressee, addressee consent, + participant, audience member and preserved node names a node. - V5: every inventory id resolves at the packet's `inventory_rev`: introduced at or before it, and not retired at or before it. - V6: an inventory id is of the family its slot takes. @@ -159,6 +159,10 @@ names the rule. predication's roles. - V12: a packet has at most 64 referents, predications and relations together (a calibration constant, capacity and not meaning). +- V14: an `addressee_consent` needs an `addressee` and names a predication. +- V15: an `addressee_consent` is set only by provenance `Authored` or + `GameTelemetry`; a decomposition of text or imported corpus text cannot mint + it. V9 is retired and not renumbered: the form of a key or a grade is its type, and an extension key's family is V6. `PacketStore::admit` returns the lowering view of a @@ -189,7 +193,11 @@ ids. Why the packet is said: `speech_act` is a speech-act id, `addressee` a node id, and `features` carry source-side nuance on content (hedge, certainty, irony and whatever the inventory grows). This is source-side communicative pressure, not -target-language register. +target-language register. `addressee_consent` is no feature: it names the +predication the addressee asked for or agreed to, a fact about what the +addressee did. The English lowerer renders no consent and refuses a packet that +carries one; a row that speaks for the request ("you asked her to") reads it +and refuses without it. ### `stance` diff --git a/schemas/cultnet/weksa.interlingua_packet.schema.json b/schemas/cultnet/weksa.interlingua_packet.schema.json index d839203..97442f9 100644 --- a/schemas/cultnet/weksa.interlingua_packet.schema.json +++ b/schemas/cultnet/weksa.interlingua_packet.schema.json @@ -164,6 +164,17 @@ } ] }, + "addressee_consent": { + "description": "The predication the addressee asked for or agreed to. A fact about what\nthe addressee did, never about how the line sounds, so it is no feature:\nonly a source that saw the request may set it (V15), it needs an\naddressee (V14) and names a predication (V14), and a lowering that\nspeaks for the addressee's request (\"you asked her to\") reads it here\nand refuses without it.", + "anyOf": [ + { + "$ref": "#/$defs/NodeId" + }, + { + "type": "null" + } + ] + }, "features": { "type": "array", "items": { From 5e2a4a0ebbe67d7f29dea37abe0f2070f8e54bb7 Mon Sep 17 00:00:00 2001 From: Weksa Date: Mon, 9 Nov 2026 07:47:17 +0000 Subject: [PATCH 2/4] Scorer coverage: words break at any non-word character, unspaced scripts read by run; near-miss chrF pairs per script Soul scorer s3 Lows. words.rs no longer lists separators (_ and .) or defers to UAX #29, which splits Han, kana and Thai per character and made honest single-word glosses read as an n>=2 leak. A word is a run of letters, digits, marks and ZWNJ/ZWJ; a run of Han, kana, Thai, Lao, Khmer or Burmese is one word. Tests pin separators, marks, joiners, NFC, every range edge, and five sacrebleu 2.4.3 near-miss pairs (Ko, Tr, Fi, Th vowel-strip, Ja). 40 hand mutants killed. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/score/words.rs | 106 +++++++++++----- crates/weksa/tests/score.rs | 152 +++++++++++++++++++++++ docs/targets/interlingua-lowering-map.md | 9 +- 3 files changed, 236 insertions(+), 31 deletions(-) diff --git a/crates/weksa/src/score/words.rs b/crates/weksa/src/score/words.rs index 58e7c1f..4909ee9 100644 --- a/crates/weksa/src/score/words.rs +++ b/crates/weksa/src/score/words.rs @@ -1,44 +1,94 @@ -//! The one way the scorer reads words: UAX #29 word segmentation, then NFC and -//! full Unicode case folding. The novel-coverage diagnostic reads text through -//! here, so every script is treated by one rule. +//! The one way the scorer reads words, then NFC and full Unicode case folding. +//! The novel-coverage diagnostic reads text through here, so every script is +//! treated by one rule. +//! +//! A word is a run of word characters: alphanumerics, combining marks and the +//! joiners ZWNJ and ZWJ. Any other character ends it, so `kick:the:bucket`, +//! `kick·the·bucket`, `kick_the_bucket` and `kick'the'bucket` read as +//! `kick the bucket`, as does `kickTheBucket` (a lowercase-to-uppercase +//! transition also ends a word). +//! +//! Han, kana, Thai, Lao, Khmer and Burmese write no word boundaries, and the +//! scorer holds no dictionary to find them. A run of those characters is one +//! word, never one word per character: per-character words would make every +//! honest single-word gloss share bigrams with a held-out sentence, which is +//! the leak signature. A change between such a run and any other script also +//! ends a word. A leak in these scripts shows at n = 1 and in chrF; a +//! dictionary segmenter is the ja-JP panel's to add. use caseless::default_case_fold_str; use unicode_normalization::UnicodeNormalization; -use unicode_segmentation::UnicodeSegmentation; +use unicode_normalization::char::is_combining_mark; /// NFC, then full default case folding. pub(super) fn normalise(s: &str) -> String { default_case_fold_str(&s.nfc().collect::()) } -/// The pieces of one segment: split at `_` and `.` (which are dropped) and -/// at every lowercase-to-uppercase transition, so `kick_the_bucket`, -/// `kick.the.bucket` and `kickTheBucket` read as `kick the bucket`. -fn pieces(segment: &str) -> Vec<&str> { +fn is_joiner(c: char) -> bool { + matches!(c, '\u{200c}' | '\u{200d}') +} + +/// A character that belongs inside a word. Marks must stay: Thai vowels and +/// tone marks, Hindi vowel signs and Hangul jamo are not alphanumeric. +fn is_word_char(c: char) -> bool { + c.is_alphanumeric() || is_combining_mark(c) || is_joiner(c) +} + +/// Han, kana, Thai, Lao, Khmer and Burmese: scripts written without spaces. +fn is_unspaced(c: char) -> bool { + matches!( + c, + '\u{3005}' + | '\u{3007}' + | '\u{3021}'..='\u{3029}' + | '\u{3038}'..='\u{303b}' + | '\u{3041}'..='\u{30ff}' + | '\u{3400}'..='\u{4dbf}' + | '\u{4e00}'..='\u{9fff}' + | '\u{f900}'..='\u{fad9}' + | '\u{ff66}'..='\u{ff9f}' + | '\u{31f0}'..='\u{31ff}' + | '\u{20000}'..='\u{323af}' + | '\u{0e01}'..='\u{0e5b}' + | '\u{0e81}'..='\u{0edf}' + | '\u{1000}'..='\u{109f}' + | '\u{1780}'..='\u{17ff}' + ) +} + +/// The normalised words of `text` in order; runs with no alphanumeric char +/// (marks alone, say) are dropped. +pub(super) fn words(text: &str) -> Vec { let mut out = Vec::new(); let mut start = 0; let mut previous: Option = None; - for (at, c) in segment.char_indices() { - let separator = c == '_' || c == '.'; - let camel = previous.is_some_and(char::is_lowercase) && c.is_uppercase(); - if separator || camel { - out.push(&segment[start..at]); - start = if separator { at + c.len_utf8() } else { at }; + let end = |from: usize, to: usize, out: &mut Vec| { + let word = &text[from..to]; + if word.chars().any(char::is_alphanumeric) { + out.push(normalise(word)); + } + }; + for (at, c) in text.char_indices() { + if !is_word_char(c) { + end(start, at, &mut out); + start = at + c.len_utf8(); + continue; + } + // a mark or joiner belongs to the character before it + let attached = is_combining_mark(c) || is_joiner(c); + let breaks = !attached + && previous.is_some_and(|p| { + (p.is_lowercase() && c.is_uppercase()) || is_unspaced(p) != is_unspaced(c) + }); + if breaks { + end(start, at, &mut out); + start = at; + } + if !attached { + previous = Some(c); } - previous = Some(c); } - out.push(&segment[start..]); + end(start, text.len(), &mut out); out } - -/// The normalised words of `text` in order: UAX #29 segments containing an -/// alphanumeric char (digits kept), each split by `pieces`, empty or -/// non-alphanumeric pieces dropped. -pub(super) fn words(text: &str) -> Vec { - text.split_word_bounds() - .filter(|segment| segment.chars().any(char::is_alphanumeric)) - .flat_map(pieces) - .filter(|piece| piece.chars().any(char::is_alphanumeric)) - .map(normalise) - .collect() -} diff --git a/crates/weksa/tests/score.rs b/crates/weksa/tests/score.rs index 6920b93..e8c363a 100644 --- a/crates/weksa/tests/score.rs +++ b/crates/weksa/tests/score.rs @@ -690,3 +690,155 @@ fn every_script_alike() { assert_eq!(got.chrf, 100.0, "{text}"); } } + +/// (reference, hypothesis, sacrebleu 2.4.3 sentence_chrf), one near miss per +/// script: the Thai hypothesis is the reference with its vowels and tone mark +/// stripped. +const SCRIPT_PAIRS: [(&str, &str, f64); 5] = [ + ("민수가 왔다", "민수는 왔다", 26.0), + ("Brokkr'ın gemisi geldi", "Brokkr'ın gemisi gitti", 77.7548), + ("Brokkrin laiva tuli", "Brokkrin laiva meni", 73.002), + ("สมชายมาแล้ว", "สมชยมแลว", 21.8989), + ("ブロッカーが来た。", "ブロッカーが行った。", 54.4013), +]; + +#[test] +fn every_script_scores_a_near_miss_as_sacrebleu_does() { + let scorer = Scorer::new(Bag); + for (reference, hypothesis, want) in SCRIPT_PAIRS { + close(score_one(&scorer, reference, hypothesis).chrf, want); + } +} + +#[test] +fn coverage_splits_at_any_non_word_character() { + let items = [ + validation("item:v:1", "mind the hatch before the storm"), + train("item:t:1", UNRELATED), + ]; + let want = read(&["mind the hatch"], &items); + assert_eq!(want, [(5, 3), (5, 2), (4, 1)]); + for joiner in [ + ":", "\u{b7}", "'", "\u{2019}", "\u{ff3f}", "-", "/", "\u{2014}", "|", "+", "\u{ff1a}", + "\u{30fb}", + ] { + let glued = format!("mind{joiner}the{joiner}hatch"); + assert_eq!(read(&[&glued], &items), want, "{joiner:?}"); + } + // a mark, a joiner or a digit is part of its word, not a break + let items = [validation( + "item:v:1", + "\u{e2a}\u{e21}\u{e0a}\u{e32}\u{e22} \u{e21}\u{e32}\u{e41}\u{e25}\u{e49}\u{e27} 0400 \u{628}\u{200c}\u{628}", + )]; + assert_eq!( + read(&["\u{e21}\u{e32}\u{e41}\u{e25}\u{e49}\u{e27}"], &items)[0], + (4, 1) + ); + assert_eq!(read(&["\u{628}\u{200c}\u{628}", "0400"], &items)[0], (4, 2)); + // a joiner keeps its word whole: the minted half is not a held-out word + let items = [validation("item:v:1", "\u{628}\u{200c}\u{62a} \u{62c}")]; + assert_eq!(read(&["\u{628}\u{200c}\u{62a}"], &items)[0], (2, 1)); + // decomposed and composed spellings are one word + let items = [validation("item:v:1", "Cafe\u{301}")]; + assert_eq!(read(&["Caf\u{e9}"], &items)[0], (1, 1)); +} + +#[test] +fn coverage_reads_unspaced_scripts_by_run() { + // Han, kana and Thai write no word boundaries and the scorer holds no + // dictionary, so a run of them is one word, never one word per character. + // Honest single-word glosses then read nothing above n=1, as they do in a + // spaced script. + for (held, glosses) in [ + ( + "\u{5c71}\u{7530}\u{592a}\u{90ce}\u{304c}\u{6765}\u{305f}", + ["\u{5c71}\u{7530}", "\u{592a}\u{90ce}", "\u{6765}\u{305f}"], + ), + ( + "\u{30d6}\u{30ed}\u{30c3}\u{30ab}\u{30fc}\u{304c}\u{6765}\u{305f}\u{3002}", + ["\u{30d6}\u{30ed}", "\u{30ab}\u{30fc}", "\u{6765}"], + ), + ( + "\u{e2a}\u{e21}\u{e0a}\u{e32}\u{e22}\u{e21}\u{e32}\u{e41}\u{e25}\u{e49}\u{e27}", + [ + "\u{e2a}\u{e21}", + "\u{e0a}\u{e32}\u{e22}", + "\u{e41}\u{e25}\u{e49}", + ], + ), + ] { + let items = [validation("item:v:1", held), train("item:t:1", UNRELATED)]; + assert_eq!(read(&glosses, &items), [(1, 0), (0, 0), (0, 0)], "{held}"); + assert_eq!(read(&[held], &items), [(1, 1), (0, 0), (0, 0)], "{held}"); + } + // Delimiters still make words: spaces and punctuation between runs, and a + // change between an unspaced script and a spaced one. + let items = [ + validation( + "item:v:1", + "\u{72ac}\u{304c} \u{8d70}\u{308b}\u{3001}\u{732b}\u{3082} Brokkr\u{5c71}", + ), + train("item:t:1", UNRELATED), + ]; + assert_eq!( + read( + &["\u{72ac}\u{304c} \u{8d70}\u{308b} \u{732b}\u{3082} brokkr \u{5c71}"], + &items + ), + [(5, 5), (4, 4), (3, 3)] + ); +} + +#[test] +fn an_unspaced_script_ends_a_word_beside_any_other_script() { + // the first and last letters or digits of every range the rule names + let unspaced = [ + '\u{3005}', + '\u{3007}', + '\u{3021}', + '\u{3029}', + '\u{3038}', + '\u{303b}', + '\u{3041}', + '\u{30ff}', + '\u{31f0}', + '\u{31ff}', + '\u{3400}', + '\u{4dbf}', + '\u{4e00}', + '\u{9fff}', + '\u{f900}', + '\u{fad9}', + '\u{ff66}', + '\u{ff9f}', + '\u{20000}', + '\u{2a6df}', + '\u{31350}', + '\u{323af}', + '\u{e01}', + '\u{e59}', + '\u{e81}', + '\u{edf}', + '\u{1000}', + '\u{1049}', + '\u{1780}', + '\u{17e9}', + ]; + for c in unspaced { + assert!(c.is_alphanumeric(), "{c:?} is a letter or digit"); + let items = [validation("item:v:1", &format!("ab{c}"))]; + assert_eq!(read(&["ab", &c.to_string()], &items)[0], (2, 2), "{c:?}"); + } + // Hangul, Latin and Cyrillic are spaced scripts: no break between them + for c in ['\u{ac00}', '\u{d7a3}', '\u{e9}', '\u{44f}', '7'] { + let glued = format!("ab{c}"); + let items = [validation("item:v:1", &glued)]; + assert_eq!(read(&[&glued], &items)[0], (1, 1), "{c:?}"); + assert_eq!(read(&["ab"], &items)[0], (1, 0), "{c:?}"); + } + // a combining mark or a joiner after an unspaced character stays on it + let held = "\u{4e00}\u{301}\u{200d}ab"; + let items = [validation("item:v:1", held)]; + assert_eq!(read(&["\u{4e00}\u{301}\u{200d}", "ab"], &items)[0], (2, 2)); + assert_eq!(read(&["\u{4e00}"], &items)[0], (2, 0)); +} diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index c828266..4517320 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -202,9 +202,12 @@ operator-attested annotations alone. Who annotates is question text against held-out items; novel coverage pooled per order, for n = 1, 2 and 3 separately, the covered share of all novel n-grams (word n-grams of a held-out item that occur in no Train item) over a split's items, with the - worst item at n >= 2. Keys split on `_`, `.` and lower-to-upper case - transitions as well as UAX #29 boundaries, so a snake_case `OptionLabel` - reads as words. Honest glosses of single words saturate n = 1 (the 54 seed + worst item at n >= 2. A word is a run of letters, digits, marks and + joiners; any other character, and a lower-to-upper case transition, ends it, + so `kick:the:bucket` and a snake_case `OptionLabel` read as words. Han, kana, + Thai, Lao, Khmer and Burmese write no word boundaries and the scorer holds no + dictionary, so a run of them is one word (per-character words would give + every honest gloss shared bigrams); a leak there shows at n = 1 and in chrF. Honest glosses of single words saturate n = 1 (the 54 seed glosses read max 1.000, s2), so the leak signature is pooled n >= 2. Costs: a live persona mints before it names (one typed write, which also buys coreference and one rendering per language); public-domain intake From 0bf9499ac0bb3477193098f499c0ba856034aab7 Mon Sep 17 00:00:00 2001 From: Weksa Date: Mon, 9 Nov 2026 10:07:52 +0000 Subject: [PATCH 3/4] Lower 'You asked Nibu to stop.' behind addressee consent: request concept and frame:3, consent gate in the lowerer Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/inventory.rs | 15 ++- crates/weksa/src/lower.rs | 80 +++++++++++++++- crates/weksa/tests/lower.rs | 114 +++++++++++++++++++++++ data/interlingua/inventory.cc | Bin 7772 -> 7967 bytes docs/interlingua-standard.md | 7 +- docs/targets/interlingua-lowering-map.md | 2 +- 6 files changed, 209 insertions(+), 9 deletions(-) diff --git a/crates/weksa/src/inventory.rs b/crates/weksa/src/inventory.rs index 405b4ae..c9d69a2 100644 --- a/crates/weksa/src/inventory.rs +++ b/crates/weksa/src/inventory.rs @@ -485,9 +485,20 @@ pub fn proof_revision() -> (InventoryRevision, Vec) { for (family, glosses) in [ ( Family::Concept, - ["endanger", "prevent", "player", "control", "ship_mind"].as_slice(), + [ + "endanger", + "prevent", + "player", + "control", + "ship_mind", + "request", + ] + .as_slice(), + ), + ( + Family::Frame, + ["hazard_warning", "intervention", "request_intervention"].as_slice(), ), - (Family::Frame, ["hazard_warning", "intervention"].as_slice()), ] { for (i, gloss) in glosses.iter().enumerate() { entries.push(InventoryEntry { diff --git a/crates/weksa/src/lower.rs b/crates/weksa/src/lower.rs index 8f7b856..9eb9ec7 100644 --- a/crates/weksa/src/lower.rs +++ b/crates/weksa/src/lower.rs @@ -17,6 +17,14 @@ //! the target language, is refused: the concept word would say a different //! thing than the name the packet cites. //! +//! A request clause ("You asked Nibu to stop.") says what the addressee did, so +//! it is gated on the packet's own record of it: `Discourse.addressee_consent` +//! (set only by authoring or game telemetry, V15) must name that predication, +//! and its agent must be the addressee. Without it the lowerer refuses +//! ([`LowerError::ConsentMissing`], [`LowerError::AskerNotAddressee`]); it does +//! not read wording or guess who asked. A consent naming any other predication +//! is an unrendered slot. +//! //! The table is a temporary owner. Each row speaks for an inventory entry and //! names the gloss it speaks for, so a row cannot outlive a change of what its id //! means. `language-ontology` moves the rows into ontology documents. @@ -65,8 +73,17 @@ const FRAMES: &[Row] = &[ "{agent} may endanger {patient} by touching {instrument}", ), ("frame:2", "intervention", "{agent} will stop {patient}"), + ( + "frame:3", + "request_intervention", + "{agent} asked {patient} to stop", + ), ]; +/// The frame of "{agent} asked {patient} to stop": the one frame that reads +/// the discourse's addressee consent. +const REQUEST_FRAME: &str = "frame:3"; + /// Speech acts: the opener of the first sentence. const SPEECH_ACTS: &[Row] = &[ ("speech_act:1", "assert", ""), @@ -95,6 +112,12 @@ pub enum LowerError { NoPronunciation { entity: EntityId }, /// IPA was requested for a packet that cites no entity. NoEntityReferent, + /// A request predication the packet does not carry the addressee's consent + /// to: no consent, or a consent that names another predication. The + /// lowerer does not guess who asked. + ConsentMissing { predication: NodeId }, + /// A request predication whose agent is not the discourse addressee. + AskerNotAddressee { predication: NodeId }, /// The entity port failed to read. EntityRead { entity: EntityId, why: String }, } @@ -139,9 +162,7 @@ pub fn lower( { return unrendered("features"); } - if input.discourse().addressee_consent.is_some() { - return unrendered("addressee consent"); - } + request_gate(input, inv)?; if !input.relations().is_empty() { return unrendered("relations"); } @@ -188,6 +209,59 @@ pub fn lower( Ok(out) } +/// The one place consent is read. A request clause ([`REQUEST_FRAME`]) says what +/// the addressee did, so it needs the packet's own record of it: the discourse +/// `addressee_consent`, which validation lets only authoring or game telemetry +/// set (V15), must name that predication, and its agent must be the addressee. +/// Nothing here reads wording. A consent naming any other predication is a slot +/// the lowerer does not render. +fn request_gate(input: &LoweringInput, inv: &Inventory) -> Result<(), LowerError> { + let consent = input.discourse().addressee_consent.as_ref(); + let requests = input + .predications() + .iter() + .filter(|p| p.frame.to_string() == REQUEST_FRAME); + for p in requests { + if consent != Some(&p.id) { + return Err(LowerError::ConsentMissing { + predication: p.id.clone(), + }); + } + let agent = inv + .entries() + .find(|e| { + e.family() == Family::Role && e.gloss == "agent" && e.live_at(input.inventory_rev()) + }) + .ok_or_else(|| LowerError::UnknownRole { + frame: p.frame, + role: "agent".to_string(), + })?; + let asker = p + .roles + .iter() + .find(|r| r.role == agent.id) + .map(|r| &r.filler); + if asker.is_none() || asker != input.discourse().addressee.as_ref() { + return Err(LowerError::AskerNotAddressee { + predication: p.id.clone(), + }); + } + } + match consent { + Some(_) + if !input + .predications() + .iter() + .any(|p| p.frame.to_string() == REQUEST_FRAME) => + { + Err(LowerError::Unrendered { + what: "addressee consent", + }) + } + _ => Ok(()), + } +} + /// The ruled IPA of every entity a packet cites, one per entity referent in /// the packet's referent order, as stored: bare, stress-marked, nothing /// derived. A rejected pronunciation is never read. An entity the port does not diff --git a/crates/weksa/tests/lower.rs b/crates/weksa/tests/lower.rs index 1d8301d..89e35ed 100644 --- a/crates/weksa/tests/lower.rs +++ b/crates/weksa/tests/lower.rs @@ -436,3 +436,117 @@ fn ipa_needs_an_entity_the_port_holds_and_a_packet_that_cites_one() { Err(LowerError::NoEntityReferent) ); } + +/// `nibu-proof-001` re-cut as the request: the addressee asked Nibu to stop, +/// one frame:3 predication (n4), consent naming it, and no other claim. +fn asked() -> Packet { + let mut p = proof_packet(); + p.referents.remove(1); + let mut ask = p.predications[1].clone(); + ask.id = nd("n4"); + ask.concept = iv("concept:6"); + ask.frame = iv("frame:3"); + ask.roles = vec![ + weksa::packet::RoleFill { + role: iv("role:1"), + filler: nd("n1"), + }, + weksa::packet::RoleFill { + role: iv("role:2"), + filler: nd("n3"), + }, + ]; + p.predications = vec![ask]; + p.discourse.speech_act = Some(iv("speech_act:1")); + p.discourse.addressee_consent = Some(nd("n4")); + p.constraints.preserve = vec![nd("n1")]; + p +} + +/// The request plus a frame:2 predication (n5) the consent could wrongly name. +fn asked_and_stopped() -> Packet { + let mut p = asked(); + let mut stop = proof_packet().predications.remove(1); + stop.roles[1].filler = nd("n4"); + p.predications.push(stop); + p +} + +#[test] +fn a_consented_request_lowers_to_what_the_addressee_asked() { + let (inv, _dir) = committed(); + assert_eq!(lowered(&asked(), &inv).unwrap(), "You asked Nibu to stop."); + let mut p = asked(); + p.provenance.source = weksa::packet::ProvenanceSource::GameTelemetry; + assert_eq!(lowered(&p, &inv).unwrap(), "You asked Nibu to stop."); +} + +#[test] +fn a_request_without_its_consent_is_refused_not_invented() { + let (inv, _dir) = committed(); + let mut p = asked(); + p.discourse.addressee_consent = None; + assert_eq!( + lowered(&p, &inv), + Err(LowerError::ConsentMissing { + predication: nd("n4") + }) + ); + // Consent that names a different predication is no consent to the request. + let mut p = asked_and_stopped(); + p.discourse.addressee_consent = Some(nd("n5")); + assert_eq!( + lowered(&p, &inv), + Err(LowerError::ConsentMissing { + predication: nd("n4") + }) + ); +} + +#[test] +fn a_request_by_someone_other_than_the_addressee_is_refused() { + let (inv, _dir) = committed(); + let mut p = asked(); + p.predications[0].roles[0].filler = nd("n3"); + p.predications[0].roles[1].filler = nd("n1"); + assert_eq!( + lowered(&p, &inv), + Err(LowerError::AskerNotAddressee { + predication: nd("n4") + }) + ); +} + +#[test] +fn consent_on_a_claim_that_is_no_request_stays_unrendered() { + let (inv, _dir) = committed(); + let mut p = proof_packet(); + p.discourse.addressee_consent = Some(nd("n5")); + assert_eq!( + lowered(&p, &inv), + Err(LowerError::Unrendered { + what: "addressee consent" + }) + ); +} + +#[test] +fn a_decomposition_or_import_cannot_reach_the_lowerer_with_consent() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let inventory = InventoryStore::open(&root.join("data/interlingua/inventory.cc")).unwrap(); + let corpus = CorpusStore::open(&root.join("data/corpus/corpus.cc")).unwrap(); + let entities = EntityStore::open(&root.join("data/entity/entity.cc")).unwrap(); + for source in [ + weksa::packet::ProvenanceSource::ModelDecomposition, + weksa::packet::ProvenanceSource::ImportedCorpus, + ] { + let mut p = asked(); + p.provenance.source = source; + let dir = tempfile::tempdir().unwrap(); + let refused = PacketStore::open(&dir.path().join("p.cc")) + .unwrap() + .admit(p, &inventory, &corpus, &entities) + .unwrap_err(); + assert!(format!("{refused:?}").contains("V15"), "{source:?}"); + } +} diff --git a/data/interlingua/inventory.cc b/data/interlingua/inventory.cc index becfc92e658d8e2da2833ee1a966e2b71bf2d296..cf353dbe05112cf2edefb596b06a4e009e41515c 100644 GIT binary patch delta 1411 zcmZXUT_{6g9LC$(v5&LwowF~sof?;D64@~yV`eKKBNuFCEk=^D4|kfla3iE6i<>10 zH&WcVla`2b<4VfC63N%4l(pm8<^T6S@BjI~@6-D}xUx-0cOuHSelR>08;(w=_$FAzMWkNR2bzqe8_1gk`vjy3(A-Lw zfQ-pjfDrODkY#xY$cH=(Btiv&TvA=aR9J;iSjH7L+1_X(G8yG+_-)2&hgj?z`PGxZ zP{Mp#(F7h>3cQ7KC6YNM-uRk|=3kUf=vAxg0kWpT^qz_|UNt7OtHud_{b&vP4QhPg zcc}3LA+;Sq2DEKJ_KV23)&m;94#}1dyyupm{em0C5>HodshfXl{&ncGiSvPnz(?awbgKVnz}( yw}aQ28NFx=l2Ihbrl}B5_+H;37Qmm%7bE!;zmuJOGRkz-^ zz;Zn2G6fUu@=q39--oOIu=Y4}b$BJ)?ERbd8ERb7e z90;wN0g_Xp(^MtEajTIWspr9YQ_lerG?-P^6u@z5kAd81Pk~cXajT|`^pbeN0}`)%YNEtXG6>a&OmZ&<&W8yvJ=2LZ zRXU;0nK=q&7R}Q@uFX?GzRiV%v&!1S$fRixf`yP6^=n zZ8+hfZ33JZnbSj$gHxb+Aa^uA7r|gwj2VNhb7mOGD-!_XwvPbWvE%MfcHG_Bg%`_q z;VU;}gm#3X*M{D9)erUDN9{>OV diff --git a/docs/interlingua-standard.md b/docs/interlingua-standard.md index ca392d1..e349a90 100644 --- a/docs/interlingua-standard.md +++ b/docs/interlingua-standard.md @@ -195,9 +195,10 @@ and `features` carry source-side nuance on content (hedge, certainty, irony and whatever the inventory grows). This is source-side communicative pressure, not target-language register. `addressee_consent` is no feature: it names the predication the addressee asked for or agreed to, a fact about what the -addressee did. The English lowerer renders no consent and refuses a packet that -carries one; a row that speaks for the request ("you asked her to") reads it -and refuses without it. +addressee did. The English lowerer's request row (`frame:3`, "{agent} asked +{patient} to stop") reads it: the consent must name that predication and its +agent must be the addressee, else `ConsentMissing` or `AskerNotAddressee`. A +consent naming any other predication is refused as unrendered. ### `stance` diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 4517320..5fa3022 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -342,7 +342,7 @@ measurable baseline from a packet: `lower` gives text, `score` gives numbers. | Whether a packet exists | `validate` / `PacketStore::admit` (unchanged) | packet, inventory, corpus, entities | admitted packet or typed refusal | the lowerer | | Wording of a packet in English | `crates/weksa/src/lower.rs`: `lower(&LoweringInput, &Inventory) -> Result` | the validated lowering view and the inventory at its revision | text, or a typed refusal naming the missing template | the corpus item, the scorer, any model | | English template table (concept words, frame templates, speech-act openers) | the same module, as rows `(inventory id, gloss it speaks for, surface)` | none; it is data in code | rows the lowerer looks up | the inventory (it owns the ids and glosses, not the English) | -| The inventory's concepts and frames | an inventory revision; here `inventory::proof_revision()` (rev 2), hand-seeded as `seed()` is | none | the 5 concepts and 2 frames `nibu-proof-001` names | the lowerer | +| The inventory's concepts and frames | an inventory revision; here `inventory::proof_revision()` (rev 2), hand-seeded as `seed()` is | none | the 6 concepts and 3 frames `nibu-proof-001` and the consent-gated request packet name | the lowerer | | What a score counts | `Scorer::score` (unchanged) | output text, reference text | `ItemScore` | `weksa score`, which only reads text and calls it | | The proof packet | `packet::proof_packet()`, typed ids of rev 2, admitted by `weksa packet proof ` through `PacketStore::admit`. Not JSON: `Packet`'s serde is CultCache's positional tuple, and JSON in is the xenos boundary the cut 7 encoder owns | none | admitted packet | the CLI decides nothing about validity | From 53e54148e0cbe8999b9ba90ae7222bc039347399 Mon Sep 17 00:00:00 2001 From: Weksa Date: Mon, 9 Nov 2026 10:12:01 +0000 Subject: [PATCH 4/4] Test: consent to one of two requests leaves the other unconsented (Soul low) Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/lower.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/crates/weksa/tests/lower.rs b/crates/weksa/tests/lower.rs index 89e35ed..3d796a5 100644 --- a/crates/weksa/tests/lower.rs +++ b/crates/weksa/tests/lower.rs @@ -503,6 +503,21 @@ fn a_request_without_its_consent_is_refused_not_invented() { ); } +#[test] +fn consent_to_one_of_two_requests_leaves_the_other_unconsented() { + let (inv, _dir) = committed(); + let mut p = asked(); + let mut second = p.predications[0].clone(); + second.id = nd("n5"); + p.predications.push(second); + assert_eq!( + lowered(&p, &inv), + Err(LowerError::ConsentMissing { + predication: nd("n5") + }) + ); +} + #[test] fn a_request_by_someone_other_than_the_addressee_is_refused() { let (inv, _dir) = committed();