Merge main into phones-line (keeps both LowerError sets and both test groups)
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Weksa
commit
0ca9b5ebfe
15 changed files with 593 additions and 44 deletions
|
|
@ -485,9 +485,20 @@ pub fn proof_revision() -> (InventoryRevision, Vec<InventoryEntry>) {
|
|||
for (family, glosses) in [
|
||||
(
|
||||
Family::Concept,
|
||||
["endanger", "prevent", "player", "control", "ship_mind"].as_slice(),
|
||||
[
|
||||
"endanger",
|
||||
"prevent",
|
||||
"player",
|
||||
"control",
|
||||
"ship_mind",
|
||||
"request",
|
||||
]
|
||||
.as_slice(),
|
||||
),
|
||||
(
|
||||
Family::Frame,
|
||||
["hazard_warning", "intervention", "request_intervention"].as_slice(),
|
||||
),
|
||||
(Family::Frame, ["hazard_warning", "intervention"].as_slice()),
|
||||
] {
|
||||
for (i, gloss) in glosses.iter().enumerate() {
|
||||
entries.push(InventoryEntry {
|
||||
|
|
|
|||
|
|
@ -17,6 +17,14 @@
|
|||
//! the target language, is refused: the concept word would say a different
|
||||
//! thing than the name the packet cites.
|
||||
//!
|
||||
//! A request clause ("You asked Nibu to stop.") says what the addressee did, so
|
||||
//! it is gated on the packet's own record of it: `Discourse.addressee_consent`
|
||||
//! (set only by authoring or game telemetry, V15) must name that predication,
|
||||
//! and its agent must be the addressee. Without it the lowerer refuses
|
||||
//! ([`LowerError::ConsentMissing`], [`LowerError::AskerNotAddressee`]); it does
|
||||
//! not read wording or guess who asked. A consent naming any other predication
|
||||
//! is an unrendered slot.
|
||||
//!
|
||||
//! The table is a temporary owner. Each row speaks for an inventory entry and
|
||||
//! names the gloss it speaks for, so a row cannot outlive a change of what its id
|
||||
//! means. `language-ontology` moves the rows into ontology documents.
|
||||
|
|
@ -68,8 +76,17 @@ const FRAMES: &[Row] = &[
|
|||
"{agent} may endanger {patient} by touching {instrument}",
|
||||
),
|
||||
("frame:2", "intervention", "{agent} will stop {patient}"),
|
||||
(
|
||||
"frame:3",
|
||||
"request_intervention",
|
||||
"{agent} asked {patient} to stop",
|
||||
),
|
||||
];
|
||||
|
||||
/// The frame of "{agent} asked {patient} to stop": the one frame that reads
|
||||
/// the discourse's addressee consent.
|
||||
const REQUEST_FRAME: &str = "frame:3";
|
||||
|
||||
/// Speech acts: the opener of the first sentence.
|
||||
const SPEECH_ACTS: &[Row] = &[
|
||||
("speech_act:1", "assert", ""),
|
||||
|
|
@ -105,6 +122,12 @@ pub enum LowerError {
|
|||
/// Phones were requested for an entity whose ruled IPA has a stress mark
|
||||
/// with no vowel after it.
|
||||
DanglingStress { entity: EntityId, at: usize },
|
||||
/// A request predication the packet does not carry the addressee's consent
|
||||
/// to: no consent, or a consent that names another predication. The
|
||||
/// lowerer does not guess who asked.
|
||||
ConsentMissing { predication: NodeId },
|
||||
/// A request predication whose agent is not the discourse addressee.
|
||||
AskerNotAddressee { predication: NodeId },
|
||||
/// The entity port failed to read.
|
||||
EntityRead { entity: EntityId, why: String },
|
||||
}
|
||||
|
|
@ -149,6 +172,7 @@ pub fn lower(
|
|||
{
|
||||
return unrendered("features");
|
||||
}
|
||||
request_gate(input, inv)?;
|
||||
if !input.relations().is_empty() {
|
||||
return unrendered("relations");
|
||||
}
|
||||
|
|
@ -195,6 +219,59 @@ pub fn lower(
|
|||
Ok(out)
|
||||
}
|
||||
|
||||
/// The one place consent is read. A request clause ([`REQUEST_FRAME`]) says what
|
||||
/// the addressee did, so it needs the packet's own record of it: the discourse
|
||||
/// `addressee_consent`, which validation lets only authoring or game telemetry
|
||||
/// set (V15), must name that predication, and its agent must be the addressee.
|
||||
/// Nothing here reads wording. A consent naming any other predication is a slot
|
||||
/// the lowerer does not render.
|
||||
fn request_gate(input: &LoweringInput, inv: &Inventory) -> Result<(), LowerError> {
|
||||
let consent = input.discourse().addressee_consent.as_ref();
|
||||
let requests = input
|
||||
.predications()
|
||||
.iter()
|
||||
.filter(|p| p.frame.to_string() == REQUEST_FRAME);
|
||||
for p in requests {
|
||||
if consent != Some(&p.id) {
|
||||
return Err(LowerError::ConsentMissing {
|
||||
predication: p.id.clone(),
|
||||
});
|
||||
}
|
||||
let agent = inv
|
||||
.entries()
|
||||
.find(|e| {
|
||||
e.family() == Family::Role && e.gloss == "agent" && e.live_at(input.inventory_rev())
|
||||
})
|
||||
.ok_or_else(|| LowerError::UnknownRole {
|
||||
frame: p.frame,
|
||||
role: "agent".to_string(),
|
||||
})?;
|
||||
let asker = p
|
||||
.roles
|
||||
.iter()
|
||||
.find(|r| r.role == agent.id)
|
||||
.map(|r| &r.filler);
|
||||
if asker.is_none() || asker != input.discourse().addressee.as_ref() {
|
||||
return Err(LowerError::AskerNotAddressee {
|
||||
predication: p.id.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
match consent {
|
||||
Some(_)
|
||||
if !input
|
||||
.predications()
|
||||
.iter()
|
||||
.any(|p| p.frame.to_string() == REQUEST_FRAME) =>
|
||||
{
|
||||
Err(LowerError::Unrendered {
|
||||
what: "addressee consent",
|
||||
})
|
||||
}
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// The ruled IPA of every entity a packet cites, one per entity referent in
|
||||
/// the packet's referent order, as stored: bare, stress-marked, nothing
|
||||
/// derived. A rejected pronunciation is never read. An entity the port does not
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ pub use crate::inventory::InvId;
|
|||
use crate::inventory::OptionLabel;
|
||||
|
||||
/// Bumped by a commit that changes these types; the published schema moves with it.
|
||||
pub const SCHEMA_VERSION: u32 = 4;
|
||||
pub const SCHEMA_VERSION: u32 = 5;
|
||||
|
||||
/// Id of a referent or predication within one packet: `n<k>`, `k` a canonical
|
||||
/// decimal 1..=9999.
|
||||
|
|
@ -261,6 +261,13 @@ pub struct Discourse {
|
|||
/// Speech-act id.
|
||||
pub speech_act: Option<InvId>,
|
||||
pub addressee: Option<NodeId>,
|
||||
/// The predication the addressee asked for or agreed to. A fact about what
|
||||
/// the addressee did, never about how the line sounds, so it is no feature:
|
||||
/// only a source that saw the request may set it (V15), it needs an
|
||||
/// addressee (V14) and names a predication (V14), and a lowering that
|
||||
/// speaks for the addressee's request ("you asked her to") reads it here
|
||||
/// and refuses without it.
|
||||
pub addressee_consent: Option<NodeId>,
|
||||
pub features: Vec<FeatureValue>,
|
||||
}
|
||||
|
||||
|
|
@ -385,6 +392,7 @@ pub fn proof_packet() -> Packet {
|
|||
discourse: Discourse {
|
||||
speech_act: Some(id("speech_act:5")),
|
||||
addressee: Some(node("n1")),
|
||||
addressee_consent: None,
|
||||
features: vec![],
|
||||
},
|
||||
referents: vec![
|
||||
|
|
|
|||
|
|
@ -1,44 +1,94 @@
|
|||
//! The one way the scorer reads words: UAX #29 word segmentation, then NFC and
|
||||
//! full Unicode case folding. The novel-coverage diagnostic reads text through
|
||||
//! here, so every script is treated by one rule.
|
||||
//! The one way the scorer reads words, then NFC and full Unicode case folding.
|
||||
//! The novel-coverage diagnostic reads text through here, so every script is
|
||||
//! treated by one rule.
|
||||
//!
|
||||
//! A word is a run of word characters: alphanumerics, combining marks and the
|
||||
//! joiners ZWNJ and ZWJ. Any other character ends it, so `kick:the:bucket`,
|
||||
//! `kick·the·bucket`, `kick_the_bucket` and `kick'the'bucket` read as
|
||||
//! `kick the bucket`, as does `kickTheBucket` (a lowercase-to-uppercase
|
||||
//! transition also ends a word).
|
||||
//!
|
||||
//! Han, kana, Thai, Lao, Khmer and Burmese write no word boundaries, and the
|
||||
//! scorer holds no dictionary to find them. A run of those characters is one
|
||||
//! word, never one word per character: per-character words would make every
|
||||
//! honest single-word gloss share bigrams with a held-out sentence, which is
|
||||
//! the leak signature. A change between such a run and any other script also
|
||||
//! ends a word. A leak in these scripts shows at n = 1 and in chrF; a
|
||||
//! dictionary segmenter is the ja-JP panel's to add.
|
||||
|
||||
use caseless::default_case_fold_str;
|
||||
use unicode_normalization::UnicodeNormalization;
|
||||
use unicode_segmentation::UnicodeSegmentation;
|
||||
use unicode_normalization::char::is_combining_mark;
|
||||
|
||||
/// NFC, then full default case folding.
|
||||
pub(super) fn normalise(s: &str) -> String {
|
||||
default_case_fold_str(&s.nfc().collect::<String>())
|
||||
}
|
||||
|
||||
/// The pieces of one segment: split at `_` and `.` (which are dropped) and
|
||||
/// at every lowercase-to-uppercase transition, so `kick_the_bucket`,
|
||||
/// `kick.the.bucket` and `kickTheBucket` read as `kick the bucket`.
|
||||
fn pieces(segment: &str) -> Vec<&str> {
|
||||
fn is_joiner(c: char) -> bool {
|
||||
matches!(c, '\u{200c}' | '\u{200d}')
|
||||
}
|
||||
|
||||
/// A character that belongs inside a word. Marks must stay: Thai vowels and
|
||||
/// tone marks, Hindi vowel signs and Hangul jamo are not alphanumeric.
|
||||
fn is_word_char(c: char) -> bool {
|
||||
c.is_alphanumeric() || is_combining_mark(c) || is_joiner(c)
|
||||
}
|
||||
|
||||
/// Han, kana, Thai, Lao, Khmer and Burmese: scripts written without spaces.
|
||||
fn is_unspaced(c: char) -> bool {
|
||||
matches!(
|
||||
c,
|
||||
'\u{3005}'
|
||||
| '\u{3007}'
|
||||
| '\u{3021}'..='\u{3029}'
|
||||
| '\u{3038}'..='\u{303b}'
|
||||
| '\u{3041}'..='\u{30ff}'
|
||||
| '\u{3400}'..='\u{4dbf}'
|
||||
| '\u{4e00}'..='\u{9fff}'
|
||||
| '\u{f900}'..='\u{fad9}'
|
||||
| '\u{ff66}'..='\u{ff9f}'
|
||||
| '\u{31f0}'..='\u{31ff}'
|
||||
| '\u{20000}'..='\u{323af}'
|
||||
| '\u{0e01}'..='\u{0e5b}'
|
||||
| '\u{0e81}'..='\u{0edf}'
|
||||
| '\u{1000}'..='\u{109f}'
|
||||
| '\u{1780}'..='\u{17ff}'
|
||||
)
|
||||
}
|
||||
|
||||
/// The normalised words of `text` in order; runs with no alphanumeric char
|
||||
/// (marks alone, say) are dropped.
|
||||
pub(super) fn words(text: &str) -> Vec<String> {
|
||||
let mut out = Vec::new();
|
||||
let mut start = 0;
|
||||
let mut previous: Option<char> = None;
|
||||
for (at, c) in segment.char_indices() {
|
||||
let separator = c == '_' || c == '.';
|
||||
let camel = previous.is_some_and(char::is_lowercase) && c.is_uppercase();
|
||||
if separator || camel {
|
||||
out.push(&segment[start..at]);
|
||||
start = if separator { at + c.len_utf8() } else { at };
|
||||
let end = |from: usize, to: usize, out: &mut Vec<String>| {
|
||||
let word = &text[from..to];
|
||||
if word.chars().any(char::is_alphanumeric) {
|
||||
out.push(normalise(word));
|
||||
}
|
||||
};
|
||||
for (at, c) in text.char_indices() {
|
||||
if !is_word_char(c) {
|
||||
end(start, at, &mut out);
|
||||
start = at + c.len_utf8();
|
||||
continue;
|
||||
}
|
||||
// a mark or joiner belongs to the character before it
|
||||
let attached = is_combining_mark(c) || is_joiner(c);
|
||||
let breaks = !attached
|
||||
&& previous.is_some_and(|p| {
|
||||
(p.is_lowercase() && c.is_uppercase()) || is_unspaced(p) != is_unspaced(c)
|
||||
});
|
||||
if breaks {
|
||||
end(start, at, &mut out);
|
||||
start = at;
|
||||
}
|
||||
if !attached {
|
||||
previous = Some(c);
|
||||
}
|
||||
previous = Some(c);
|
||||
}
|
||||
out.push(&segment[start..]);
|
||||
end(start, text.len(), &mut out);
|
||||
out
|
||||
}
|
||||
|
||||
/// The normalised words of `text` in order: UAX #29 segments containing an
|
||||
/// alphanumeric char (digits kept), each split by `pieces`, empty or
|
||||
/// non-alphanumeric pieces dropped.
|
||||
pub(super) fn words(text: &str) -> Vec<String> {
|
||||
text.split_word_bounds()
|
||||
.filter(|segment| segment.chars().any(char::is_alphanumeric))
|
||||
.flat_map(pieces)
|
||||
.filter(|piece| piece.chars().any(char::is_alphanumeric))
|
||||
.map(normalise)
|
||||
.collect()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
//! returns `Ok(())` or the refusals. [`LoweringInput`], the only thing a
|
||||
//! lowerer sees, comes from `PacketStore::admit` alone, which builds it through
|
||||
//! the crate-private `admissible`. The stance is checked like every feature
|
||||
//! list, in slot `Stance`. The rules are V1..V8 and V10..V13: the shape of an id
|
||||
//! list, in slot `Stance`. The rules are V1..V8 and V10..V15: the shape of an id
|
||||
//! or a grade is its type's parse, not a rule, so V9 is retired and not
|
||||
//! renumbered. V13 guards the one other way features reach a lowering view:
|
||||
//! [`LoweringInput::with_scene`].
|
||||
|
|
@ -29,7 +29,7 @@ use std::fmt;
|
|||
use crate::inventory::{Family, InvId, Inventory, InventoryEntry, Slot, ValueKind};
|
||||
use crate::packet::{
|
||||
Constraints, Context, Discourse, Extension, FeatValue, FeatureValue, NodeId, Packet,
|
||||
PacketKind, Predication, Referent, Relation, SCHEMA_VERSION, Stance,
|
||||
PacketKind, Predication, ProvenanceSource, Referent, Relation, SCHEMA_VERSION, Stance,
|
||||
};
|
||||
|
||||
/// A broken rule, named.
|
||||
|
|
@ -71,6 +71,15 @@ pub enum Refusal {
|
|||
/// state the speaker's relation to the addressee, so with none they
|
||||
/// describe nobody.
|
||||
V13SceneWithoutAddressee,
|
||||
/// V14: an addressee consent that no addressee can have given, or that
|
||||
/// names a node that is not a predication. It is what the addressee asked
|
||||
/// for or agreed to, an act, so a referent cannot be its object.
|
||||
V14ConsentWithoutAddressee,
|
||||
V14ConsentNotAPredication { id: NodeId },
|
||||
/// V15: an addressee consent set by a source that only read wording.
|
||||
/// A decomposition of text, or imported corpus text, sees "you asked her
|
||||
/// to" as words; only authoring or game telemetry saw the request.
|
||||
V15ConsentSource { source: ProvenanceSource },
|
||||
}
|
||||
|
||||
impl Refusal {
|
||||
|
|
@ -90,6 +99,10 @@ impl Refusal {
|
|||
| Refusal::V11DuplicateRole { .. } => "V11",
|
||||
Refusal::V12TooManyNodes { .. } => "V12",
|
||||
Refusal::V13SceneWithoutAddressee => "V13",
|
||||
Refusal::V14ConsentWithoutAddressee | Refusal::V14ConsentNotAPredication { .. } => {
|
||||
"V14"
|
||||
}
|
||||
Refusal::V15ConsentSource { .. } => "V15",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -156,6 +169,7 @@ pub(crate) fn admissible(
|
|||
let nodes = check_nodes(packet, &mut out);
|
||||
check_refs(packet, &nodes, &mut out);
|
||||
check_repeats(packet, &mut out);
|
||||
check_consent(packet, &nodes, &mut out);
|
||||
if inventory.has_revision(packet.inventory_rev) {
|
||||
Inventoried {
|
||||
inv: inventory,
|
||||
|
|
@ -208,6 +222,9 @@ fn check_refs(packet: &Packet, nodes: &BTreeSet<NodeId>, out: &mut Vec<Refusal>)
|
|||
if let Some(a) = &packet.discourse.addressee {
|
||||
need(a, "addressee");
|
||||
}
|
||||
if let Some(c) = &packet.discourse.addressee_consent {
|
||||
need(c, "addressee consent");
|
||||
}
|
||||
for (list, at) in node_lists(packet) {
|
||||
for id in list {
|
||||
need(id, at);
|
||||
|
|
@ -215,6 +232,26 @@ fn check_refs(packet: &Packet, nodes: &BTreeSet<NodeId>, out: &mut Vec<Refusal>)
|
|||
}
|
||||
}
|
||||
|
||||
/// V14 and V15: the addressee's consent is a typed fact with one origin. A
|
||||
/// dangling id is V4's, so V14 judges only a node that exists.
|
||||
fn check_consent(packet: &Packet, nodes: &BTreeSet<NodeId>, out: &mut Vec<Refusal>) {
|
||||
let Some(consent) = &packet.discourse.addressee_consent else {
|
||||
return;
|
||||
};
|
||||
if packet.discourse.addressee.is_none() {
|
||||
out.push(Refusal::V14ConsentWithoutAddressee);
|
||||
}
|
||||
if nodes.contains(consent) && !packet.predications.iter().any(|p| &p.id == consent) {
|
||||
out.push(Refusal::V14ConsentNotAPredication {
|
||||
id: consent.clone(),
|
||||
});
|
||||
}
|
||||
match packet.provenance.source {
|
||||
ProvenanceSource::Authored | ProvenanceSource::GameTelemetry => {}
|
||||
source => out.push(Refusal::V15ConsentSource { source }),
|
||||
}
|
||||
}
|
||||
|
||||
fn node_lists(packet: &Packet) -> [(&Vec<NodeId>, &'static str); 3] {
|
||||
[
|
||||
(&packet.context.participants, "participant"),
|
||||
|
|
|
|||
|
|
@ -266,6 +266,7 @@ pub fn good() -> Packet {
|
|||
discourse: Discourse {
|
||||
speech_act: Some(iv(WARN)),
|
||||
addressee: Some(nd("n1")),
|
||||
addressee_consent: None,
|
||||
features: vec![FeatureValue {
|
||||
feature: iv(HEDGE),
|
||||
value: FeatValue::Grade(grade(300)),
|
||||
|
|
|
|||
|
|
@ -397,7 +397,7 @@ fn packet_entities_resolve() {
|
|||
|
||||
#[test]
|
||||
fn packet_schema_v2() {
|
||||
assert_eq!(SCHEMA_VERSION, 4);
|
||||
assert_eq!(SCHEMA_VERSION, 5);
|
||||
let mut p = good();
|
||||
p.schema_version = 2;
|
||||
let Err(refused) = validate(&p, &inventory()) else {
|
||||
|
|
|
|||
|
|
@ -142,6 +142,7 @@ fn a_packet_carrying_what_the_lowerer_does_not_render_is_refused() {
|
|||
filler,
|
||||
})
|
||||
};
|
||||
let consent = |p: &mut Packet| p.discourse.addressee_consent = Some(nd("n5"));
|
||||
let lone_referent = |p: &mut Packet| {
|
||||
let mut r = p.referents[0].clone();
|
||||
r.id = nd("n99");
|
||||
|
|
@ -150,6 +151,7 @@ fn a_packet_carrying_what_the_lowerer_does_not_render_is_refused() {
|
|||
for (what, edit) in [
|
||||
("features", &discourse as &dyn Fn(&mut Packet)),
|
||||
("features", &predication),
|
||||
("addressee consent", &consent),
|
||||
("relations", &relation),
|
||||
("extensions", &extension),
|
||||
("no predication", &no_predication),
|
||||
|
|
@ -571,3 +573,132 @@ fn the_inventory_parser_refuses_what_it_cannot_read() {
|
|||
assert!(PhoneInventory::parse("# c\n\ni\tvowel\ttense\ta\n").is_ok());
|
||||
assert!(PhoneInventory::for_language("sa-Latn").is_none());
|
||||
}
|
||||
|
||||
/// `nibu-proof-001` re-cut as the request: the addressee asked Nibu to stop,
|
||||
/// one frame:3 predication (n4), consent naming it, and no other claim.
|
||||
fn asked() -> Packet {
|
||||
let mut p = proof_packet();
|
||||
p.referents.remove(1);
|
||||
let mut ask = p.predications[1].clone();
|
||||
ask.id = nd("n4");
|
||||
ask.concept = iv("concept:6");
|
||||
ask.frame = iv("frame:3");
|
||||
ask.roles = vec![
|
||||
weksa::packet::RoleFill {
|
||||
role: iv("role:1"),
|
||||
filler: nd("n1"),
|
||||
},
|
||||
weksa::packet::RoleFill {
|
||||
role: iv("role:2"),
|
||||
filler: nd("n3"),
|
||||
},
|
||||
];
|
||||
p.predications = vec![ask];
|
||||
p.discourse.speech_act = Some(iv("speech_act:1"));
|
||||
p.discourse.addressee_consent = Some(nd("n4"));
|
||||
p.constraints.preserve = vec![nd("n1")];
|
||||
p
|
||||
}
|
||||
|
||||
/// The request plus a frame:2 predication (n5) the consent could wrongly name.
|
||||
fn asked_and_stopped() -> Packet {
|
||||
let mut p = asked();
|
||||
let mut stop = proof_packet().predications.remove(1);
|
||||
stop.roles[1].filler = nd("n4");
|
||||
p.predications.push(stop);
|
||||
p
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_consented_request_lowers_to_what_the_addressee_asked() {
|
||||
let (inv, _dir) = committed();
|
||||
assert_eq!(lowered(&asked(), &inv).unwrap(), "You asked Nibu to stop.");
|
||||
let mut p = asked();
|
||||
p.provenance.source = weksa::packet::ProvenanceSource::GameTelemetry;
|
||||
assert_eq!(lowered(&p, &inv).unwrap(), "You asked Nibu to stop.");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_request_without_its_consent_is_refused_not_invented() {
|
||||
let (inv, _dir) = committed();
|
||||
let mut p = asked();
|
||||
p.discourse.addressee_consent = None;
|
||||
assert_eq!(
|
||||
lowered(&p, &inv),
|
||||
Err(LowerError::ConsentMissing {
|
||||
predication: nd("n4")
|
||||
})
|
||||
);
|
||||
// Consent that names a different predication is no consent to the request.
|
||||
let mut p = asked_and_stopped();
|
||||
p.discourse.addressee_consent = Some(nd("n5"));
|
||||
assert_eq!(
|
||||
lowered(&p, &inv),
|
||||
Err(LowerError::ConsentMissing {
|
||||
predication: nd("n4")
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn consent_to_one_of_two_requests_leaves_the_other_unconsented() {
|
||||
let (inv, _dir) = committed();
|
||||
let mut p = asked();
|
||||
let mut second = p.predications[0].clone();
|
||||
second.id = nd("n5");
|
||||
p.predications.push(second);
|
||||
assert_eq!(
|
||||
lowered(&p, &inv),
|
||||
Err(LowerError::ConsentMissing {
|
||||
predication: nd("n5")
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_request_by_someone_other_than_the_addressee_is_refused() {
|
||||
let (inv, _dir) = committed();
|
||||
let mut p = asked();
|
||||
p.predications[0].roles[0].filler = nd("n3");
|
||||
p.predications[0].roles[1].filler = nd("n1");
|
||||
assert_eq!(
|
||||
lowered(&p, &inv),
|
||||
Err(LowerError::AskerNotAddressee {
|
||||
predication: nd("n4")
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn consent_on_a_claim_that_is_no_request_stays_unrendered() {
|
||||
let (inv, _dir) = committed();
|
||||
let mut p = proof_packet();
|
||||
p.discourse.addressee_consent = Some(nd("n5"));
|
||||
assert_eq!(
|
||||
lowered(&p, &inv),
|
||||
Err(LowerError::Unrendered {
|
||||
what: "addressee consent"
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_decomposition_or_import_cannot_reach_the_lowerer_with_consent() {
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
|
||||
let inventory = InventoryStore::open(&root.join("data/interlingua/inventory.cc")).unwrap();
|
||||
let corpus = CorpusStore::open(&root.join("data/corpus/corpus.cc")).unwrap();
|
||||
let entities = EntityStore::open(&root.join("data/entity/entity.cc")).unwrap();
|
||||
for source in [
|
||||
weksa::packet::ProvenanceSource::ModelDecomposition,
|
||||
weksa::packet::ProvenanceSource::ImportedCorpus,
|
||||
] {
|
||||
let mut p = asked();
|
||||
p.provenance.source = source;
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let refused = PacketStore::open(&dir.path().join("p.cc"))
|
||||
.unwrap()
|
||||
.admit(p, &inventory, &corpus, &entities)
|
||||
.unwrap_err();
|
||||
assert!(format!("{refused:?}").contains("V15"), "{source:?}");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -690,3 +690,155 @@ fn every_script_alike() {
|
|||
assert_eq!(got.chrf, 100.0, "{text}");
|
||||
}
|
||||
}
|
||||
|
||||
/// (reference, hypothesis, sacrebleu 2.4.3 sentence_chrf), one near miss per
|
||||
/// script: the Thai hypothesis is the reference with its vowels and tone mark
|
||||
/// stripped.
|
||||
const SCRIPT_PAIRS: [(&str, &str, f64); 5] = [
|
||||
("민수가 왔다", "민수는 왔다", 26.0),
|
||||
("Brokkr'ın gemisi geldi", "Brokkr'ın gemisi gitti", 77.7548),
|
||||
("Brokkrin laiva tuli", "Brokkrin laiva meni", 73.002),
|
||||
("สมชายมาแล้ว", "สมชยมแลว", 21.8989),
|
||||
("ブロッカーが来た。", "ブロッカーが行った。", 54.4013),
|
||||
];
|
||||
|
||||
#[test]
|
||||
fn every_script_scores_a_near_miss_as_sacrebleu_does() {
|
||||
let scorer = Scorer::new(Bag);
|
||||
for (reference, hypothesis, want) in SCRIPT_PAIRS {
|
||||
close(score_one(&scorer, reference, hypothesis).chrf, want);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn coverage_splits_at_any_non_word_character() {
|
||||
let items = [
|
||||
validation("item:v:1", "mind the hatch before the storm"),
|
||||
train("item:t:1", UNRELATED),
|
||||
];
|
||||
let want = read(&["mind the hatch"], &items);
|
||||
assert_eq!(want, [(5, 3), (5, 2), (4, 1)]);
|
||||
for joiner in [
|
||||
":", "\u{b7}", "'", "\u{2019}", "\u{ff3f}", "-", "/", "\u{2014}", "|", "+", "\u{ff1a}",
|
||||
"\u{30fb}",
|
||||
] {
|
||||
let glued = format!("mind{joiner}the{joiner}hatch");
|
||||
assert_eq!(read(&[&glued], &items), want, "{joiner:?}");
|
||||
}
|
||||
// a mark, a joiner or a digit is part of its word, not a break
|
||||
let items = [validation(
|
||||
"item:v:1",
|
||||
"\u{e2a}\u{e21}\u{e0a}\u{e32}\u{e22} \u{e21}\u{e32}\u{e41}\u{e25}\u{e49}\u{e27} 0400 \u{628}\u{200c}\u{628}",
|
||||
)];
|
||||
assert_eq!(
|
||||
read(&["\u{e21}\u{e32}\u{e41}\u{e25}\u{e49}\u{e27}"], &items)[0],
|
||||
(4, 1)
|
||||
);
|
||||
assert_eq!(read(&["\u{628}\u{200c}\u{628}", "0400"], &items)[0], (4, 2));
|
||||
// a joiner keeps its word whole: the minted half is not a held-out word
|
||||
let items = [validation("item:v:1", "\u{628}\u{200c}\u{62a} \u{62c}")];
|
||||
assert_eq!(read(&["\u{628}\u{200c}\u{62a}"], &items)[0], (2, 1));
|
||||
// decomposed and composed spellings are one word
|
||||
let items = [validation("item:v:1", "Cafe\u{301}")];
|
||||
assert_eq!(read(&["Caf\u{e9}"], &items)[0], (1, 1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn coverage_reads_unspaced_scripts_by_run() {
|
||||
// Han, kana and Thai write no word boundaries and the scorer holds no
|
||||
// dictionary, so a run of them is one word, never one word per character.
|
||||
// Honest single-word glosses then read nothing above n=1, as they do in a
|
||||
// spaced script.
|
||||
for (held, glosses) in [
|
||||
(
|
||||
"\u{5c71}\u{7530}\u{592a}\u{90ce}\u{304c}\u{6765}\u{305f}",
|
||||
["\u{5c71}\u{7530}", "\u{592a}\u{90ce}", "\u{6765}\u{305f}"],
|
||||
),
|
||||
(
|
||||
"\u{30d6}\u{30ed}\u{30c3}\u{30ab}\u{30fc}\u{304c}\u{6765}\u{305f}\u{3002}",
|
||||
["\u{30d6}\u{30ed}", "\u{30ab}\u{30fc}", "\u{6765}"],
|
||||
),
|
||||
(
|
||||
"\u{e2a}\u{e21}\u{e0a}\u{e32}\u{e22}\u{e21}\u{e32}\u{e41}\u{e25}\u{e49}\u{e27}",
|
||||
[
|
||||
"\u{e2a}\u{e21}",
|
||||
"\u{e0a}\u{e32}\u{e22}",
|
||||
"\u{e41}\u{e25}\u{e49}",
|
||||
],
|
||||
),
|
||||
] {
|
||||
let items = [validation("item:v:1", held), train("item:t:1", UNRELATED)];
|
||||
assert_eq!(read(&glosses, &items), [(1, 0), (0, 0), (0, 0)], "{held}");
|
||||
assert_eq!(read(&[held], &items), [(1, 1), (0, 0), (0, 0)], "{held}");
|
||||
}
|
||||
// Delimiters still make words: spaces and punctuation between runs, and a
|
||||
// change between an unspaced script and a spaced one.
|
||||
let items = [
|
||||
validation(
|
||||
"item:v:1",
|
||||
"\u{72ac}\u{304c} \u{8d70}\u{308b}\u{3001}\u{732b}\u{3082} Brokkr\u{5c71}",
|
||||
),
|
||||
train("item:t:1", UNRELATED),
|
||||
];
|
||||
assert_eq!(
|
||||
read(
|
||||
&["\u{72ac}\u{304c} \u{8d70}\u{308b} \u{732b}\u{3082} brokkr \u{5c71}"],
|
||||
&items
|
||||
),
|
||||
[(5, 5), (4, 4), (3, 3)]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unspaced_script_ends_a_word_beside_any_other_script() {
|
||||
// the first and last letters or digits of every range the rule names
|
||||
let unspaced = [
|
||||
'\u{3005}',
|
||||
'\u{3007}',
|
||||
'\u{3021}',
|
||||
'\u{3029}',
|
||||
'\u{3038}',
|
||||
'\u{303b}',
|
||||
'\u{3041}',
|
||||
'\u{30ff}',
|
||||
'\u{31f0}',
|
||||
'\u{31ff}',
|
||||
'\u{3400}',
|
||||
'\u{4dbf}',
|
||||
'\u{4e00}',
|
||||
'\u{9fff}',
|
||||
'\u{f900}',
|
||||
'\u{fad9}',
|
||||
'\u{ff66}',
|
||||
'\u{ff9f}',
|
||||
'\u{20000}',
|
||||
'\u{2a6df}',
|
||||
'\u{31350}',
|
||||
'\u{323af}',
|
||||
'\u{e01}',
|
||||
'\u{e59}',
|
||||
'\u{e81}',
|
||||
'\u{edf}',
|
||||
'\u{1000}',
|
||||
'\u{1049}',
|
||||
'\u{1780}',
|
||||
'\u{17e9}',
|
||||
];
|
||||
for c in unspaced {
|
||||
assert!(c.is_alphanumeric(), "{c:?} is a letter or digit");
|
||||
let items = [validation("item:v:1", &format!("ab{c}"))];
|
||||
assert_eq!(read(&["ab", &c.to_string()], &items)[0], (2, 2), "{c:?}");
|
||||
}
|
||||
// Hangul, Latin and Cyrillic are spaced scripts: no break between them
|
||||
for c in ['\u{ac00}', '\u{d7a3}', '\u{e9}', '\u{44f}', '7'] {
|
||||
let glued = format!("ab{c}");
|
||||
let items = [validation("item:v:1", &glued)];
|
||||
assert_eq!(read(&[&glued], &items)[0], (1, 1), "{c:?}");
|
||||
assert_eq!(read(&["ab"], &items)[0], (1, 0), "{c:?}");
|
||||
}
|
||||
// a combining mark or a joiner after an unspaced character stays on it
|
||||
let held = "\u{4e00}\u{301}\u{200d}ab";
|
||||
let items = [validation("item:v:1", held)];
|
||||
assert_eq!(read(&["\u{4e00}\u{301}\u{200d}", "ab"], &items)[0], (2, 2));
|
||||
assert_eq!(read(&["\u{4e00}"], &items)[0], (2, 0));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -989,14 +989,14 @@ fn packet_admission_goes_through_the_validator() {
|
|||
let mut w = world();
|
||||
let mut store = PacketStore::open(&w.dir.path().join("p.cc")).unwrap();
|
||||
let mut bad = good();
|
||||
bad.schema_version = 5;
|
||||
bad.schema_version = 6;
|
||||
match store.admit(bad, &w.inventory, &w.corpus, &w.entities) {
|
||||
Err(e @ StoreError::Invalid(_)) => {
|
||||
assert!(e.to_string().contains("V1"), "{e}");
|
||||
let StoreError::Invalid(r) = e else {
|
||||
unreachable!()
|
||||
};
|
||||
assert_eq!(r, vec![Refusal::V1SchemaVersion { found: 5 }]);
|
||||
assert_eq!(r, vec![Refusal::V1SchemaVersion { found: 6 }]);
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -683,3 +683,62 @@ fn v11_role_repeat_in_one_predication_refused() {
|
|||
ok.predications.push(second);
|
||||
assert_eq!(refusals(&ok), vec![]);
|
||||
}
|
||||
|
||||
// The addressee's consent: which predication the addressee asked for, set only
|
||||
// by a source that saw the request.
|
||||
|
||||
fn consenting() -> Packet {
|
||||
let mut p = good();
|
||||
p.discourse.addressee_consent = Some(nd("n2"));
|
||||
p
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn consent_to_a_predication_by_an_authored_or_telemetry_source_passes() {
|
||||
assert_eq!(refusals(&consenting()), vec![]);
|
||||
let mut p = consenting();
|
||||
p.provenance.source = ProvenanceSource::GameTelemetry;
|
||||
assert_eq!(refusals(&p), vec![]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn consent_needs_an_addressee() {
|
||||
let mut p = consenting();
|
||||
p.discourse.addressee = None;
|
||||
assert_eq!(refusals(&p), vec![Refusal::V14ConsentWithoutAddressee]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn consent_names_a_predication_not_a_referent() {
|
||||
let mut p = consenting();
|
||||
p.discourse.addressee_consent = Some(nd("n1"));
|
||||
assert_eq!(
|
||||
refusals(&p),
|
||||
vec![Refusal::V14ConsentNotAPredication { id: nd("n1") }]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn consent_to_nothing_is_dangling_only() {
|
||||
let mut p = consenting();
|
||||
p.discourse.addressee_consent = Some(nd("n99"));
|
||||
assert_eq!(
|
||||
refusals(&p),
|
||||
vec![Refusal::V4DanglingRef {
|
||||
id: nd("n99"),
|
||||
at: "addressee consent"
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_source_that_only_read_wording_cannot_set_consent() {
|
||||
for source in [
|
||||
ProvenanceSource::ModelDecomposition,
|
||||
ProvenanceSource::ImportedCorpus,
|
||||
] {
|
||||
let mut p = consenting();
|
||||
p.provenance.source = source;
|
||||
assert_eq!(refusals(&p), vec![Refusal::V15ConsentSource { source }]);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -120,7 +120,7 @@ Packet
|
|||
id, schema_version, inventory_rev, kind
|
||||
provenance { source, corpus_item, encoder, confidence }
|
||||
context { setting, medium, participants, audience }
|
||||
discourse { speech_act, addressee, features }
|
||||
discourse { speech_act, addressee, addressee_consent, features }
|
||||
referents [ { id, kind, concept, entity, features } ]
|
||||
predications [ { id, concept, frame, roles: [ { role, filler } ], features } ]
|
||||
relations [ { relation, from, to, features } ]
|
||||
|
|
@ -144,8 +144,8 @@ names the rule.
|
|||
- V1: `schema_version` is the version these types define.
|
||||
- V2: at least one of `referents`, `predications`, `relations` is present.
|
||||
- V3: a node id appears once across referents and predications.
|
||||
- V4: every role filler, relation end, addressee, participant, audience member
|
||||
and preserved node names a node.
|
||||
- V4: every role filler, relation end, addressee, addressee consent,
|
||||
participant, audience member and preserved node names a node.
|
||||
- V5: every inventory id resolves at the packet's `inventory_rev`: introduced
|
||||
at or before it, and not retired at or before it.
|
||||
- V6: an inventory id is of the family its slot takes.
|
||||
|
|
@ -159,6 +159,10 @@ names the rule.
|
|||
predication's roles.
|
||||
- V12: a packet has at most 64 referents, predications and relations together
|
||||
(a calibration constant, capacity and not meaning).
|
||||
- V14: an `addressee_consent` needs an `addressee` and names a predication.
|
||||
- V15: an `addressee_consent` is set only by provenance `Authored` or
|
||||
`GameTelemetry`; a decomposition of text or imported corpus text cannot mint
|
||||
it.
|
||||
|
||||
V9 is retired and not renumbered: the form of a key or a grade is its type, and
|
||||
an extension key's family is V6. `PacketStore::admit` returns the lowering view of a
|
||||
|
|
@ -189,7 +193,12 @@ ids.
|
|||
Why the packet is said: `speech_act` is a speech-act id, `addressee` a node id,
|
||||
and `features` carry source-side nuance on content (hedge, certainty, irony and
|
||||
whatever the inventory grows). This is source-side communicative pressure, not
|
||||
target-language register.
|
||||
target-language register. `addressee_consent` is no feature: it names the
|
||||
predication the addressee asked for or agreed to, a fact about what the
|
||||
addressee did. The English lowerer's request row (`frame:3`, "{agent} asked
|
||||
{patient} to stop") reads it: the consent must name that predication and its
|
||||
agent must be the addressee, else `ConsentMissing` or `AskerNotAddressee`. A
|
||||
consent naming any other predication is refused as unrendered.
|
||||
|
||||
### `stance`
|
||||
|
||||
|
|
|
|||
|
|
@ -202,9 +202,12 @@ operator-attested annotations alone. Who annotates is question
|
|||
text against held-out items; novel coverage pooled per order, for n = 1, 2
|
||||
and 3 separately, the covered share of all novel n-grams (word n-grams of a
|
||||
held-out item that occur in no Train item) over a split's items, with the
|
||||
worst item at n >= 2. Keys split on `_`, `.` and lower-to-upper case
|
||||
transitions as well as UAX #29 boundaries, so a snake_case `OptionLabel`
|
||||
reads as words. Honest glosses of single words saturate n = 1 (the 54 seed
|
||||
worst item at n >= 2. A word is a run of letters, digits, marks and
|
||||
joiners; any other character, and a lower-to-upper case transition, ends it,
|
||||
so `kick:the:bucket` and a snake_case `OptionLabel` read as words. Han, kana,
|
||||
Thai, Lao, Khmer and Burmese write no word boundaries and the scorer holds no
|
||||
dictionary, so a run of them is one word (per-character words would give
|
||||
every honest gloss shared bigrams); a leak there shows at n = 1 and in chrF. Honest glosses of single words saturate n = 1 (the 54 seed
|
||||
glosses read max 1.000, s2), so the leak signature is pooled n >= 2.
|
||||
Costs: a live persona mints before it names (one typed write, which also
|
||||
buys coreference and one rendering per language); public-domain intake
|
||||
|
|
@ -339,7 +342,7 @@ measurable baseline from a packet: `lower` gives text, `score` gives numbers.
|
|||
| Whether a packet exists | `validate` / `PacketStore::admit` (unchanged) | packet, inventory, corpus, entities | admitted packet or typed refusal | the lowerer |
|
||||
| Wording of a packet in English | `crates/weksa/src/lower.rs`: `lower(&LoweringInput, &Inventory) -> Result<String, LowerError>` | the validated lowering view and the inventory at its revision | text, or a typed refusal naming the missing template | the corpus item, the scorer, any model |
|
||||
| English template table (concept words, frame templates, speech-act openers) | the same module, as rows `(inventory id, gloss it speaks for, surface)` | none; it is data in code | rows the lowerer looks up | the inventory (it owns the ids and glosses, not the English) |
|
||||
| The inventory's concepts and frames | an inventory revision; here `inventory::proof_revision()` (rev 2), hand-seeded as `seed()` is | none | the 5 concepts and 2 frames `nibu-proof-001` names | the lowerer |
|
||||
| The inventory's concepts and frames | an inventory revision; here `inventory::proof_revision()` (rev 2), hand-seeded as `seed()` is | none | the 6 concepts and 3 frames `nibu-proof-001` and the consent-gated request packet name | the lowerer |
|
||||
| What a score counts | `Scorer::score` (unchanged) | output text, reference text | `ItemScore` | `weksa score`, which only reads text and calls it |
|
||||
| The proof packet | `packet::proof_packet()`, typed ids of rev 2, admitted by `weksa packet proof <store>` through `PacketStore::admit`. Not JSON: `Packet`'s serde is CultCache's positional tuple, and JSON in is the xenos boundary the cut 7 encoder owns | none | admitted packet | the CLI decides nothing about validity |
|
||||
|
||||
|
|
|
|||
|
|
@ -164,6 +164,17 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
"addressee_consent": {
|
||||
"description": "The predication the addressee asked for or agreed to. A fact about what\nthe addressee did, never about how the line sounds, so it is no feature:\nonly a source that saw the request may set it (V15), it needs an\naddressee (V14) and names a predication (V14), and a lowering that\nspeaks for the addressee's request (\"you asked her to\") reads it here\nand refuses without it.",
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/NodeId"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"features": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
|
|
|
|||
Loading…
Reference in a new issue