diff --git a/.gitignore b/.gitignore index e00572b..1428a57 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,3 @@ build/ .venv/ venv/ target/ -tools/show-working/out/ diff --git a/AGENTS.md b/AGENTS.md index 55d488a..a82bc28 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -60,4 +60,4 @@ language-project truth. `CARGO_HOME=/var/lib/mind-weksa-self/cargo-home cargo test --workspace`. - `weksa packet proof ` admits the nibu proof packet under the key `packet:6e6962752d303031` (hex of `nibu-001`), not `nibu-proof-001`. Use that key - with `weksa lower `. + with `weksa pronounce `. diff --git a/README.md b/README.md index 2b5d0d3..8becda9 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,8 @@ measured. - `settings/`: setting-specific proving grounds for language projects. **Live versus planned.** Live: docs, fixtures, examples, ontology data and the -packet crate `crates/weksa`, including `weksa lower` (the template lowerer for -proof fixtures) and `weksa score`, and the golf loop's round and edit-accept +packet crate `crates/weksa`, including `weksa pronounce` (the ruled IPA and phones +of the entities a packet cites), `weksa score`, and the golf loop's round and edit-accept cycle (`weksa loop open|round|cycle`), which encodes and lowers through the model (a prompted Claude subagent, per staged-distill). Planned: Score and lowering documents and the other panel languages. No diff --git a/crates/weksa/examples/show_working.rs b/crates/weksa/examples/show_working.rs deleted file mode 100644 index 6a31b26..0000000 --- a/crates/weksa/examples/show_working.rs +++ /dev/null @@ -1,398 +0,0 @@ -//! Build-time generator for the "Show her working" page. Runs the real -//! admission door, lowerer and IPA reader over the committed proof packet and -//! variants of it, and prints one JSON document on stdout: the typed slots, the -//! inventory entries the packet chose, and the outcome, which is lowered text or -//! a named refusal. Scoring is not here: `tools/show-working/build.py` calls the -//! `weksa score` CLI on the text this emits. -//! -//! Run from the repository root: `cargo run --release --example show_working`. -//! Every variant is a mutation of `proof_packet()`, spelled in its `source`. - -use std::path::Path; - -use serde_json::{Value, json}; -use weksa::packet::{ - EntityId, FeatValue, FeatureValue, Grade, NodeId, ProvenanceSource, RoleFill, proof_packet, -}; -use weksa::{ - CorpusStore, EntityStore, InvId, Inventory, InventoryStore, LowerError, Packet, PacketStore, - StoreError, lower, pronounce, -}; - -struct Fixture { - id: &'static str, - title: &'static str, - note: &'static str, - source: &'static str, - packet: Packet, -} - -fn id(s: &str) -> InvId { - s.parse().expect("inventory id") -} - -fn node(s: &str) -> NodeId { - NodeId::parse(s).expect("node id") -} - -fn grade(feature: &str, v: u16) -> FeatureValue { - FeatureValue { - feature: id(feature), - value: FeatValue::Grade(Grade::new(v).expect("grade")), - } -} - -fn fixtures() -> Vec { - let proof = proof_packet(); - let mut out = vec![Fixture { - id: "proof", - title: "Nibu warns the player off the control", - note: "The committed proof packet, unchanged.", - source: "proof_packet() in crates/weksa/src/packet.rs", - packet: proof.clone(), - }]; - let mut add = |id, title, note, source, f: &dyn Fn(&mut Packet)| { - let mut packet = proof.clone(); - f(&mut packet); - out.push(Fixture { id, title, note, source, packet }); - }; - add( - "no-addressee", - "Same meaning, nobody addressed", - "With the addressee unset the player is a third party: no \"you\", no reflexive.", - "proof_packet() with discourse.addressee = None", - &|p| p.discourse.addressee = None, - ); - add( - "stance-held-back", - "Anger held back, restraint high", - "Stance is the speaker's felt affect. The lowerer has no row that reads it, so the text is the proof's text.", - "proof_packet() with stance anger=300, restraint=700", - &|p| p.stance.features = vec![grade("feature:9", 300), grade("feature:18", 700)], - ); - add( - "hedged", - "Hedged and uncertain warning", - "Hedge and certainty are meaning the lowerer cannot yet say. It refuses rather than drop them.", - "proof_packet() with discourse hedge=600, certainty=400", - &|p| p.discourse.features = vec![grade("feature:6", 600), grade("feature:7", 400)], - ); - add( - "negated", - "Negated intervention (polarity: negative)", - "Dropping the negation would say the opposite, so the lowerer refuses. This is the shape of a negated Dao line awaiting polarity.", - "proof_packet() with predication n5 polarity=negative", - &|p| { - p.predications[1].features = vec![FeatureValue { - feature: id("feature:1"), - value: FeatValue::Option("negative".parse().expect("label")), - }] - }, - ); - add( - "consent-authored", - "Addressee consent, authored source", - "Someone who saw the request says the addressee asked for it. No lowering speaks for that request yet.", - "proof_packet() with discourse.addressee_consent = n4 (source Authored)", - &|p| p.discourse.addressee_consent = Some(node("n4")), - ); - add( - "consent-imported", - "Addressee consent, imported-corpus source", - "A reader of wording cannot have seen a request, so the packet is refused at admission (V15), before any lowering.", - "proof_packet() with addressee_consent = n4 and provenance.source = ImportedCorpus", - &|p| { - p.discourse.addressee_consent = Some(node("n4")); - p.provenance.source = ProvenanceSource::ImportedCorpus; - }, - ); - add( - "relation", - "A causal relation between the predications", - "Relations are not rendered; the lowerer refuses instead of asserting a different meaning.", - "proof_packet() with relation:8 (causes) n4 -> n5", - &|p| { - p.relations = vec![weksa::packet::Relation { - relation: id("relation:8"), - from: node("n4"), - to: node("n5"), - features: vec![], - }] - }, - ); - add( - "dangling-filler", - "A role filled by a node that does not exist", - "Refused at admission (V4): the packet is not a packet.", - "proof_packet() with predication n5 agent filler n9", - &|p| { - p.predications[1].roles[0] = RoleFill { role: id("role:1"), filler: node("n9") } - }, - ); - add( - "unknown-entity", - "A referent naming an entity nobody holds", - "The entity store is the only owner of names; a packet cannot invent one.", - "proof_packet() with referent n3 entity = entity:nobody", - &|p| p.referents[2].entity = Some(EntityId::parse("entity:nobody").expect("entity")), - ); - add( - "common-noun-agent", - "The ship mind as a common noun", - "No entity: the referent lowers as \"the ship mind\", and there is no name to pronounce.", - "proof_packet() with referent n3 entity = None", - &|p| p.referents[2].entity = None, - ); - out -} - -fn gloss(inv: &Inventory, rev: u32, i: &InvId) -> Value { - match inv.resolve(i, rev) { - Some(e) => json!({ "id": i.to_string(), "gloss": e.gloss }), - None => json!({ "id": i.to_string(), "gloss": null }), - } -} - -fn feature_json(inv: &Inventory, rev: u32, slot: &str, at: &str, f: &FeatureValue) -> Value { - let mut g = gloss(inv, rev, &f.feature); - g["slot"] = json!(slot); - g["at"] = json!(at); - g["value"] = serde_json::to_value(&f.value).expect("value"); - g -} - -/// Every feature the packet carries, with where it sits. -fn features(p: &Packet, inv: &Inventory) -> Vec { - let rev = p.inventory_rev; - let mut v = Vec::new(); - for f in &p.discourse.features { - v.push(feature_json(inv, rev, "discourse", "discourse", f)); - } - for f in &p.stance.features { - v.push(feature_json(inv, rev, "stance", "stance", f)); - } - for r in &p.referents { - for f in &r.features { - v.push(feature_json(inv, rev, "referent", r.id.as_str(), f)); - } - } - for pr in &p.predications { - for f in &pr.features { - v.push(feature_json(inv, rev, "predication", pr.id.as_str(), f)); - } - } - v -} - -/// Ids the packet chose, once each, with the inventory's gloss at its revision. -fn chosen(p: &Packet, inv: &Inventory) -> Vec { - let rev = p.inventory_rev; - let mut ids: Vec = Vec::new(); - let mut push = |i: &InvId| { - if !ids.contains(i) { - ids.push(*i) - } - }; - p.context.setting.iter().for_each(&mut push); - p.context.medium.iter().for_each(&mut push); - p.discourse.speech_act.iter().for_each(&mut push); - for r in &p.referents { - push(&r.kind); - push(&r.concept); - } - for pr in &p.predications { - push(&pr.concept); - push(&pr.frame); - pr.roles.iter().for_each(|f| push(&f.role)); - } - p.relations.iter().for_each(|r| push(&r.relation)); - features(p, inv) - .iter() - .filter_map(|f| f["id"].as_str().and_then(|s| s.parse().ok())) - .for_each(|i: InvId| push(&i)); - ids.iter().map(|i| gloss(inv, rev, i)).collect() -} - -fn slots(p: &Packet) -> Value { - let j = |v: &dyn erased::Ser| v.to_value(); - json!({ - "meaning": { - "referents": p.referents.iter().map(|r| j(r)).collect::>(), - "predications": p.predications.iter().map(|r| j(r)).collect::>(), - "relations": p.relations.iter().map(|r| j(r)).collect::>(), - }, - "stance": j(&p.stance), - "discourse": j(&p.discourse), - "context": j(&p.context), - "extensions": p.extensions.iter().map(|r| j(r)).collect::>(), - "constraints": j(&p.constraints), - "provenance": { "source": j(&p.provenance.source), "corpus_item": p.provenance.corpus_item.to_string(), "confidence": p.provenance.confidence.get() }, - "kind": j(&p.kind), - "inventory_rev": p.inventory_rev, - }) -} - -mod erased { - pub trait Ser { - fn to_value(&self) -> serde_json::Value; - } - impl Ser for T { - fn to_value(&self) -> serde_json::Value { - serde_json::to_value(self).expect("serialize") - } - } -} - -fn lower_error(e: &LowerError) -> Value { - let (variant, reason) = match e { - LowerError::Unrendered { what } => ("Unrendered", format!("the packet carries {what}, which the lowerer does not render")), - LowerError::NoRow { id } => ("NoRow", format!("no table row for {id}")), - LowerError::GlossMismatch { id, row } => ("GlossMismatch", format!("{id} is not the {row} the row speaks for")), - LowerError::UnknownRole { frame, role } => ("UnknownRole", format!("{frame} names role {role}, which the inventory lacks")), - LowerError::MissingRole { predication, role } => ("MissingRole", format!("{predication} fills no {role} role")), - LowerError::UnknownEntity { entity } => ("UnknownEntity", format!("the entity store does not hold {entity}")), - LowerError::NoForm { entity } => ("NoForm", format!("{entity} has no en-US form")), - LowerError::NoPronunciation { entity } => ("NoPronunciation", format!("{entity} has no ruled en-US pronunciation")), - LowerError::NoEntityReferent => ("NoEntityReferent", "the packet cites no entity".to_string()), - LowerError::UnknownPhone { entity, at, .. } => ("UnknownPhone", format!("the ruled pronunciation of {entity} holds a symbol at {at} with no listed phone")), - LowerError::DanglingStress { entity, at } => ("DanglingStress", format!("the ruled pronunciation of {entity} has a stress mark at {at} with no vowel after it")), - LowerError::ConsentMissing { predication } => ("ConsentMissing", format!("{predication} is a request the packet does not carry the addressee's consent to")), - LowerError::AskerNotAddressee { predication } => ("AskerNotAddressee", format!("the agent of {predication} is not the addressee")), - LowerError::EntityRead { entity, .. } => ("EntityRead", format!("the entity store failed reading {entity}")), - LowerError::SceneUnread => ("SceneUnread", "the view's scene was never read".to_string()), - LowerError::SpeakerTokenTaken => ("SpeakerTokenTaken", "a referent cites the entity id the speaker token uses".to_string()), - }; - json!({ "variant": variant, "reason": reason }) -} - -struct World { - inventory: InventoryStore, - corpus: CorpusStore, - entities: EntityStore, - inv: Inventory, -} - -/// Admit `packet` through a fresh packet store and lower it. `Ok` carries the -/// text and the IPA outcome; `Err` is the refusal, as JSON. -fn run(w: &World, dir: &Path, packet: &Packet) -> Result<(String, Value), Value> { - let mut store = PacketStore::open(&dir.join("p.cc")).expect("packet store"); - let view = match store.admit(packet.clone(), &w.inventory, &w.corpus, &w.entities) { - Ok(v) => v, - Err(StoreError::Invalid(rs)) => { - return Err(json!({ - "kind": "admission_refused", - "refusals": rs.iter().map(|r| json!({ "rule": r.rule(), "detail": r.to_string() })).collect::>(), - })); - } - Err(e) => { - return Err(json!({ "kind": "admission_refused", "refusals": [{ "rule": "store", "detail": e.to_string() }] })); - } - }; - let text = lower(&view, &w.inv, &w.entities).map_err(|e| { - json!({ "kind": "lower_error", "error": lower_error(&e) }) - })?; - let ipa = match pronounce(&view, &w.entities) { - Ok(v) => json!({ "ok": v }), - Err(e) => json!({ "error": lower_error(&e) }), - }; - Ok((text, ipa)) -} - -fn main() { - let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); - let inventory = InventoryStore::open(&root.join("data/interlingua/inventory.cc")).expect("inventory"); - let inv = inventory.inventory().expect("inventory view"); - let w = World { - corpus: CorpusStore::open(&root.join("data/corpus/corpus.cc")).expect("corpus"), - entities: EntityStore::open(&root.join("data/entity/entity.cc")).expect("entities"), - inventory, - inv, - }; - let dir = std::env::temp_dir().join(format!("show-working-{}", std::process::id())); - std::fs::create_dir_all(&dir).expect("scratch dir"); - let mut out = Vec::new(); - for fx in fixtures() { - let p = &fx.packet; - let first = run(&w, &dir, p); - let outcome = match first { - Err(e) if e["kind"] == "admission_refused" => e, - first => { - // Ablation: run again with each feature removed, against the - // baseline outcome. It is how the page says whether the lowerer - // read a feature, instead of the page asserting it. - let base = first.as_ref().ok().map(|(t, _)| t.clone()); - let n_disc = p.discourse.features.len(); - let n_stance = p.stance.features.len(); - let total = features(p, &w.inv).len(); - let mut effects = Vec::new(); - for k in 0..total { - let mut q = p.clone(); - if k < n_disc { - q.discourse.features.remove(k); - } else if k < n_disc + n_stance { - q.stance.features.remove(k - n_disc); - } else { - let mut at = k - n_disc - n_stance; - 'o: { - for r in &mut q.referents { - if at < r.features.len() { r.features.remove(at); break 'o; } - at -= r.features.len(); - } - for pr in &mut q.predications { - if at < pr.features.len() { pr.features.remove(at); break 'o; } - at -= pr.features.len(); - } - } - } - let _ = std::fs::remove_file(dir.join("p.cc")); - effects.push(match (&base, run(&w, &dir, &q)) { - (Some(t), Ok((u, _))) if *t == u => "Without it the text is identical: the lowerer did not read it.", - (Some(_), Ok(_)) => "Without it the text changes: the lowerer read it.", - (Some(_), Err(_)) => "Without it the packet is refused.", - (None, Ok(_)) => "Without it the packet lowers: this feature is what the lowerer refuses.", - (None, Err(_)) => "Still refused without this feature alone.", - }); - } - match first { - Ok((text, ipa)) => json!({ "kind": "lowered", "text": text, "ipa": ipa, "feature_effects": effects }), - Err(e) => { let mut e = e; e["feature_effects"] = json!(effects); e } - } - } - }; - let _ = std::fs::remove_file(dir.join("p.cc")); - let mut slots = slots(p); - slots["features"] = json!(features(p, &w.inv)); - out.push(json!({ - "id": fx.id, - "title": fx.title, - "note": fx.note, - "source": fx.source, - "packet_id": p.id.to_string(), - "corpus_item": p.provenance.corpus_item.to_string(), - "slots": slots, - "chosen": chosen(p, &w.inv), - "outcome": outcome, - })); - } - let _ = std::fs::remove_dir_all(&dir); - println!("{}", serde_json::to_string_pretty(&json!({ "fixtures": out })).expect("json")); -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn refusal_carries_the_closed_reason_never_a_debug_of_the_error() { - // UnknownPhone holds the ruled IPA text; no field of the output may. - let e = LowerError::UnknownPhone { - entity: EntityId::parse("entity:nibu").expect("entity id"), - at: 3, - text: "CANARY-IPA".into(), - }; - let v = lower_error(&e); - assert!(!v.to_string().contains("CANARY")); - let keys: Vec<_> = v.as_object().expect("object").keys().cloned().collect(); - assert_eq!(keys, ["reason", "variant"]); - } -} diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index 0ecd067..5b64758 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -759,6 +759,9 @@ impl Prompt { } } +#[cfg(test)] +mod mutants; + #[cfg(test)] mod tests { use super::*; diff --git a/crates/weksa/src/context/mutants.rs b/crates/weksa/src/context/mutants.rs new file mode 100644 index 0000000..876816f --- /dev/null +++ b/crates/weksa/src/context/mutants.rs @@ -0,0 +1,14 @@ +//! Pins for private reach of `context.rs` that no run can produce. + +use super::*; + +/// A corpus with no live Train item of the panel's languages holds an empty +/// train. (A run refuses a panel without Train items at open, so only the +/// builder shows it.) +#[test] +fn no_train_item_of_the_panel_is_an_empty_train() { + let dir = tempfile::tempdir().unwrap(); + let corpus = CorpusStore::open(&dir.path().join("c.cc")).unwrap(); + let train = TrainItems::from_corpus(&corpus, &["en-US".to_string()]).unwrap(); + assert_eq!((train.len(), train.is_empty()), (0, true)); +} diff --git a/crates/weksa/src/corpus.rs b/crates/weksa/src/corpus.rs index 4afa662..1fb7fa1 100644 --- a/crates/weksa/src/corpus.rs +++ b/crates/weksa/src/corpus.rs @@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize}; use crate::form::{FormError, is_slug, string_form}; use crate::packet::EntityId; +use crate::speaker::Annotator; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub struct Turn { @@ -103,6 +104,11 @@ pub struct CorpusItem { /// Which entities the text names, and where. Fixed with the text at intake. #[cultcache(key = 10)] pub names: Vec, + /// Who placed `names`: the annotator of the intake that wrote them. `None` + /// when nothing records one (the cast seeds, `weksa corpus add`, the Dao De + /// Jing bed); it labels the spans, never the line or its speaker. + #[cultcache(key = 11, default)] + pub names_by: Option, } impl CorpusItem { diff --git a/crates/weksa/src/encode.rs b/crates/weksa/src/encode.rs index 1ddabd3..500d575 100644 --- a/crates/weksa/src/encode.rs +++ b/crates/weksa/src/encode.rs @@ -7,7 +7,8 @@ //! provenance, constraints and extensions, and an empty trace, so a model that //! saw a Validation item cannot hand its words to whoever reads packets. The //! editor writes family, gloss, value kind and slots only; `into_revision` sets -//! every id and revision number. +//! every id and revision number. The public-domain intake file (`WorkDraft`) and +//! its name-span answer (`MarkedLine`) are read here too. use std::fmt; use std::num::NonZeroU32; @@ -15,7 +16,12 @@ use std::num::NonZeroU32; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; -use crate::corpus::CorpusItemId; +use unicode_normalization::UnicodeNormalization; + +use crate::corpus::{CorpusItem, CorpusItemId, NameSpan, Source, item_key}; +use crate::entity::{Entity, EntityOrigin, LanguageForms}; +use crate::form::is_slug; +use crate::packet::EntityId; use crate::inventory::{ Change, Family, InvId, Inventory, InventoryEntry, InventoryRevision, Slot, ValueKind, }; @@ -358,6 +364,196 @@ impl LoweringDraft { } } +/// One cast member of a work: the entity and the forms the work spells it with, +/// in the work's language, citation form first. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CastDraft { + pub entity: EntityId, + pub forms: Vec, +} + +/// One line of a work: its scene group, the speaker its speech prefix names, text. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LineDraft { + pub group: String, + pub speaker: EntityId, + pub text: String, +} + +/// A transcription of a public-domain work in hand: provenance, cast forms, +/// lines, scene groups and the speaker each speech prefix names. It holds no +/// distributor header, licence or trademark text, and nothing reads a URL. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WorkDraft { + pub slug: String, + pub language: String, + pub admitted_on: String, + pub source: Source, + pub cast: Vec, + pub lines: Vec, +} + +/// One line of a name-span answer: the line number and the line copied with +/// `[[entity:|
]]` around each place it names a cast member. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MarkedLine { + pub line: u32, + pub marked: String, +} + +impl MarkedLine { + pub fn parse(answer: &str) -> Result { + from_json(answer) + } +} + +/// Why an import was refused before any document was written. The text names +/// the kind and the line number only, never a value. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ImportRefusal { + NotPublicDomain, + CastRepeated, + NoSuchLine { line: u32 }, + LineMarkedTwice { line: u32 }, + Group { line: u32 }, + SpeakerNotInCast { line: u32 }, + Markup { line: u32 }, + NotInCast { line: u32 }, +} + +impl fmt::Display for ImportRefusal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NotPublicDomain => write!(f, "the work's source is not public domain"), + Self::CastRepeated => write!(f, "a cast entity is listed twice"), + Self::NoSuchLine { line } => write!(f, "the answer marks a line that is not there (line {line})"), + Self::LineMarkedTwice { line } => write!(f, "line {line} is marked twice"), + Self::Group { line } => write!(f, "the group of line {line} is not a slug"), + Self::SpeakerNotInCast { line } => write!(f, "the speaker of line {line} is not in the cast"), + Self::Markup { line } => write!(f, "the marked text of line {line} is not its text with markup"), + Self::NotInCast { line } => write!(f, "line {line} marks an entity outside the cast"), + } + } +} + +impl std::error::Error for ImportRefusal {} + +impl WorkDraft { + pub fn parse(file: &str) -> Result { + from_json(file) + } + + /// The cast entities and the corpus items for the lines `marked` covers, in + /// line order, each item's spans read from its markup and labelled + /// `annotator`. Decides everything before any write; whether the documents + /// are admissible is `EntityStore::put` and `CorpusStore::add`'s to refuse. + pub fn into_import( + self, + marked: &[MarkedLine], + annotator: &Annotator, + ) -> Result<(Vec, Vec), ImportRefusal> { + let Source::PublicDomain { work, .. } = &self.source else { + return Err(ImportRefusal::NotPublicDomain); + }; + let mut entities = Vec::new(); + for c in &self.cast { + if entities.iter().any(|e: &Entity| e.id == c.entity) { + return Err(ImportRefusal::CastRepeated); + } + entities.push(Entity { + id: c.entity.clone(), + forms: vec![LanguageForms { + language: self.language.clone(), + forms: c.forms.iter().map(|f| f.nfc().collect()).collect(), + pronunciation: None, + }], + origin: EntityOrigin::PublicDomain { work: work.clone() }, + }); + } + let in_cast = |id: &EntityId| entities.iter().any(|e| &e.id == id); + let mut answer: Vec<&MarkedLine> = marked.iter().collect(); + answer.sort_by_key(|m| m.line); + let mut items = Vec::new(); + for (i, m) in answer.iter().enumerate() { + let line = m.line; + if line == 0 || line as usize > self.lines.len() { + return Err(ImportRefusal::NoSuchLine { line }); + } + if i > 0 && answer[i - 1].line == line { + return Err(ImportRefusal::LineMarkedTwice { line }); + } + let draft = &self.lines[line as usize - 1]; + let group = format!("{}-{}", self.slug, draft.group); + if !is_slug(&group) { + return Err(ImportRefusal::Group { line }); + } + if !in_cast(&draft.speaker) { + return Err(ImportRefusal::SpeakerNotInCast { line }); + } + let text: String = draft.text.nfc().collect(); + if text.contains("[[") || text.contains("]]") { + return Err(ImportRefusal::Markup { line }); + } + let marked_nfc: String = m.marked.nfc().collect(); + let (plain, names) = match unmark(&marked_nfc) { + Some((plain, names)) if plain == text => (plain, names), + _ => return Err(ImportRefusal::Markup { line }), + }; + if names.iter().any(|n| !in_cast(&n.entity)) { + return Err(ImportRefusal::NotInCast { line }); + } + items.push(CorpusItem { + id: item_key(&self.slug, line), + text: plain, + language: self.language.clone(), + speaker: draft.speaker.clone(), + group, + prior_turns: Vec::new(), + source: self.source.clone(), + admitted_on: self.admitted_on.clone(), + supersedes: None, + withdrawn: false, + names, + names_by: Some(annotator.clone()), + }); + } + Ok((entities, items)) + } +} + +/// The text of `marked` with `[[entity|form]]` replaced by `form`, and the +/// byte span of each form in that text. `None` for any malformed markup. The +/// only search is for the markup delimiters. +fn unmark(marked: &str) -> Option<(String, Vec)> { + let mut out = String::with_capacity(marked.len()); + let mut names = Vec::new(); + let mut rest = marked; + while let Some(at) = rest.find("[[") { + out.push_str(&rest[..at]); + let inner = &rest[at + 2..]; + let (id, after) = inner.split_once('|')?; + let (form, tail) = after.split_once("]]")?; + if form.is_empty() || form.contains("[[") || id.contains("]]") { + return None; + } + let entity = EntityId::parse(id).ok()?; + let start = u32::try_from(out.len()).ok()?; + out.push_str(form); + names.push(NameSpan { + entity, + start, + end: u32::try_from(out.len()).ok()?, + }); + rest = tail; + } + out.push_str(rest); + Some((out, names)) +} + /// `encoder:`, a digest of the model id and the prompt digest. pub fn encoder_id(model: &str, prompt_digest: &str) -> EncoderId { let digest = fnv1a64(format!("weksa-encoder-v1:{model}:{prompt_digest}").as_bytes()); @@ -370,6 +566,9 @@ pub fn packet_id(item: &CorpusItemId, rev: u32, encoder: &EncoderId) -> PacketId PacketId::parse(&format!("packet:{}", hex16(digest))).expect("a packet id is well formed") } +#[cfg(test)] +mod mutants; + #[cfg(test)] mod tests { use super::*; @@ -746,4 +945,145 @@ mod tests { assert!(entries.iter().all(|e| e.annotator == by)); assert_eq!(entries[1].examples, ["x"]); } + + const WORK: &str = r#"{"slug":"lantern-wager","language":"en-US","admitted_on":"2026-11-13", + "source":{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}}, + "cast":[{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:tamsin","forms":["Tamsin","Tam"]},{"entity":"entity:wexmoor","forms":["Wexmoor"]}], + "lines":[{"group":"s1","speaker":"entity:orsa","text":"Tamsin, bring the lantern."}, + {"group":"s1","speaker":"entity:tamsin","text":"Orsa, I will not, said Tam."}, + {"group":"s2","speaker":"entity:wexmoor","text":"Quiet, both of you."}]}"#; + + fn marked(line: u32, text: &str) -> MarkedLine { + MarkedLine { line, marked: text.into() } + } + + fn model_annotator() -> Annotator { + Annotator::Model { + family: crate::corpus::ModelFamily::Anthropic, + generator: "g".into(), + } + } + + fn import( + work: &str, + answer: &[MarkedLine], + ) -> Result<(Vec, Vec), ImportRefusal> { + WorkDraft::parse(work).unwrap().into_import(answer, &model_annotator()) + } + + #[test] + fn a_work_imports_its_cast_and_marked_lines() { + let answer = [ + marked(1, "[[entity:tamsin|Tamsin]], bring the lantern."), + marked(2, "[[entity:orsa|Orsa]], I will not, said [[entity:tamsin|Tam]]."), + ]; + let (entities, items) = import(WORK, &answer).unwrap(); + assert_eq!(entities.len(), 3); + for e in &entities { + assert!(matches!(&e.origin, EntityOrigin::PublicDomain { work } if work == "The Lantern Wager")); + assert_eq!(e.forms[0].language, "en-US"); + } + assert_eq!(items.len(), 2); + assert_eq!(items[0].id, "item:lantern-wager:1"); + assert_eq!(items[1].id, "item:lantern-wager:2"); + assert_eq!(items[0].group, "lantern-wager-s1"); + let span = |e: &str, text: &str, form: &str| { + let start = text.find(form).unwrap() as u32; + NameSpan { entity: EntityId::parse(e).unwrap(), start, end: start + form.len() as u32 } + }; + let t2 = &items[1].text; + assert_eq!(items[0].names, [span("entity:tamsin", &items[0].text, "Tamsin")]); + assert_eq!( + items[1].names, + [span("entity:orsa", t2, "Orsa"), span("entity:tamsin", t2, "Tam")] + ); + // "will" is unmarked and unspanned; the line's speaker is the file's. + assert_eq!(items[1].speaker, EntityId::parse("entity:tamsin").unwrap()); + for i in &items { + assert_eq!(i.names_by, Some(model_annotator())); + assert!(i.prior_turns.is_empty() && i.supersedes.is_none() && !i.withdrawn); + } + } + + #[test] + fn an_import_refuses_before_any_document() { + let ok = marked(1, "Tamsin, bring the lantern."); + let refused = |work: &str, answer: &[MarkedLine]| import(work, answer).unwrap_err(); + let cast_authored = WORK.replace( + r#"{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}}"#, + r#""OperatorWritten""#, + ); + assert_eq!(refused(&cast_authored, std::slice::from_ref(&ok)), ImportRefusal::NotPublicDomain); + let repeated = WORK.replace(r#"{"entity":"entity:orsa","forms":["Orsa"]}"#, r#"{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:orsa","forms":["O"]}"#); + assert_eq!(refused(&repeated, std::slice::from_ref(&ok)), ImportRefusal::CastRepeated); + let stranger = WORK.replace(r#""speaker":"entity:orsa""#, r#""speaker":"entity:stranger""#); + assert_eq!(refused(&stranger, std::slice::from_ref(&ok)), ImportRefusal::SpeakerNotInCast { line: 1 }); + assert_eq!(refused(WORK, &[marked(9, "x")]), ImportRefusal::NoSuchLine { line: 9 }); + assert_eq!(refused(WORK, &[marked(0, "x")]), ImportRefusal::NoSuchLine { line: 0 }); + assert_eq!( + refused(WORK, &[ok.clone(), ok.clone()]), + ImportRefusal::LineMarkedTwice { line: 1 } + ); + for bad in [ + "[[entity:tamsin|Tamsin, bring the lantern.", + "[[entity:tamsin Tamsin]], bring the lantern.", + "[[tamsin|Tamsin]], bring the lantern.", + "[[entity:tamsin|]]Tamsin, bring the lantern.", + "Tamsin, bring the lantern", + ] { + assert_eq!(refused(WORK, &[marked(1, bad)]), ImportRefusal::Markup { line: 1 }, "{bad}"); + } + assert_eq!( + refused(WORK, &[marked(1, "[[entity:stranger|Tamsin]], bring the lantern.")]), + ImportRefusal::NotInCast { line: 1 } + ); + let group = WORK.replace(r#""group":"s1""#, r#""group":"S 1""#); + assert_eq!(refused(&group, &[ok]), ImportRefusal::Group { line: 1 }); + let bracketed = WORK.replace("Tamsin, bring the lantern.", "Tamsin [[bring]] the lantern."); + assert_eq!( + refused(&bracketed, &[marked(1, "[[entity:tamsin|Tamsin]] [[entity:orsa|bring]] the lantern.")]), + ImportRefusal::Markup { line: 1 } + ); + // The refusal text names the kind and the line, no value. + assert_eq!( + ImportRefusal::Markup { line: 1 }.to_string(), + "the marked text of line 1 is not its text with markup" + ); + } + + #[test] + fn markup_offsets_are_bytes_of_the_nfc_text() { + let work = WORK + .replace("Tamsin, bring the lantern.", "Caf\u{e9} Zo\u{eb}") + .replace(r#""forms":["Orsa"]"#, r#""forms":["Zoe\u0308"]"#); + let decomposed = "Cafe\u{301} [[entity:orsa|Zoe\u{308}]]"; + let (entities, items) = import(&work, &[marked(1, decomposed)]).unwrap(); + assert_eq!(entities[0].forms[0].forms, ["Zo\u{eb}"]); + let item = &items[0]; + assert_eq!(item.text, "Caf\u{e9} Zo\u{eb}"); + let n = &item.names[0]; + assert_eq!(&item.text[n.start as usize..n.end as usize], "Zo\u{eb}"); + assert_eq!(n.start, 6); + } + + #[test] + fn a_stray_close_bracket_and_a_scattered_duplicate_answer_are_refused() { + // A text holding only `]]` is not a marked line. + let close = WORK.replace("Quiet, both of you.", "Quiet ]] both of you."); + assert_eq!( + import(&close, &[marked(3, "Quiet ]] both of you.")]).unwrap_err(), + ImportRefusal::Markup { line: 3 } + ); + // The answer's order does not matter, and a duplicate need not be adjacent. + let (_, items) = import(WORK, &[marked(3, "Quiet, both of you."), marked(1, "Tamsin, bring the lantern.")]).unwrap(); + assert_eq!(items.iter().map(|i| i.id.as_str()).collect::>(), ["item:lantern-wager:1", "item:lantern-wager:3"]); + assert_eq!( + import( + WORK, + &[marked(1, "Tamsin, bring the lantern."), marked(3, "Quiet, both of you."), marked(1, "Tamsin, bring the lantern.")] + ) + .unwrap_err(), + ImportRefusal::LineMarkedTwice { line: 1 } + ); + } } diff --git a/crates/weksa/src/encode/mutants.rs b/crates/weksa/src/encode/mutants.rs new file mode 100644 index 0000000..344c2a9 --- /dev/null +++ b/crates/weksa/src/encode/mutants.rs @@ -0,0 +1,13 @@ +//! Pins for `unmark` that the import door only reaches through a whole work. + +use super::*; + +/// A form that opens another markup is malformed, though its entity id is +/// sound: the model wrote a nested or unclosed mark, not a name. +#[test] +fn a_form_that_opens_a_mark_is_malformed() { + assert_eq!(unmark("[[entity:orsa|Or[[sa]] came"), None); + // The same line without the nested opener unmarks. + let (plain, names) = unmark("[[entity:orsa|Orsa]] came").unwrap(); + assert_eq!((plain.as_str(), names.len()), ("Orsa came", 1)); +} diff --git a/crates/weksa/src/lib.rs b/crates/weksa/src/lib.rs index 2510fae..77026ca 100644 --- a/crates/weksa/src/lib.rs +++ b/crates/weksa/src/lib.rs @@ -31,7 +31,7 @@ pub use inventory::{ Change, Family, InvId, Inventory, InventoryEntry, InventoryRevision, OptionLabel, OptionSet, Slot, ValueKind, }; -pub use lower::{EntityForms, LowerError, PhoneLine, TARGET_LANGUAGE, lower, phones, pronounce}; +pub use lower::{EntityForms, LowerError, PhoneLine, TARGET_LANGUAGE, phones, pronounce}; pub use model::{AnswerFiles, ModelPort}; pub use ontology::{Condition, Layer, Ontology, OntologyEntry, Test, select}; pub use packet::{ diff --git a/crates/weksa/src/lower.rs b/crates/weksa/src/lower.rs index 12fbb61..411172d 100644 --- a/crates/weksa/src/lower.rs +++ b/crates/weksa/src/lower.rs @@ -1,52 +1,28 @@ -//! The English template lowerer: the validated lowering view of a packet to -//! text, by a fixed table keyed by inventory ids. It is the deterministic -//! lowerer of `weksa lower`, the proof fixtures and their tests; the golf loop -//! lowers through the `ModelPort` (`run.rs`, `Prompt::lower`), and follow-up -//! `table-retirement` decides this table's future. It renders predications, referents and the -//! speech act, and nothing else: no style, no model, no corpus text. It takes a -//! [`LoweringInput`], so it cannot see the trace or the corpus item. A packet -//! carrying a feature, relation, extension, role fill or referent it does not -//! render is refused, because rendering part of a meaning would assert a different -//! one (`polarity: negative` dropped says the opposite). Context, kind and -//! constraints steer delivery and are not rendered; the concept of a predication -//! must resolve but the frame owns the verb. +//! The entity port and the ruled pronunciation of a packet's entities. //! -//! A referent that cites an entity renders as that entity's own form, read -//! through the [`EntityForms`] port: the first (citation) form stored for -//! [`TARGET_LANGUAGE`], verbatim. The choice is a table lookup on the language -//! tag and the form's position, not a reading of the form. Forms after the -//! first are inflected or particle-bound shapes; no English frame asks for one, -//! so none is selected. An entity the port does not hold, or one with no form in -//! the target language, is refused: the concept word would say a different -//! thing than the name the packet cites. -//! -//! A request clause ("You asked Nibu to stop.") says what the addressee did, so -//! it is gated on the packet's own record of it: `Discourse.addressee_consent` -//! (set only by authoring or game telemetry, V15) must name that predication, -//! and its agent must be the addressee. Without it the lowerer refuses -//! ([`LowerError::ConsentMissing`], [`LowerError::AskerNotAddressee`]); it does -//! not read wording or guess who asked. A consent naming any other predication -//! is an unrendered slot. -//! -//! The table is a temporary owner. Each row speaks for an inventory entry and -//! names the gloss it speaks for, so a row cannot outlive a change of what its id -//! means. `language-ontology` moves the rows into ontology documents. +//! Wording of a packet in a target language is the `ModelPort`'s answer to +//! `Prompt::lower` (`context.rs`); nothing here produces lowered text. This +//! module holds what the entity store owns and the lowering path reads: the +//! narrow [`EntityForms`] port, the ruled IPA of every entity a packet cites +//! ([`pronounce`]), and its segmentation into phones ([`phones`]). A referent +//! that cites an entity is read through the port; an entity the port does not +//! hold, one with no form or no ruled pronunciation in [`TARGET_LANGUAGE`], is +//! refused, because spelling is not a pronunciation and there is no +//! grapheme-to-phoneme guess to offer in its place. use std::fmt; use serde::Serialize; use crate::entity::Entity; -use crate::inventory::{Family, InvId, Inventory}; -use crate::packet::{EntityId, NodeId, Predication}; +use crate::packet::EntityId; use crate::phones::{Phone, PhoneInventory, SegmentError}; use crate::store::EntityStore; use crate::validate::LoweringInput; -/// The language tag of the template table and the phones: what `weksa lower` -/// says in. The golf loop lowers each item in its own language and does not -/// read this. Spelled as an entity's `LanguageForms::language` and a corpus -/// item's language are. +/// The language tag of [`pronounce`] and [`phones`] only. The golf loop lowers +/// each item in its own language and does not read this. Spelled as an +/// entity's `LanguageForms::language` and a corpus item's language are. pub const TARGET_LANGUAGE: &str = "en-US"; /// The lowerer's whole view of the entity store: one entity by id. Narrow so a @@ -62,98 +38,8 @@ impl EntityForms for EntityStore { } } -/// `(inventory id, the gloss the row speaks for, surface)`. -type Row = (&'static str, &'static str, &'static str); - -/// Referent concepts, as the noun phrase after "the". -const CONCEPTS: &[Row] = &[ - ("concept:3", "player", "player"), - ("concept:4", "control", "control"), - ("concept:5", "ship_mind", "ship mind"), - // The Dao De Jing test bed (`daodejing.rs`). Nouns, then the verbs a - // `{verb}` frame speaks. - ("concept:7", "sage", "sage"), - ("concept:8", "mind", "minds"), - ("concept:9", "belly", "bellies"), - ("concept:10", "will", "wills"), - ("concept:11", "bone", "bones"), - ("concept:12", "empty", "empties"), - ("concept:13", "fill", "fills"), - ("concept:14", "weaken", "weakens"), - ("concept:15", "strengthen", "strengthens"), - ("concept:16", "heaven", "heaven"), - ("concept:17", "earth", "earth"), - ("concept:18", "is_long_enduring", "is long-enduring"), - ("concept:19", "continues_long", "continues long"), - ("concept:20", "highest_excellence", "highest excellence"), - ("concept:21", "water", "water"), - ("concept:22", "true_words", "words that are strictly true"), - ("concept:23", "seem_paradoxical", "seem paradoxical"), - ( - "concept:24", - "space_between_heaven_and_earth", - "space between heaven and earth", - ), - ("concept:25", "bellows", "bellows"), - ("concept:26", "compare", "compare"), - ("concept:27", "resemble", "resemble"), -]; - -/// Frames: the clause a predication of that frame says. `{gloss}` is the filler -/// of the role whose inventory gloss it is. -const FRAMES: &[Row] = &[ - ( - "frame:1", - "hazard_warning", - "{agent} may endanger {patient} by touching {instrument}", - ), - ("frame:2", "intervention", "{agent} will stop {patient}"), - ( - "frame:3", - "request_intervention", - "{agent} asked {patient} to stop", - ), - // The Dao De Jing test bed. Frame 4 was written from a Train chapter, 5 and - // 6 from Validation chapters, 7 and 8 from Test chapters. - ("frame:4", "act", "{agent} {verb} {patient}"), - ("frame:5", "intransitive", "{experiencer} {verb}"), - ("frame:6", "likeness", "{experiencer} is like {goal}"), - ( - "frame:7", - "seems_paradoxical", - "{experiencer} seem paradoxical", - ), - ( - "frame:8", - "may_be_compared", - "{experiencer} may be compared to {goal}", - ), -]; - -/// The frame of "{agent} asked {patient} to stop": the one frame that reads -/// the discourse's addressee consent. -const REQUEST_FRAME: &str = "frame:3"; - -/// Speech acts: the opener of the first sentence. -const SPEECH_ACTS: &[Row] = &[ - ("speech_act:1", "assert", ""), - ("speech_act:2", "ask", ""), - ("speech_act:5", "warn", "Warning: "), -]; - #[derive(Clone, Debug, PartialEq, Eq)] pub enum LowerError { - /// The packet carries a slot the lowerer does not render. - Unrendered { what: &'static str }, - /// No table row for an id the packet uses. - NoRow { id: InvId }, - /// The row's gloss is not the gloss the inventory gives its id at the - /// packet's revision, or the id does not resolve there. - GlossMismatch { id: InvId, row: &'static str }, - /// A frame template names a role the inventory has no live entry for. - UnknownRole { frame: InvId, role: String }, - /// The predication fills no role its frame's template needs. - MissingRole { predication: NodeId, role: String }, /// A referent cites an entity the entity port does not hold. UnknownEntity { entity: EntityId }, /// The entity has no form in [`TARGET_LANGUAGE`]. @@ -165,7 +51,7 @@ pub enum LowerError { NoEntityReferent, /// Phones were requested for an entity whose ruled IPA holds a symbol the /// language's phone inventory does not list (or the language has no - /// inventory). The lowerer does not guess a phone. + /// inventory). No phone is guessed. UnknownPhone { entity: EntityId, at: usize, @@ -174,12 +60,6 @@ pub enum LowerError { /// Phones were requested for an entity whose ruled IPA has a stress mark /// with no vowel after it. DanglingStress { entity: EntityId, at: usize }, - /// A request predication the packet does not carry the addressee's consent - /// to: no consent, or a consent that names another predication. The - /// lowerer does not guess who asked. - ConsentMissing { predication: NodeId }, - /// A request predication whose agent is not the discourse addressee. - AskerNotAddressee { predication: NodeId }, /// The entity port failed to read. EntityRead { entity: EntityId, why: String }, /// A lowering prompt was asked of a view whose scene was never read. @@ -192,11 +72,6 @@ pub enum LowerError { impl fmt::Display for LowerError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(match self { - LowerError::Unrendered { .. } => "packet carries a slot the lowerer does not render", - LowerError::NoRow { .. } => "no table row for an inventory id the packet uses", - LowerError::GlossMismatch { .. } => "table row does not match the inventory's gloss", - LowerError::UnknownRole { .. } => "frame template names a role the inventory lacks", - LowerError::MissingRole { .. } => "predication fills no role its frame needs", LowerError::UnknownEntity { .. } => "referent cites an entity the port does not hold", LowerError::NoForm { .. } => "entity has no form in the target language", LowerError::NoPronunciation { .. } => "entity has no ruled pronunciation", @@ -205,149 +80,17 @@ impl fmt::Display for LowerError { "ruled pronunciation holds a symbol with no listed phone" } LowerError::DanglingStress { .. } => "stress mark with no vowel after it", - LowerError::ConsentMissing { .. } => "request lacks the addressee's consent", - LowerError::AskerNotAddressee { .. } => "request agent is not the addressee", LowerError::EntityRead { .. } => "entity port failed to read", LowerError::SceneUnread => "the view's scene was never read", - LowerError::SpeakerTokenTaken => "a referent cites the entity id the speaker token uses", + LowerError::SpeakerTokenTaken => { + "a referent cites the entity id the speaker token uses" + } }) } } impl std::error::Error for LowerError {} -fn row( - table: &'static [Row], - id: InvId, - inv: &Inventory, - rev: u32, -) -> Result<&'static str, LowerError> { - let (_, gloss, surface) = table - .iter() - .find(|(key, _, _)| *key == id.to_string()) - .ok_or(LowerError::NoRow { id })?; - match inv.resolve(&id, rev) { - Some(e) if e.gloss == *gloss => Ok(surface), - _ => Err(LowerError::GlossMismatch { id, row: gloss }), - } -} - -pub fn lower( - input: &LoweringInput, - inv: &Inventory, - entities: &dyn EntityForms, -) -> Result { - let rev = input.inventory_rev(); - let unrendered = |what| Err(LowerError::Unrendered { what }); - if input.predications().is_empty() { - return unrendered("no predication"); - } - if !input.discourse().features.is_empty() - || input.referents().iter().any(|r| !r.features.is_empty()) - || input.predications().iter().any(|p| !p.features.is_empty()) - { - return unrendered("features"); - } - request_gate(input, inv)?; - if !input.relations().is_empty() { - return unrendered("relations"); - } - if !input.extensions().is_empty() { - return unrendered("extensions"); - } - let opener = match input.discourse().speech_act { - Some(act) => row(SPEECH_ACTS, act, inv, rev)?, - None => "", - }; - let mut sentences = Vec::new(); - for p in input.predications() { - sentences.push(clause(input, inv, entities, p)?); - } - let mentioned = |r: &NodeId| { - input - .predications() - .iter() - .any(|p| p.roles.iter().any(|f| &f.filler == r)) - }; - if input.referents().iter().any(|r| !mentioned(&r.id)) { - return unrendered("a referent no role mentions"); - } - let mut out = String::new(); - for (i, (clause, name_first)) in sentences.iter().enumerate() { - if i > 0 { - out.push(' '); - } - if i == 0 { - out.push_str(opener); - } - let mut chars = clause.chars(); - if let Some(first) = chars.next() { - // A name opens its clause as written; only template words are cased. - if (i == 0 && !opener.is_empty()) || *name_first { - out.push(first); - } else { - out.extend(first.to_uppercase()); - } - out.push_str(chars.as_str()); - } - out.push('.'); - } - Ok(out) -} - -/// The one place consent is read. A request clause ([`REQUEST_FRAME`]) says what -/// the addressee did, so it needs the packet's own record of it: the discourse -/// `addressee_consent`, which validation lets only authoring or game telemetry -/// set (V15), must name that predication, and its agent must be the addressee. -/// Nothing here reads wording. A consent naming any other predication is a slot -/// the lowerer does not render. -fn request_gate(input: &LoweringInput, inv: &Inventory) -> Result<(), LowerError> { - let consent = input.discourse().addressee_consent.as_ref(); - let requests = input - .predications() - .iter() - .filter(|p| p.frame.to_string() == REQUEST_FRAME); - for p in requests { - if consent != Some(&p.id) { - return Err(LowerError::ConsentMissing { - predication: p.id.clone(), - }); - } - let agent = inv - .entries() - .find(|e| { - e.family() == Family::Role && e.gloss == "agent" && e.live_at(input.inventory_rev()) - }) - .ok_or_else(|| LowerError::UnknownRole { - frame: p.frame, - role: "agent".to_string(), - })?; - let asker = p - .roles - .iter() - .find(|r| r.role == agent.id) - .map(|r| &r.filler); - if asker.is_none() || asker != input.discourse().addressee.as_ref() { - return Err(LowerError::AskerNotAddressee { - predication: p.id.clone(), - }); - } - } - match consent { - Some(_) - if !input - .predications() - .iter() - .any(|p| p.frame.to_string() == REQUEST_FRAME) => - { - Err(LowerError::Unrendered { - what: "addressee consent", - }) - } - _ => Ok(()), - } -} - /// The ruled IPA of every entity a packet cites, one per entity referent in /// the packet's referent order, as stored: bare, stress-marked, nothing /// derived. A rejected pronunciation is never read. An entity the port does not @@ -424,91 +167,6 @@ fn ruled( Ok(out) } -fn clause( - input: &LoweringInput, - inv: &Inventory, - entities: &dyn EntityForms, - p: &Predication, -) -> Result<(String, bool), LowerError> { - let rev = input.inventory_rev(); - let template = row(FRAMES, p.frame, inv, rev)?; - let mut said: Vec<&NodeId> = Vec::new(); - let mut out = String::new(); - let mut name_first = false; - let mut rest = template; - while let Some(open) = rest.find('{') { - out.push_str(&rest[..open]); - let close = rest[open..].find('}').map_or(rest.len(), |c| open + c); - let name = &rest[open + 1..close]; - rest = &rest[(close + 1).min(rest.len())..]; - if name == "verb" { - out.push_str(row(CONCEPTS, p.concept, inv, rev)?); - continue; - } - let role = inv - .entries() - .find(|e| e.family() == Family::Role && e.gloss == name && e.live_at(rev)) - .ok_or_else(|| LowerError::UnknownRole { - frame: p.frame, - role: name.to_string(), - })?; - let fill = - p.roles - .iter() - .find(|r| r.role == role.id) - .ok_or_else(|| LowerError::MissingRole { - predication: p.id.clone(), - role: name.to_string(), - })?; - let (phrase, is_name) = filler(input, inv, entities, &fill.filler, &said)?; - name_first |= is_name && out.is_empty(); - out.push_str(&phrase); - said.push(&fill.filler); - } - out.push_str(rest); - if p.roles.len() != said.len() { - return Err(LowerError::Unrendered { - what: "a role the frame does not name", - }); - } - Ok((out, name_first)) -} - -/// A role filler as a phrase: the addressee is "you", a predication is "that", -/// another referent is its entity's citation form when it cites one, else "the" -/// and its concept word. The addressee said twice in -/// one predication is reflexive; no other referent is, because the packet has no -/// animacy to choose "themself" or "itself" by. -fn filler( - input: &LoweringInput, - inv: &Inventory, - entities: &dyn EntityForms, - node: &NodeId, - said: &[&NodeId], -) -> Result<(String, bool), LowerError> { - let again = said.contains(&node); - if let Some(r) = input.referents().iter().find(|r| &r.id == node) { - if input.discourse().addressee.as_ref() == Some(node) { - return Ok((if again { "yourself" } else { "you" }.to_string(), false)); - } - if let Some(id) = &r.entity { - return Ok((name(entities, id)?, true)); - } - let word = row(CONCEPTS, r.concept, inv, input.inventory_rev())?; - return Ok((format!("the {word}"), false)); - } - Ok(("that".to_string(), false)) -} - -/// The entity's citation form in [`TARGET_LANGUAGE`]. -fn name(entities: &dyn EntityForms, id: &EntityId) -> Result { - entity(entities, id)? - .forms_in(TARGET_LANGUAGE) - .and_then(<[String]>::first) - .cloned() - .ok_or_else(|| LowerError::NoForm { entity: id.clone() }) -} - pub(crate) fn entity(entities: &dyn EntityForms, id: &EntityId) -> Result { entities .entity(id) diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index e6a6cbb..a0974b6 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -1,14 +1,14 @@ //! The `weksa` command line: the same validator and store admission the library //! exposes, plus text projections (`dump`) that are for review and are not stores. -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use anyhow::{Context, Result, bail}; use clap::error::{ContextKind, ContextValue, ErrorKind}; use clap::{Args, Parser, Subcommand, ValueEnum}; use unicode_normalization::UnicodeNormalization; use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn}; -use weksa::encode::{EditDraft, OntologyDraft}; +use weksa::encode::{EditDraft, MarkedLine, OntologyDraft, WorkDraft}; use weksa::entity::{Entity, EntityOrigin, LanguageForms, Pronunciation}; use weksa::inventory::{proof_revision, seed, speaker_scene_revision}; use weksa::packet::{FeatValue, FeatureValue, Grade}; @@ -20,7 +20,7 @@ use weksa::run::{ use weksa::score::ScoreInput; use weksa::{ Annotator, CorpusStore, EntityId, EntityStore, Family, InventoryStore, OllamaEmbedder, OntologyStore, OptionLabel, - PacketStore, PairRelation, Party, Scene, Slot, SpeakerStateStore, ValueKind, Scorer, Split, SplitScheme, StoreError, lower, pronounce, validate, + PacketStore, PairRelation, Party, Scene, Slot, SpeakerStateStore, ValueKind, Scorer, Split, SplitScheme, StoreError, StoreRefusal, pronounce, validate, }; #[derive(Parser)] @@ -64,17 +64,14 @@ enum Command { Scene(SceneCommand), #[command(subcommand)] Packet(PacketCommand), - /// Lower a stored packet to English text. - Lower { + /// Print the ruled IPA of each entity a stored packet cites, one per line; + /// refused for an entity with no ruled pronunciation. + Pronounce { store: PathBuf, key: String, - /// Print the ruled IPA of each entity the packet cites, one per line, - /// instead of the text; refused for an entity with no ruled pronunciation. - #[arg(long, conflicts_with = "phones")] - ipa: bool, - /// Print the phones line of each entity the packet cites, one JSON - /// object per line ({language, source, phones: [{ipa, stress}]}), - /// instead of the text; refused for a phone the inventory lacks. + /// Print the phones line of each entity instead, one JSON object per + /// line ({language, source, phones: [{ipa, stress}]}); refused for a + /// phone the inventory lacks. #[arg(long)] phones: bool, }, @@ -242,6 +239,22 @@ enum CorpusCommand { Withdraw { key: String, }, + /// Import a transcribed public-domain work: mint its cast as entities and + /// add each line a name-span answer covers (`docs/name-span-annotation.md`). + Import { + /// The work file: provenance, cast, lines. + work: PathBuf, + /// The annotator's answer: one `{"line": N, "marked": "..."}` per line. + names: PathBuf, + #[arg(long, requires = "generator", conflicts_with = "operator")] + model_family: Option, + /// The attestation string naming the annotating model. + #[arg(long, requires = "model_family")] + generator: Option, + /// Annotated by the operator, not a model. + #[arg(long)] + operator: bool, + }, } #[derive(Subcommand)] @@ -681,6 +694,7 @@ impl AddArgs { supersedes: self.supersedes, withdrawn: false, names, + names_by: None, }) } } @@ -698,6 +712,22 @@ fn split_text(item: &CorpusItem) -> String { .map_or_else(|| "excluded".to_string(), |s| format!("{s:?}")) } +/// The annotator the flags name: a model family with its generator, or the operator. +fn annotator( + model_family: Option, + generator: Option, + operator: bool, +) -> Result { + match (model_family, generator, operator) { + (Some(family), Some(generator), false) => Ok(Annotator::Model { + family: family.into(), + generator, + }), + (None, None, true) => Ok(Annotator::Operator), + _ => bail!("name the annotator: --model-family with --generator, or --operator"), + } +} + fn refused(e: StoreError) -> anyhow::Error { anyhow::anyhow!("{e}") } @@ -737,13 +767,78 @@ fn main() -> Result<()> { .map_err(refused)?; println!("{outcome:?} {key}"); } + Command::Corpus(CorpusCommand::Import { + work, + names, + model_family, + generator, + operator, + }) => { + let annotator = annotator(model_family, generator, operator)?; + let draft = WorkDraft::parse(&std::fs::read_to_string(&work)?)?; + let mut marked = Vec::new(); + for (i, line) in std::fs::read_to_string(&names)?.lines().enumerate() { + if line.trim().is_empty() { + continue; + } + marked.push( + MarkedLine::parse(line) + .map_err(|e| anyhow::anyhow!("names line {}: {e}", i + 1))?, + ); + } + let (new_entities, items) = draft.into_import(&marked, &annotator)?; + // Every document passes the real doors on scratch copies of the + // stores before the first write, so a refusal leaves them untouched. + let scratch = std::env::temp_dir().join(format!("weksa-import-{}", std::process::id())); + std::fs::create_dir_all(&scratch)?; + let dry = (|| -> Result<()> { + let copy = |from: &Path, name: &str| -> Result { + let to = scratch.join(name); + if from.exists() { + std::fs::copy(from, &to)?; + } + Ok(to) + }; + let mut entities = EntityStore::open(©(&cli.entities, "e.cc")?)?; + let mut store = CorpusStore::open(©(&cli.corpus, "c.cc")?)?; + for e in new_entities.clone() { + entities + .put(e) + .map_err(|e| anyhow::anyhow!("a cast entity: {e}"))?; + } + for item in items.clone() { + let key = item.id.clone(); + store.add(item, &entities).map_err(|e| match e { + StoreError::Refused(StoreRefusal::S6TextHeld { held_by, .. }) => { + anyhow::anyhow!("{key} repeats the text of {held_by}") + } + e => anyhow::anyhow!("{key}: {e}"), + })?; + } + Ok(()) + })(); + let _ = std::fs::remove_dir_all(&scratch); + dry?; + let mut entities = EntityStore::open(&cli.entities)?; + for e in new_entities { + let id = e.id.clone(); + let outcome = entities.put(e).map_err(refused)?; + println!("{outcome:?} {id}"); + } + let mut store = CorpusStore::open(&cli.corpus)?; + for item in items { + let (key, split) = (item.id.clone(), split_text(&item)); + let outcome = store.add(item, &entities).map_err(refused)?; + println!("{outcome:?} {key} {split}"); + } + } Command::Corpus(CorpusCommand::Dump) => { for i in CorpusStore::open(&cli.corpus)?.all()? { let split = split_text(&i); let withdrawn = if i.withdrawn { " withdrawn" } else { "" }; println!( - "{} [{} {} group={} {split}{withdrawn}]\n {:?}\n names: {:?}\n source: {:?}", - i.id, i.language, i.speaker, i.group, i.text, i.names, i.source + "{} [{} {} group={} {split}{withdrawn}]\n {:?}\n names: {:?} by {:?}\n source: {:?}", + i.id, i.language, i.speaker, i.group, i.text, i.names, i.names_by, i.source ); } } @@ -880,14 +975,7 @@ fn main() -> Result<()> { generator, operator, }) => { - let annotator = match (model_family, generator, operator) { - (Some(family), Some(generator), false) => Annotator::Model { - family: family.into(), - generator, - }, - (None, None, true) => Annotator::Operator, - _ => bail!("name the annotator: --model-family with --generator, or --operator"), - }; + let annotator = annotator(model_family, generator, operator)?; let inventory_store = InventoryStore::open(&cli.inventory)?; let inventory = inventory_store.inventory()?; let corpus = CorpusStore::open(&cli.corpus)?; @@ -1029,14 +1117,8 @@ fn main() -> Result<()> { .map_err(refused)?; println!("admitted {key}"); } - Command::Lower { - store, - key, - ipa, - phones, - } => { + Command::Pronounce { store, key, phones } => { let inventory_store = InventoryStore::open(&cli.inventory)?; - let inventory = inventory_store.inventory()?; let mut packets = PacketStore::open(&store)?; let packet = packets.get(&key)?.context("no such packet")?; // Admission is the only source of a LoweringInput; re-admitting the @@ -1050,16 +1132,14 @@ fn main() -> Result<()> { ) .map_err(refused)?; let entities = EntityStore::open(&cli.entities)?; - if ipa { - for line in pronounce(&view, &entities)? { - println!("{line}"); - } - } else if phones { + if phones { for line in weksa::phones(&view, &entities)? { println!("{}", serde_json::to_string(&line)?); } } else { - println!("{}", lower(&view, &inventory, &entities)?); + for line in pronounce(&view, &entities)? { + println!("{line}"); + } } } Command::Score(args) => { diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 7339a6e..fb2b94a 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -19,8 +19,7 @@ //! take caller stores (`validate`, `packet proof|dao`, `lower`) are unchanged. //! //! A round encodes, admits, then lowers each admitted view through the -//! `ModelPort` (`Prompt::lower`), in two answer phases; the CLI's `lower` still -//! uses the template table. The prompt source pin covers `context.rs`, +//! `ModelPort` (`Prompt::lower`), in two answer phases; no other path produces lowered text. The prompt source pin covers `context.rs`, //! `split.rs` and `lower.rs`. //! //! `RunStores::stance_ablation`, like `recall`, is a diagnostic in its own @@ -1485,13 +1484,11 @@ impl RunStores { p.stance = Stance::default(); variants[0] = Some(prompt_of(1, p)?); } - if held.len() > 1 { - let mut p = packet.clone(); - p.stance = held[(i + 1) % held.len()].1.stance.clone(); - let prompt = prompt_of(2, p)?; - if prompt.digest() != base.digest() { - variants[1] = Some(prompt); - } + let mut p = packet.clone(); + p.stance = held[(i + 1) % held.len()].1.stance.clone(); + let prompt = prompt_of(2, p)?; + if prompt.digest() != base.digest() { + variants[1] = Some(prompt); } // The round's own answer must exist; a missing one is not a gap to ask for. let recorded_text = match asked(recorded, &base)? { @@ -3112,6 +3109,9 @@ fn discard_inventory(dir: &Path) { } } +#[cfg(test)] +mod mutants; + #[cfg(test)] mod tests { use super::*; @@ -3972,7 +3972,7 @@ mod tests { ); } if prompt.text().starts_with(crate::context::LOWER_INSTRUCTIONS) { - // The table's English for the Dao items, the lowerer's answer. + // The fake lowerer's answer for the Dao items. let text = match prompt.item_keys()[0].as_str() { "item:daodejing:1" => "The sage empties the minds. The sage fills the bellies. The sage weakens the wills. The sage strengthens the bones.", "item:daodejing:2" => "The heaven is long-enduring. The earth continues long.", diff --git a/crates/weksa/src/run/mutants.rs b/crates/weksa/src/run/mutants.rs new file mode 100644 index 0000000..75b1e63 --- /dev/null +++ b/crates/weksa/src/run/mutants.rs @@ -0,0 +1,21 @@ +//! Pins for private functions of `run.rs` that the doors reach only through a +//! whole run. + +use super::*; + +/// A line is probed only when it has at least four words, and then it is +/// split in two near its byte middle, the pieces trimmed. +#[test] +fn a_recall_line_splits_from_four_words_and_not_before() { + assert_eq!(recall_split("one two three"), None); + assert_eq!(recall_split(""), None); + assert_eq!( + recall_split("one two three four"), + Some(("one two".to_string(), "three four".to_string())) + ); + // No spaces: each ideograph is a word, the split falls at the middle. + assert_eq!( + recall_split("天地不仁以万物为刍狗"), + Some(("天地不仁以".to_string(), "万物为刍狗".to_string())) + ); +} diff --git a/crates/weksa/src/split.rs b/crates/weksa/src/split.rs index 898439d..20f971d 100644 --- a/crates/weksa/src/split.rs +++ b/crates/weksa/src/split.rs @@ -172,6 +172,9 @@ impl Checkpoint { } } +#[cfg(test)] +mod mutants; + /// Item builders for the unit tests that live in `src`: the twins of /// `tests/common`'s `group_in_bucket`, `item_in`, `cast` and `model`. #[cfg(test)] @@ -208,6 +211,7 @@ pub(crate) mod fixture { language: "en-US".into(), speaker: EntityId::parse("entity:nibu").unwrap(), names: vec![], + names_by: None, group: group.into(), prior_turns: vec![], source, diff --git a/crates/weksa/src/split/mutants.rs b/crates/weksa/src/split/mutants.rs new file mode 100644 index 0000000..99cde9f --- /dev/null +++ b/crates/weksa/src/split/mutants.rs @@ -0,0 +1,11 @@ +//! Pins of private items of `split` found unpinned by mutation testing. + +use super::Checkpoint; + +/// A checkpoint reports the round it was declared at, whatever the round. +#[test] +fn a_checkpoint_reports_its_round() { + assert_eq!(Checkpoint::for_test(7).round(), 7); + assert_eq!(Checkpoint::for_test(0).round(), 0); + assert_eq!(Checkpoint::for_test(1).round(), 1); +} diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index 56fcc5b..e942a66 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -344,6 +344,9 @@ impl CorpusStore { if let Some(why) = empty_attestation(&item.source) { return Err(malformed(&item.id, why)); } + if let Some(by) = &item.names_by { + check_annotator(&item.id, by)?; + } check_entities(&item, entities)?; for held in self.all()? { if normalized(&held.text) == text && item.supersedes.as_deref() != Some(&held.id) { @@ -487,13 +490,13 @@ pub(crate) fn normalized(text: &str) -> String { } /// The one-to-one folds `normalized` adds to NFKC and case folding, applied -/// before the case fold (U+0130 would otherwise fold to i plus a combining dot, -/// which the key keeps as a mark). +/// before the case fold. (U+0130 needs no arm: it folds to i plus a combining +/// dot, and the `dotless` loop in `normalized` drops that dot after an `i`, so +/// the key is plain `i`.) fn fold_variant(c: char) -> impl Iterator { let one = |c: char| (c, None); let (first, second) = match c { '\u{131}' => one('i'), - '\u{130}' => one('I'), // Voiced katakana wa, wi, we, wo: the hiragana (U+308F-3092) plus the // combining voiced mark, which is how NFKC spells the hiragana forms. '\u{30f7}'..='\u{30fa}' => ( diff --git a/crates/weksa/src/validate.rs b/crates/weksa/src/validate.rs index 49c670f..71c92d5 100644 --- a/crates/weksa/src/validate.rs +++ b/crates/weksa/src/validate.rs @@ -4,7 +4,7 @@ //! lowerer sees, comes from `PacketStore::admit` alone, which builds it from //! the admitted packet and its corpus item through the crate-private //! `LoweringInput::of`. The stance is checked like every feature -//! list, in slot `Stance`. The rules are V1..V8, V10..V12 and V14..V17: the +//! list, in slot `Stance`. The rules are V1..V8, V10..V12 and V14..V18: the //! shape of an id or a grade is its type's parse, not a rule, so V9 is retired //! and not renumbered; V13 is retired too. [`LoweringInput::with_scene`] reads the //! scene of the view's own item group from the speaker-state store, and @@ -120,6 +120,12 @@ pub enum Refusal { V17AddresseeNotAReferent { id: NodeId, }, + /// V18: a predication whose frame is the inventory's `request_intervention`, + /// with no `addressee_consent` naming it. It says what the addressee did, so + /// only the packet's own record may license it. + V18RequestWithoutConsent { + predication: NodeId, + }, } impl Refusal { @@ -144,6 +150,7 @@ impl Refusal { Refusal::V15ConsentSource { .. } => "V15", Refusal::V16SpeakerNotInScene => "V16", Refusal::V17AddresseeNotAReferent { .. } => "V17", + Refusal::V18RequestWithoutConsent { .. } => "V18", } } } @@ -173,6 +180,9 @@ impl fmt::Display for Refusal { "the scene does not cover the speaker of every live item of its group" } Refusal::V17AddresseeNotAReferent { .. } => "the addressee names no referent", + Refusal::V18RequestWithoutConsent { .. } => { + "a request frame needs the addressee's consent naming it" + } }; write!(f, "{}: {what}", self.rule()) } @@ -221,6 +231,7 @@ pub fn validate(packet: &Packet, inventory: &Inventory) -> Result<(), Vec, out: &mut Vec) { + for p in &packet.predications { + if inventory + .resolve(&p.frame, packet.inventory_rev) + .is_some_and(|e| e.gloss == REQUEST_FRAME_GLOSS) + && packet.discourse.addressee_consent.as_ref() != Some(&p.id) + { + out.push(Refusal::V18RequestWithoutConsent { + predication: p.id.clone(), + }); + } + } +} + fn node_lists(packet: &Packet) -> [(&Vec, &'static str); 3] { [ (&packet.context.participants, "participant"), diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 7f624b8..ef21abc 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -904,7 +904,7 @@ fn fake_embedder() -> String { } #[test] -fn proof_packet_lowers_and_the_lowering_scores_against_items() { +fn proof_packet_pronounces_and_the_item_scores_against_its_own_text() { let d = tempfile::tempdir().unwrap(); cast_entities(d.path(), &["nibu"]); for args in [&["inventory", "seed"][..], &["inventory", "proof"]] { @@ -939,21 +939,11 @@ fn proof_packet_lowers_and_the_lowering_scores_against_items() { "{}", out(&admitted) ); - let lowered = weksa(d.path(), &["lower", "./p.cc", "packet:6e6962752d303031"]); - assert!(lowered.status.success(), "{}", out(&lowered)); - let text = String::from_utf8(lowered.stdout).unwrap(); - assert!( - text.starts_with("Warning: you may endanger yourself"), - "{text}" - ); - // The referent cites entity:nibu: its stored en-US form (NIBU here) reaches the text as stored. - assert!(text.contains("NIBU will stop"), "{text}"); - - let missing = weksa(d.path(), &["lower", "./p.cc", "packet:0000000000000000"]); + let missing = weksa(d.path(), &["pronounce", "./p.cc", "packet:0000000000000000"]); assert!(!missing.status.success() && out(&missing).contains("no such packet")); // IPA is the ruled pronunciation or nothing: spelling is not one. - let args = ["lower", "./p.cc", "packet:6e6962752d303031", "--ipa"]; + let args = ["pronounce", "./p.cc", "packet:6e6962752d303031"]; let unruled = weksa(d.path(), &args); assert!( !unruled.status.success() && out(&unruled).contains("entity has no ruled pronunciation"), @@ -993,11 +983,10 @@ fn proof_packet_lowers_and_the_lowering_scores_against_items() { let ipa = weksa(d.path(), &args); assert!(ipa.status.success(), "{}", out(&ipa)); assert_eq!(String::from_utf8(ipa.stdout).unwrap(), "ˈnibu\n"); - // The phones line is the committed fixture, byte for byte, and is a different - // output from --ipa, so the two flags cannot be combined. + // The phones line is the committed fixture, byte for byte. let phones = weksa( d.path(), - &["lower", "./p.cc", "packet:6e6962752d303031", "--phones"], + &["pronounce", "./p.cc", "packet:6e6962752d303031", "--phones"], ); assert!(phones.status.success(), "{}", out(&phones)); let fixture = std::fs::read_to_string( @@ -1006,21 +995,13 @@ fn proof_packet_lowers_and_the_lowering_scores_against_items() { ) .unwrap(); assert_eq!(String::from_utf8(phones.stdout).unwrap(), fixture); - let both = weksa( - d.path(), - &[ - "lower", - "./p.cc", - "packet:6e6962752d303031", - "--phones", - "--ipa", - ], - ); - assert!(!both.status.success(), "{}", out(&both)); - let again = weksa(d.path(), &["lower", "./p.cc", "packet:6e6962752d303031"]); - assert_eq!(String::from_utf8(again.stdout).unwrap(), text); + // The old lowering subcommand is gone: clap's usage error, exit 2. + let gone = weksa(d.path(), &["lower", "./p.cc", "packet:6e6962752d303031"]); + assert_eq!(gone.status.code(), Some(2), "{}", out(&gone)); - std::fs::write(d.path().join("out.txt"), &text).unwrap(); + // The item's own text, as the scorer's output. + let text = "Warning: you may endanger yourself by touching the control. NIBU will stop that."; + std::fs::write(d.path().join("out.txt"), text).unwrap(); let url = fake_embedder(); let scored = weksa( d.path(), @@ -1180,7 +1161,7 @@ fn cli_errors_never_echo_identifiers() { ]), "--model-family", ); - // Keys on withdraw, split, validate, lower and score. + // Keys on withdraw, split, validate, pronounce and score. refuses_without_echo(p, &["corpus", "withdraw", "item:canary:7"], "no such item"); refuses_without_echo(p, &["split", "item:canary:7"], "no such item"); std::fs::write(p.join("out.txt"), "text").unwrap(); @@ -1198,7 +1179,7 @@ fn cli_errors_never_echo_identifiers() { &["validate", "p.cc", "packet:canary:7"], "no such packet", ); - refuses_without_echo(p, &["lower", "p.cc", "packet:canary:7"], "no such packet"); + refuses_without_echo(p, &["pronounce", "p.cc", "packet:canary:7"], "no such packet"); // S6 names no held key; S5 names no group. let held = weksa( p, @@ -2164,3 +2145,130 @@ fn inventory_admit_cap_is_256_and_a_refusal_writes_nothing() { assert!(admitted.status.success(), "{}", out(&admitted)); assert!(out(&admitted).contains("admitted rev:2 with 256 entries")); } + +#[test] +fn corpus_import_labels_spans_and_is_idempotent() { + let d = tempfile::tempdir().unwrap(); + let work = d.path().join("work.json"); + std::fs::write( + &work, + r#"{"slug":"lantern-wager","language":"en-US","admitted_on":"2026-11-13", + "source":{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}}, + "cast":[{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:tamsin","forms":["Tamsin","Tam"]}], + "lines":[{"group":"s1","speaker":"entity:orsa","text":"Tamsin, bring the lantern."}, + {"group":"s1","speaker":"entity:tamsin","text":"Orsa, I will not."}]}"#, + ) + .unwrap(); + let names = d.path().join("names.jsonl"); + std::fs::write( + &names, + concat!( + r#"{"line":1,"marked":"[[entity:tamsin|Tamsin]], bring the lantern."}"#, "\n\n", + r#"{"line":2,"marked":"[[entity:orsa|Orsa]], I will not."}"#, "\n", + ), + ) + .unwrap(); + let (w, n) = (work.to_str().unwrap(), names.to_str().unwrap()); + let bare = weksa(d.path(), &["corpus", "import", w, n]); + assert!(!bare.status.success()); + let flags = ["--model-family", "anthropic", "--generator", "a name annotator"]; + let run = || weksa(d.path(), &[&["corpus", "import", w, n][..], &flags].concat()); + let first = run(); + assert!(first.status.success(), "{}", out(&first)); + let text = out(&first); + assert!(text.contains("Added item:lantern-wager:1 ") && text.contains("Added item:lantern-wager:2 "), "{text}"); + assert!(text.contains("Added entity:orsa"), "{text}"); + let second = out(&run()); + assert!(second.contains("Unchanged item:lantern-wager:1 ") && !second.contains("Added"), "{second}"); + let dump = out(&weksa(d.path(), &["corpus", "dump"])); + assert!(dump.contains("by Some(Model"), "{dump}"); +} + +const LANTERN_CAST: &str = r#""cast":[{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:tamsin","forms":["Tamsin","Tam"]}]"#; +const LANTERN_SOURCE: &str = r#""source":{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}}"#; + +/// Writes a work of `lines` (text per line, all said by Orsa) and an answer of +/// `marked` per line, returns both paths. +fn lantern(dir: &std::path::Path, lines: &[&str], marked: &[&str]) -> (String, String) { + let ls: Vec = lines + .iter() + .map(|t| format!(r#"{{"group":"s1","speaker":"entity:orsa","text":"{t}"}}"#)) + .collect(); + let work = dir.join("w.json"); + std::fs::write( + &work, + format!( + r#"{{"slug":"lantern-wager","language":"en-US","admitted_on":"2026-11-13",{LANTERN_SOURCE},{LANTERN_CAST},"lines":[{}]}}"#, + ls.join(",") + ), + ) + .unwrap(); + let names = dir.join("n.jsonl"); + let rows: Vec = marked + .iter() + .enumerate() + .map(|(i, m)| format!(r#"{{"line":{},"marked":"{m}"}}"#, i + 1)) + .collect(); + std::fs::write(&names, rows.join("\n")).unwrap(); + (work.to_str().unwrap().into(), names.to_str().unwrap().into()) +} + +const MODEL: [&str; 4] = ["--model-family", "anthropic", "--generator", "a name annotator"]; + +#[test] +fn a_refused_import_writes_nothing_and_a_redraw_imports() { + let d = tempfile::tempdir().unwrap(); + let lines = ["Tamsin, bring the lantern.", "Tamsin's lantern, said Orsa."]; + let (w, n) = lantern(d.path(), &lines, &["[[entity:tamsin|Tamsin]], bring the lantern.", "[[entity:tamsin|Tamsin's]] lantern, said Orsa."]); + let import = |w: &str, n: &str| weksa(d.path(), &[&["corpus", "import", w, n][..], &MODEL].concat()); + let bad = import(&w, &n); + assert!(!bad.status.success()); + assert_eq!(out(&weksa(d.path(), &["corpus", "dump"])).trim(), ""); + assert!(!d.path().join("e.cc").exists() || EntityStore::open(&d.path().join("e.cc")).unwrap().all().unwrap().is_empty()); + // The redrawn answer, different on line 1 too, imports. + let (w, n) = lantern(d.path(), &lines, &["Tamsin, bring the lantern.", "[[entity:tamsin|Tamsin]]'s lantern, said Orsa."]); + let good = import(&w, &n); + assert!(good.status.success(), "{}", out(&good)); + assert!(out(&good).contains("Added item:lantern-wager:2")); +} + +#[test] +fn a_work_repeating_a_line_is_refused_naming_both_lines() { + let d = tempfile::tempdir().unwrap(); + let (w, n) = lantern(d.path(), &["No, my lord.", "No, my lord!", "Quiet, both of you."], &["No, my lord.", "No, my lord!", "Quiet, both of you."]); + let o = weksa(d.path(), &[&["corpus", "import", &w, &n][..], &MODEL].concat()); + assert!(!o.status.success()); + let err = String::from_utf8_lossy(&o.stderr).to_string(); + assert!(err.contains("item:lantern-wager:2") && err.contains("item:lantern-wager:1"), "{err}"); + assert_eq!(out(&weksa(d.path(), &["corpus", "dump"])).trim(), ""); +} + +#[test] +fn the_annotator_flags_decide_the_label() { + let d = tempfile::tempdir().unwrap(); + let (w, n) = lantern(d.path(), &["Quiet, both of you."], &["Quiet, both of you."]); + let none = weksa(d.path(), &["corpus", "import", &w, &n]); + assert!(!none.status.success()); + let blank = weksa(d.path(), &["corpus", "import", &w, &n, "--model-family", "anthropic", "--generator", " "]); + assert!(!blank.status.success()); + assert_eq!(out(&weksa(d.path(), &["corpus", "dump"])).trim(), ""); + let op = weksa(d.path(), &["corpus", "import", &w, &n, "--operator"]); + assert!(op.status.success(), "{}", out(&op)); + let dump = out(&weksa(d.path(), &["corpus", "dump"])); + assert!(dump.contains("by Some(Operator)"), "{dump}"); +} + +#[test] +fn an_import_refused_on_a_cast_entity_leaves_both_stores_untouched() { + let d = tempfile::tempdir().unwrap(); + // entity:tamsin is already held by a cast origin, so the work's PublicDomain + // claim on it (cast position 2, after orsa) is refused by EntityStore::put. + cast_entities(d.path(), &["tamsin"]); + let (w, n) = lantern(d.path(), &["Quiet, both of you."], &["Quiet, both of you."]); + let before = std::fs::read(d.path().join("e.cc")).unwrap(); + let o = weksa(d.path(), &[&["corpus", "import", &w, &n][..], &MODEL].concat()); + assert!(!o.status.success()); + assert!(String::from_utf8_lossy(&o.stderr).contains("a cast entity"), "{}", String::from_utf8_lossy(&o.stderr)); + assert_eq!(std::fs::read(d.path().join("e.cc")).unwrap(), before); + assert!(!d.path().join("c.cc").exists() || CorpusStore::open(&d.path().join("c.cc")).unwrap().all().unwrap().is_empty()); +} diff --git a/crates/weksa/tests/common/mod.rs b/crates/weksa/tests/common/mod.rs index fa2deb9..8ee0596 100644 --- a/crates/weksa/tests/common/mod.rs +++ b/crates/weksa/tests/common/mod.rs @@ -205,6 +205,7 @@ pub fn item_in(key: &str, text: &str, group: &str, source: Source) -> CorpusItem language: "en-US".into(), speaker: eid("nibu"), names: vec![], + names_by: None, group: group.into(), prior_turns: vec![], source, diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs index d2550de..a8993fc 100644 --- a/crates/weksa/tests/context.rs +++ b/crates/weksa/tests/context.rs @@ -593,6 +593,41 @@ mod lowering { assert!(Prompt::lower(&read, &item, &away(), "en-US", &inv, &held, &empty).is_ok()); } + /// The packet the encoder emits for a request: frame:3, no consent, source + /// ModelDecomposition. Admission refuses it, so no lowering view exists. + #[test] + fn an_encoder_request_never_reaches_a_lowerer() { + let cast = InventoryStore::open(&data("interlingua/inventory.cc")).unwrap(); + let entities = EntityStore::open(&data("entity/entity.cc")).unwrap(); + let corpus = CorpusStore::open(&data("corpus/corpus.cc")).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let mut store = PacketStore::open(&dir.path().join("p.cc")).unwrap(); + let mut packet = proof_packet(); + packet.provenance.source = weksa::packet::ProvenanceSource::ModelDecomposition; + packet.predications[0].frame = weksa::InvId::parse("frame:3").unwrap(); + match store.admit(packet.clone(), &cast, &corpus, &entities) { + Err(weksa::StoreError::Invalid(r)) => assert_eq!( + r, + vec![weksa::Refusal::V18RequestWithoutConsent { + predication: weksa::packet::NodeId::parse("n4").unwrap() + }] + ), + other => panic!("expected V18, got {:?}", other.map(|_| ())), + } + // Not blanket: telemetry that records the consent admits and lowers. + packet.provenance.source = weksa::packet::ProvenanceSource::GameTelemetry; + packet.discourse.addressee_consent = Some(weksa::packet::NodeId::parse("n4").unwrap()); + drop(store); + let view = admitted(packet, &cast, "corpus/corpus.cc", &entities); + let item = CorpusItemId::parse("item:nibu:1").unwrap(); + let held = Held(vec![nibu(&[("en-US", &["Nibu"])])]); + let empty = weksa::Ontology::empty(); + assert!( + Prompt::lower(&view, &item, &away(), "en-US", &cast.inventory().unwrap(), &held, &empty) + .is_ok() + ); + } + fn dao(index: usize) -> (LoweringInput, Inventory, EntityStore, CorpusItemId) { let inv = InventoryStore::open(&data("daodejing/inventory.cc")).unwrap(); let entities = EntityStore::open(&data("daodejing/entity.cc")).unwrap(); diff --git a/crates/weksa/tests/daodejing.rs b/crates/weksa/tests/daodejing.rs index 7e5f9ae..160c3c6 100644 --- a/crates/weksa/tests/daodejing.rs +++ b/crates/weksa/tests/daodejing.rs @@ -1,11 +1,11 @@ //! The Dao De Jing test bed (`data/daodejing`): its Legge items carry full -//! public-domain provenance, and its packets are admitted and lower to the -//! English the first golf-loop run scored (`docs/targets/dao-de-jing-test-bed.md`). +//! public-domain provenance, and its packets are admitted +//! (`docs/targets/dao-de-jing-test-bed.md`). use std::path::Path; use weksa::daodejing::packets; -use weksa::{CorpusStore, EntityStore, InventoryStore, PacketStore, Source, lower}; +use weksa::{CorpusStore, EntityStore, InventoryStore, PacketStore, Source}; fn root() -> std::path::PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/daodejing") @@ -39,30 +39,15 @@ fn every_legge_item_names_licence_jurisdiction_translator_and_edition_year() { } #[test] -fn the_slice_packets_are_admitted_and_lower_to_the_scored_english() { +fn the_slice_packets_are_admitted_and_the_committed_store_holds_them() { let inventory = InventoryStore::open(&root().join("inventory.cc")).unwrap(); let corpus = CorpusStore::open(&root().join("corpus.cc")).unwrap(); let entities = EntityStore::open(&root().join("entity.cc")).unwrap(); let dir = tempfile::tempdir().unwrap(); let mut store = PacketStore::open(&dir.path().join("p.cc")).unwrap(); - let inv = inventory.inventory().unwrap(); - let lowered: Vec = packets() - .into_iter() - .map(|p| { - let view = store.admit(p, &inventory, &corpus, &entities).unwrap(); - lower(&view, &inv, &entities).unwrap() - }) - .collect(); - assert_eq!( - lowered, - [ - "The sage empties the minds. The sage fills the bellies. The sage weakens the wills. The sage strengthens the bones.", - "The heaven is long-enduring. The earth continues long.", - "The highest excellence is like the water.", - "The words that are strictly true seem paradoxical.", - "The space between heaven and earth may be compared to the bellows.", - ] - ); + for p in packets() { + store.admit(p, &inventory, &corpus, &entities).unwrap(); + } // The committed packet store holds exactly what the module builds. let stored = PacketStore::open(&root().join("packets.cc")).unwrap(); for p in packets() { diff --git a/crates/weksa/tests/display.rs b/crates/weksa/tests/display.rs index 20b5b6f..de4369c 100644 --- a/crates/weksa/tests/display.rs +++ b/crates/weksa/tests/display.rs @@ -33,20 +33,6 @@ fn assert_clean_and_distinct(all: Vec #[test] fn lower_error_display_names_the_variant_and_no_identifier() { assert_clean_and_distinct(vec![ - LowerError::Unrendered { what: SENTINEL }, - LowerError::NoRow { id: inv() }, - LowerError::GlossMismatch { - id: inv(), - row: SENTINEL, - }, - LowerError::UnknownRole { - frame: inv(), - role: SENTINEL.into(), - }, - LowerError::MissingRole { - predication: node(), - role: SENTINEL.into(), - }, LowerError::UnknownEntity { entity: entity() }, LowerError::NoForm { entity: entity() }, LowerError::NoPronunciation { entity: entity() }, @@ -60,12 +46,6 @@ fn lower_error_display_names_the_variant_and_no_identifier() { entity: entity(), at: 4242, }, - LowerError::ConsentMissing { - predication: node(), - }, - LowerError::AskerNotAddressee { - predication: node(), - }, LowerError::EntityRead { entity: entity(), why: SENTINEL.into(), @@ -124,6 +104,7 @@ fn refusal_display_names_the_rule_and_no_identifier() { }, Refusal::V16SpeakerNotInScene, Refusal::V17AddresseeNotAReferent { id: node() }, + Refusal::V18RequestWithoutConsent { predication: node() }, ]; assert_eq!( Refusal::V16SpeakerNotInScene.to_string(), diff --git a/crates/weksa/tests/entities.rs b/crates/weksa/tests/entities.rs index 5f1f7cf..9d316bf 100644 --- a/crates/weksa/tests/entities.rs +++ b/crates/weksa/tests/entities.rs @@ -502,3 +502,96 @@ fn a_malformed_pronunciation_is_refused() { } assert_eq!(store.get(&eid("kay")).unwrap(), None); } + +const LANTERN: &str = r#"{"slug":"lantern-wager","language":"en-US","admitted_on":"2026-11-13", + "source":{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}}, + "cast":[{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:tamsin","forms":["Tamsin","Tam"]}], + "lines":[{"group":"s1","speaker":"entity:orsa","text":"Tamsin, bring the lantern."}, + {"group":"s1","speaker":"entity:tamsin","text":"Tamsin's lantern, said Orsa."}, + {"group":"s2","speaker":"entity:orsa","text":"Quiet, both of you."}]}"#; + +fn marked(line: u32, text: &str) -> weksa::encode::MarkedLine { + weksa::encode::MarkedLine { line, marked: text.into() } +} + +#[test] +fn an_imported_work_passes_the_store_doors() { + use weksa::Annotator; + use weksa::corpus::ModelFamily; + let d = tempfile::tempdir().unwrap(); + let mut entities = EntityStore::open(&d.path().join("e.cc")).unwrap(); + let mut corpus = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + let model = Annotator::Model { family: ModelFamily::Anthropic, generator: "g".into() }; + let work = || weksa::encode::WorkDraft::parse(LANTERN).unwrap(); + let (cast, items) = work() + .into_import(&[marked(1, "[[entity:tamsin|Tamsin]], bring the lantern.")], &model) + .unwrap(); + for e in cast { + assert_eq!(entities.put(e).unwrap(), AddOutcome::Added); + } + for i in items.clone() { + assert_eq!(corpus.add(i, &entities).unwrap(), AddOutcome::Added); + } + drop(corpus); + let mut corpus = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + let stored = corpus.get("item:lantern-wager:1").unwrap().unwrap(); + assert_eq!(stored.names_by, Some(model.clone())); + assert_eq!(stored.names.len(), 1); + assert_eq!(corpus.add(items[0].clone(), &entities).unwrap(), AddOutcome::Unchanged); + + // A form the entity does not list is the store's refusal, not the importer's. + let (_, bad) = work() + .into_import(&[marked(2, "[[entity:tamsin|Tamsin's]] lantern, said Orsa.")], &model) + .unwrap(); + assert!(matches!( + refusal(corpus.add(bad[0].clone(), &entities)), + StoreRefusal::NameSpanNotAForm { .. } + )); + + // An operator run over another line labels its own spans. + let (_, op) = work() + .into_import(&[marked(3, "Quiet, both of you.")], &Annotator::Operator) + .unwrap(); + assert_eq!(corpus.add(op[0].clone(), &entities).unwrap(), AddOutcome::Added); + assert_eq!( + corpus.get("item:lantern-wager:3").unwrap().unwrap().names_by, + Some(Annotator::Operator) + ); + // The first run's line keeps its label. + assert_eq!(corpus.get("item:lantern-wager:1").unwrap().unwrap().names_by, Some(model)); +} + +#[test] +fn committed_items_read_with_no_names_label() { + let data = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data"); + for sub in ["daodejing", "original"] { + let d = tempfile::tempdir().unwrap(); + let copy = d.path().join("c.cc"); + std::fs::copy(data.join(sub).join("corpus.cc"), ©).unwrap(); + let items = CorpusStore::open(©).unwrap().all().unwrap(); + assert!(!items.is_empty(), "{sub}"); + assert!(items.iter().all(|i| i.names_by.is_none()), "{sub}"); + } +} + +#[test] +fn a_blank_names_by_generator_is_refused_by_the_corpus_door() { + use weksa::Annotator; + use weksa::corpus::ModelFamily; + let d = tempfile::tempdir().unwrap(); + let mut entities = EntityStore::open(&d.path().join("e.cc")).unwrap(); + let mut corpus = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + let blank = Annotator::Model { family: ModelFamily::Anthropic, generator: " ".into() }; + let (cast, items) = weksa::encode::WorkDraft::parse(LANTERN) + .unwrap() + .into_import(&[marked(3, "Quiet, both of you.")], &blank) + .unwrap(); + for e in cast { + entities.put(e).unwrap(); + } + assert!(matches!( + refusal(corpus.add(items[0].clone(), &entities)), + StoreRefusal::Malformed { .. } + )); + assert!(corpus.all().unwrap().is_empty()); +} diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 0d75b3b..f521ba3 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -2742,15 +2742,6 @@ fn source_tripwires() { .collect(); let count = |needle: &str| code.iter().filter(|l| l.contains(needle)).count(); assert!(!run.contains("pub fn new")); - // The round does not call the template table: the model lowers. - assert_eq!( - code.iter() - .filter(|l| l.contains("lower(&") && !l.contains("Prompt::lower(")) - .count(), - 0, - "run.rs calls the table" - ); - assert_eq!(count("crate::lower::lower"), 0, "run.rs imports the table"); assert_eq!( count("Prompt::lower("), 2, diff --git a/crates/weksa/tests/lower.rs b/crates/weksa/tests/lower.rs index 3ec94cf..0cf5347 100644 --- a/crates/weksa/tests/lower.rs +++ b/crates/weksa/tests/lower.rs @@ -1,22 +1,16 @@ -//! The English lowerer against the committed inventory (revision 2) and the -//! `nibu-proof-001` packet, and the rules that keep a table row from speaking -//! for a meaning it does not carry. +//! The entity port, `pronounce` and `phones` against the committed stores and +//! the `nibu-proof-001` packet. mod common; use std::path::Path; -use common::{iv, nd}; use weksa::entity::{Entity, EntityOrigin, LanguageForms}; -use weksa::inventory::proof_revision; -use weksa::packet::{EntityId, FeatValue, FeatureValue, Packet, proof_packet}; +use weksa::packet::{EntityId, Packet, proof_packet}; use weksa::{ CorpusStore, EntityForms, EntityStore, Inventory, InventoryStore, LowerError, LoweringInput, - PacketStore, PhoneInventory, PhoneLine, Pronunciation, Stress, lower, pronounce, validate, + PacketStore, PhoneInventory, PhoneLine, Pronunciation, Stress, pronounce, }; -const PROOF: &str = "Warning: you may endanger yourself by touching the control. \ - Nibu will stop that."; - fn committed() -> (Inventory, tempfile::TempDir) { let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); let inv = InventoryStore::open(&root.join("data/interlingua/inventory.cc")).unwrap(); @@ -48,181 +42,6 @@ fn committed_entities() -> EntityStore { EntityStore::open(&root.join("data/entity/entity.cc")).unwrap() } -fn lowered(packet: &Packet, inv: &Inventory) -> Result { - lower(&view(packet, inv), inv, &committed_entities()) -} - -#[test] -fn the_proof_packet_is_admitted_by_the_store_and_lowers_to_its_sentences() { - let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); - let (inv, dir) = committed(); - let inventory = InventoryStore::open(&root.join("data/interlingua/inventory.cc")).unwrap(); - let corpus = CorpusStore::open(&root.join("data/corpus/corpus.cc")).unwrap(); - let entities = EntityStore::open(&root.join("data/entity/entity.cc")).unwrap(); - let mut packets = PacketStore::open(&dir.path().join("p.cc")).unwrap(); - packets - .admit(proof_packet(), &inventory, &corpus, &entities) - .unwrap(); - let stored = packets.get("packet:6e6962752d303031").unwrap().unwrap(); - assert_eq!(lowered(&stored, &inv).unwrap(), PROOF); -} - -#[test] -fn only_the_addressee_is_you_and_is_reflexive() { - let (inv, _dir) = committed(); - let mut p = proof_packet(); - p.discourse.addressee = None; - assert_eq!( - lowered(&p, &inv).unwrap(), - "Warning: the player may endanger the player by touching the control. \ - Nibu will stop that." - ); -} - -#[test] -fn no_speech_act_means_no_opener_and_a_capital() { - let (inv, _dir) = committed(); - let mut p = proof_packet(); - p.discourse.speech_act = None; - assert!(lowered(&p, &inv).unwrap().starts_with("You may endanger")); - p.discourse.speech_act = Some(iv("speech_act:1")); - assert!(lowered(&p, &inv).unwrap().starts_with("You may endanger")); -} - -#[test] -fn a_packet_carrying_what_the_lowerer_does_not_render_is_refused() { - let dir = tempfile::tempdir().unwrap(); - let mut store = InventoryStore::open(&dir.path().join("i.cc")).unwrap(); - let (rev1, entries) = weksa::inventory::seed(); - store.admit_revision(rev1, entries).unwrap(); - let (rev2, entries) = proof_revision(); - store.admit_revision(rev2, entries).unwrap(); - let mut ext = common::entry("extension:1", 3); - ext.value_kind = Some(weksa::ValueKind::Grade); - common::admit(&mut store, 3, vec![ext]); - let inv = store.inventory().unwrap(); - let discourse = |p: &mut Packet| { - p.discourse.features.push(FeatureValue { - feature: iv("feature:6"), - value: FeatValue::Grade(common::grade(300)), - }) - }; - let predication = |p: &mut Packet| { - p.predications[0].features.push(FeatureValue { - feature: iv("feature:1"), - value: FeatValue::Option(common::lab("negative")), - }) - }; - let relation = |p: &mut Packet| { - p.relations.push(weksa::packet::Relation { - relation: iv("relation:8"), - from: nd("n4"), - to: nd("n5"), - features: vec![], - }) - }; - let extension = |p: &mut Packet| { - p.inventory_rev = 3; - p.extensions.push(weksa::packet::Extension { - key: iv("extension:1"), - value: FeatValue::Grade(common::grade(3)), - }) - }; - let referent_feature = |p: &mut Packet| { - p.referents[0].features.push(FeatureValue { - feature: iv("feature:6"), - value: FeatValue::Grade(common::grade(1)), - }) - }; - let no_predication = |p: &mut Packet| p.predications.clear(); - let extra_role = |p: &mut Packet| { - let filler = p.predications[0].roles[0].filler.clone(); - p.predications[0].roles.push(weksa::packet::RoleFill { - role: iv("role:3"), - filler, - }) - }; - let consent = |p: &mut Packet| p.discourse.addressee_consent = Some(nd("n5")); - let lone_referent = |p: &mut Packet| { - let mut r = p.referents[0].clone(); - r.id = nd("n99"); - p.referents.push(r) - }; - for (what, edit) in [ - ("features", &discourse as &dyn Fn(&mut Packet)), - ("features", &predication), - ("addressee consent", &consent), - ("relations", &relation), - ("extensions", &extension), - ("no predication", &no_predication), - ("a role the frame does not name", &extra_role), - ("a referent no role mentions", &lone_referent), - ] { - let mut p = proof_packet(); - edit(&mut p); - assert_eq!( - lower(&admitted(&p, &store), &inv, &committed_entities()), - Err(LowerError::Unrendered { what }), - "{what}" - ); - } - // The seed lets no feature sit on a referent, so that arm is refused by validate. - let mut p = proof_packet(); - referent_feature(&mut p); - assert!(validate(&p, &inv).is_err()); -} - -#[test] -fn an_unknown_row_and_an_unfilled_role_are_refused_by_name() { - let (inv, _dir) = committed(); - let mut p = proof_packet(); - p.discourse.speech_act = Some(iv("speech_act:3")); - assert_eq!( - lowered(&p, &inv), - Err(LowerError::NoRow { - id: iv("speech_act:3") - }) - ); - let mut p = proof_packet(); - p.predications[0].roles.pop(); - assert_eq!( - lowered(&p, &inv), - Err(LowerError::MissingRole { - predication: nd("n4"), - role: "patient".into() - }) - ); -} - -#[test] -fn a_row_cannot_speak_for_an_id_whose_gloss_differs() { - // Revision 2 with concept:4 glossed "hatch" in place of "control": the row - // for concept:4 says "control" and must not render it. - let dir = tempfile::tempdir().unwrap(); - let mut store = InventoryStore::open(&dir.path().join("i.cc")).unwrap(); - let (rev1, entries) = weksa::inventory::seed(); - store.admit_revision(rev1, entries).unwrap(); - let (rev2, mut entries) = proof_revision(); - for e in &mut entries { - if e.id == iv("concept:4") { - e.gloss = "hatch".into(); - } - } - store.admit_revision(rev2, entries).unwrap(); - let inv = store.inventory().unwrap(); - assert_eq!( - lower( - &admitted(&proof_packet(), &store), - &inv, - &committed_entities() - ), - Err(LowerError::GlossMismatch { - id: iv("concept:4"), - row: "control" - }) - ); -} - /// An entity port a test controls: ids to entities, and an optional read fault. #[derive(Default)] struct Entities { @@ -257,109 +76,6 @@ fn nibu(forms: &[(&str, &[&str])]) -> Entity { } } -fn through(entities: &Entities) -> Result { - let (inv, _dir) = committed(); - lower(&view(&proof_packet(), &inv), &inv, entities) -} - -#[test] -fn a_name_survives_lowering_verbatim_where_the_concept_word_was() { - // The entity's own form, not "the ship mind", and not re-cased or inflected: - // the form is lowercase and stays so mid-sentence. - let mut e = Entities::default(); - let n = nibu(&[("en-US", &["nibu-of-the-deep"])]); - e.held.push(n); - // It opens the second sentence, where a template word would be capitalised. - assert_eq!( - through(&e).unwrap(), - "Warning: you may endanger yourself by touching the control. \ - nibu-of-the-deep will stop that." - ); -} - -#[test] -fn only_the_target_languages_citation_form_is_chosen() { - // Forms after the first are inflected shapes no English frame asks for; a - // form under another language tag is never read, whatever order it is in. - let mut e = Entities::default(); - let n = nibu(&[ - ("sa-Latn", &["Nibuwa"]), - ("en-US", &["Nibu", "Nibu's", "Nibus"]), - ]); - e.held.push(n); - assert_eq!(through(&e).unwrap(), PROOF); -} - -#[test] -fn an_entity_the_port_does_not_hold_is_refused_not_replaced_by_its_concept_word() { - let id = EntityId::parse("entity:nibu").unwrap(); - assert_eq!( - through(&Entities::default()), - Err(LowerError::UnknownEntity { entity: id }) - ); -} - -#[test] -fn an_entity_with_no_target_language_form_is_refused() { - let mut e = Entities::default(); - let n = nibu(&[("sa-Latn", &["Nibuwa"])]); - let id = n.id.clone(); - e.held.push(n); - assert_eq!(through(&e), Err(LowerError::NoForm { entity: id })); -} - -#[test] -fn a_failing_entity_port_is_refused_by_name() { - let e = Entities { - fault: true, - ..Entities::default() - }; - assert_eq!( - through(&e), - Err(LowerError::EntityRead { - entity: EntityId::parse("entity:nibu").unwrap(), - why: "store unreadable".into() - }) - ); -} - -#[test] -fn an_addressee_entity_still_lowers_to_you_and_is_never_looked_up() { - // Naming the addressee would break "you"/"yourself": the port is not asked, - // so a port that faults on every read changes nothing. - let (inv, _dir) = committed(); - let mut p = proof_packet(); - p.referents[0].entity = Some(EntityId::parse("entity:nibu").unwrap()); - p.referents[2].entity = None; - let faulting = Entities { - fault: true, - ..Entities::default() - }; - assert_eq!( - lower(&view(&p, &inv), &inv, &faulting).unwrap(), - "Warning: you may endanger yourself by touching the control. \ - The ship mind will stop that." - ); -} - -#[test] -fn a_purpose_the_frame_does_not_say_is_refused_not_dropped() { - // The intervention row says "will stop"; a packet that also fills the - // beneficiary role would assert a purpose the text does not carry. - let (inv, _dir) = committed(); - let mut p = proof_packet(); - p.predications[1].roles.push(weksa::packet::RoleFill { - role: iv("role:8"), - filler: nd("n1"), - }); - assert_eq!( - lowered(&p, &inv), - Err(LowerError::Unrendered { - what: "a role the frame does not name" - }) - ); -} - fn ruled(ipa: &str, rejected: &[&str]) -> Pronunciation { Pronunciation { ipa: ipa.into(), @@ -375,6 +91,21 @@ fn say(entities: &dyn EntityForms) -> Result, LowerError> { pronounce(&view(&proof_packet(), &inv), entities) } +#[test] +fn a_failing_entity_port_is_refused_by_name() { + let e = Entities { + fault: true, + ..Entities::default() + }; + assert_eq!( + say(&e), + Err(LowerError::EntityRead { + entity: EntityId::parse("entity:nibu").unwrap(), + why: "store unreadable".into() + }) + ); +} + #[test] fn nibu_round_trips_from_the_committed_seed_to_its_ruled_ipa() { let seed = committed_entities(); @@ -573,160 +304,3 @@ fn the_inventory_parser_refuses_what_it_cannot_read() { assert!(PhoneInventory::parse("# c\n\ni\tvowel\ttense\ta\n").is_ok()); assert!(PhoneInventory::for_language("sa-Latn").is_none()); } - -/// `nibu-proof-001` re-cut as the request: the addressee asked Nibu to stop, -/// one frame:3 predication (n4), consent naming it, and no other claim. -fn asked() -> Packet { - let mut p = proof_packet(); - p.referents.remove(1); - let mut ask = p.predications[1].clone(); - ask.id = nd("n4"); - ask.concept = iv("concept:6"); - ask.frame = iv("frame:3"); - ask.roles = vec![ - weksa::packet::RoleFill { - role: iv("role:1"), - filler: nd("n1"), - }, - weksa::packet::RoleFill { - role: iv("role:2"), - filler: nd("n3"), - }, - ]; - p.predications = vec![ask]; - p.discourse.speech_act = Some(iv("speech_act:1")); - p.discourse.addressee_consent = Some(nd("n4")); - p.constraints.preserve = vec![nd("n1")]; - p -} - -/// The request plus a frame:2 predication (n5) the consent could wrongly name. -fn asked_and_stopped() -> Packet { - let mut p = asked(); - let mut stop = proof_packet().predications.remove(1); - stop.roles[1].filler = nd("n4"); - p.predications.push(stop); - p -} - -#[test] -fn a_consented_request_lowers_to_what_the_addressee_asked() { - let (inv, _dir) = committed(); - assert_eq!(lowered(&asked(), &inv).unwrap(), "You asked Nibu to stop."); - let mut p = asked(); - p.provenance.source = weksa::packet::ProvenanceSource::GameTelemetry; - assert_eq!(lowered(&p, &inv).unwrap(), "You asked Nibu to stop."); -} - -#[test] -fn a_request_without_its_consent_is_refused_not_invented() { - let (inv, _dir) = committed(); - let mut p = asked(); - p.discourse.addressee_consent = None; - assert_eq!( - lowered(&p, &inv), - Err(LowerError::ConsentMissing { - predication: nd("n4") - }) - ); - // Consent that names a different predication is no consent to the request. - let mut p = asked_and_stopped(); - p.discourse.addressee_consent = Some(nd("n5")); - assert_eq!( - lowered(&p, &inv), - Err(LowerError::ConsentMissing { - predication: nd("n4") - }) - ); -} - -#[test] -fn consent_to_one_of_two_requests_leaves_the_other_unconsented() { - let (inv, _dir) = committed(); - let mut p = asked(); - let mut second = p.predications[0].clone(); - second.id = nd("n5"); - p.predications.push(second); - assert_eq!( - lowered(&p, &inv), - Err(LowerError::ConsentMissing { - predication: nd("n5") - }) - ); -} - -#[test] -fn the_asker_is_the_role_the_inventory_glosses_agent_not_the_first_role() { - // role:1 and role:2 swap glosses: "agent" is now role:2, so the asker of - // `asked()` (role:1 filled by the addressee) is the one role:2 names, Nibu. - // A lookup that took the first role, or the first live entry, would read the - // addressee and let the request through. - let dir = tempfile::tempdir().unwrap(); - let mut store = InventoryStore::open(&dir.path().join("i.cc")).unwrap(); - let (rev1, mut entries) = weksa::inventory::seed(); - for e in &mut entries { - if e.id == iv("role:1") { - e.gloss = "patient".into(); - } else if e.id == iv("role:2") { - e.gloss = "agent".into(); - } - } - store.admit_revision(rev1, entries).unwrap(); - let (rev2, entries) = proof_revision(); - store.admit_revision(rev2, entries).unwrap(); - let inv = store.inventory().unwrap(); - assert_eq!( - lower(&admitted(&asked(), &store), &inv, &committed_entities()), - Err(LowerError::AskerNotAddressee { - predication: nd("n4") - }) - ); -} - -#[test] -fn a_request_by_someone_other_than_the_addressee_is_refused() { - let (inv, _dir) = committed(); - let mut p = asked(); - p.predications[0].roles[0].filler = nd("n3"); - p.predications[0].roles[1].filler = nd("n1"); - assert_eq!( - lowered(&p, &inv), - Err(LowerError::AskerNotAddressee { - predication: nd("n4") - }) - ); -} - -#[test] -fn consent_on_a_claim_that_is_no_request_stays_unrendered() { - let (inv, _dir) = committed(); - let mut p = proof_packet(); - p.discourse.addressee_consent = Some(nd("n5")); - assert_eq!( - lowered(&p, &inv), - Err(LowerError::Unrendered { - what: "addressee consent" - }) - ); -} - -#[test] -fn a_decomposition_or_import_cannot_reach_the_lowerer_with_consent() { - let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); - let inventory = InventoryStore::open(&root.join("data/interlingua/inventory.cc")).unwrap(); - let corpus = CorpusStore::open(&root.join("data/corpus/corpus.cc")).unwrap(); - let entities = EntityStore::open(&root.join("data/entity/entity.cc")).unwrap(); - for source in [ - weksa::packet::ProvenanceSource::ModelDecomposition, - weksa::packet::ProvenanceSource::ImportedCorpus, - ] { - let mut p = asked(); - p.provenance.source = source; - let dir = tempfile::tempdir().unwrap(); - let refused = PacketStore::open(&dir.path().join("p.cc")) - .unwrap() - .admit(p, &inventory, &corpus, &entities) - .unwrap_err(); - assert!(format!("{refused:?}").contains("V15"), "{source:?}"); - } -} diff --git a/crates/weksa/tests/mutants_a.rs b/crates/weksa/tests/mutants_a.rs new file mode 100644 index 0000000..a9ddda5 --- /dev/null +++ b/crates/weksa/tests/mutants_a.rs @@ -0,0 +1,25 @@ +//! Pins for behaviour of the small and middle weksa files that mutation testing +//! found unpinned. Each test states the behaviour in domain terms. + +use weksa::form::is_calendar_date; + +/// A calendar date is exactly `YYYY-MM-DD`: both hyphens sit where the format +/// puts them, so a ten-byte string with a digit in the second hyphen's place is +/// no date. +#[test] +fn a_calendar_date_needs_both_hyphens_in_place() { + assert!(is_calendar_date("2026-01-01")); + assert!(!is_calendar_date("2026-01201")); + assert!(!is_calendar_date("2026101-01")); +} + +/// February has 29 days in a Gregorian leap year and 28 otherwise. +#[test] +fn february_has_29_days_only_in_gregorian_leap_years() { + assert!(is_calendar_date("2024-02-29")); + assert!(is_calendar_date("2000-02-29")); + assert!(!is_calendar_date("1900-02-29")); + assert!(!is_calendar_date("2023-02-29")); + assert!(is_calendar_date("2023-02-28")); + assert!(!is_calendar_date("2024-02-30")); +} diff --git a/crates/weksa/tests/mutants_b.rs b/crates/weksa/tests/mutants_b.rs new file mode 100644 index 0000000..89830a6 --- /dev/null +++ b/crates/weksa/tests/mutants_b.rs @@ -0,0 +1,297 @@ +//! Pins found by mutation testing store.rs, inventory.rs and main.rs. Each test +//! states what the production code must do, in domain terms. +mod common; +use common::*; +use std::process::{Command, Output}; +use weksa::packet::Grade; +use weksa::store::EntityBound; +use weksa::{ + Annotator, Condition, EntityStore, Layer, OntologyEntry, OntologyStore, Pronunciation, Refusal, + Slot, StoreError, StoreRefusal, Test, +}; + +fn refusal(r: Result) -> StoreRefusal { + match r { + Err(StoreError::Refused(r)) => r, + other => panic!("expected a store refusal, got {other:?}"), + } +} + +fn kay_saying(p: Pronunciation) -> weksa::Entity { + let mut e = ent("kay", "en-US", &["Kay"]); + e.forms[0].pronunciation = Some(p); + e +} + +fn ruling(ipa: &str, rejected: Vec) -> Pronunciation { + Pronunciation { + ipa: ipa.into(), + ruled_by: "Kay".into(), + date: "2026-11-07".into(), + source: "test".into(), + rejected, + } +} + +#[test] +fn a_pronunciation_with_a_control_or_invisible_format_character_is_refused() { + let d = tempfile::tempdir().unwrap(); + let mut store = EntityStore::open(&d.path().join("e.cc")).unwrap(); + // Either kind alone is refused: a control character, and a zero-width space. + for ipa in ["ˈk\u{7}eɪ", "ˈk\u{200b}eɪ"] { + assert!( + matches!( + store.put(kay_saying(ruling(ipa, vec![]))), + Err(StoreError::Refused(StoreRefusal::Malformed { .. })) + ), + "{ipa:?}" + ); + } + // The same for a rejected pronunciation. + for bad in ["ˈk\u{7}aɪ", "ˈk\u{200b}aɪ"] { + assert!( + matches!( + store.put(kay_saying(ruling("ˈkeɪ", vec![bad.into()]))), + Err(StoreError::Refused(StoreRefusal::Malformed { .. })) + ), + "{bad:?}" + ); + } + assert_eq!(store.get(&eid("kay")).unwrap(), None); +} + +#[test] +fn a_pronunciation_keeps_at_most_eight_rejected_variants() { + let d = tempfile::tempdir().unwrap(); + let mut store = EntityStore::open(&d.path().join("e.cc")).unwrap(); + let variants = |n: usize| -> Vec { (0..n).map(|i| format!("ˈkaɪ{i}")).collect() }; + assert_eq!( + refusal(store.put(kay_saying(ruling("ˈkeɪ", variants(9))))), + StoreRefusal::EntityBounds { + id: eid("kay"), + bound: EntityBound::Forms + } + ); + store + .put(kay_saying(ruling("ˈkeɪ", variants(8)))) + .expect("eight rejected variants are the bound, not beyond it"); +} + +#[test] +fn an_ontology_entry_read_at_an_unknown_revision_is_refused_with_one_refusal() { + let w = world(); + let mut onto = OntologyStore::open(&w.dir.path().join("o2.cc")).unwrap(); + let entry = OntologyEntry { + language: "en-US".into(), + n: 1, + inventory_rev: 99, + layer: Layer::Discourse, + when: vec![Condition { + slot: Slot::Stance, + feature: iv(ANGER), + test: Test::AtLeast(Grade::new(200).unwrap()), + }], + choice: "WARM".into(), + examples: vec![], + annotator: Annotator::Operator, + }; + match onto.admit(vec![entry], &w.inventory) { + Err(StoreError::Invalid(r)) => assert_eq!(r, vec![Refusal::V10UnknownRevision { rev: 99 }]), + other => panic!("{other:?}"), + } +} + +fn weksa(dir: &std::path::Path, args: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_weksa")) + .current_dir(dir) + .args([ + "--corpus", + "./c.cc", + "--inventory", + "./i.cc", + "--entities", + "./e.cc", + "--ontology", + "./o.cc", + "--speaker-state", + "./s.cc", + ]) + .args(args) + .output() + .unwrap() +} + +fn out(o: &Output) -> String { + String::from_utf8_lossy(&o.stdout).into_owned() + &String::from_utf8_lossy(&o.stderr) +} + +fn ok(o: Output) -> String { + assert!(o.status.success(), "{}", out(&o)); + out(&o) +} + +/// A scene bed: entities nibu and brokkr, the inventory through the +/// speaker-scene revision (rev 3). +fn scene_bed() -> tempfile::TempDir { + let d = tempfile::tempdir().unwrap(); + for slug in ["nibu", "brokkr"] { + let id = format!("entity:{slug}"); + let form = format!("en-US={}", slug.to_uppercase()); + ok(weksa( + d.path(), + &[ + "entity", + "put", + &id, + "--form", + &form, + "--origin", + "cast", + "--persona", + slug, + "--fixture", + "f.yaml", + ], + )); + } + for step in ["seed", "proof", "speaker-scene"] { + ok(weksa(d.path(), &["inventory", step])); + } + d +} + +/// Adds a cast-authored line; returns the key it printed ("Added "). +fn add_line(d: &std::path::Path, slug: &str, speaker: &str, group: &str, text: &str) -> String { + let speaker = format!("entity:{speaker}"); + let printed = ok(weksa( + d, + &[ + "corpus", + "add", + "--slug", + slug, + "--text", + text, + "--speaker", + &speaker, + "--group", + group, + "--admitted-on", + "2026-10-29", + "--source", + "cast-authored", + "--language", + "en-US", + "--persona", + "nibu", + "--fixture", + "f.yaml", + "--author", + "Ada Writer", + ], + )); + printed.split_whitespace().nth(1).unwrap().to_string() +} + +fn scene_put(d: &std::path::Path, group: &str, to: &str) -> Output { + weksa( + d, + &[ + "scene", + "put", + "--group", + group, + "--to", + to, + "--inventory-rev", + "3", + "--model-family", + "anthropic", + "--generator", + "a scene annotator", + "--set", + "intimacy=200", + ], + ) +} + +#[test] +fn a_scene_group_holds_one_live_speaker_among_its_own_live_items() { + let d = scene_bed(); + add_line(d.path(), "nibu", "nibu", "scene-1", "Mind the hatch."); + // Another group's speaker and a withdrawn line of this group are not the + // scene's speakers. + add_line(d.path(), "brokkr", "brokkr", "scene-2", "Brokkr hums."); + let gone = add_line( + d.path(), + "brokkr", + "brokkr", + "scene-1", + "Brokkr leaves early.", + ); + ok(weksa(d.path(), &["corpus", "withdraw", &gone])); + let put = ok(scene_put(d.path(), "scene-1", "unnamed")); + assert!(put.contains("scene:scene-1@1"), "{put}"); + assert!(ok(weksa(d.path(), &["scene", "dump"])).contains("entity:nibu->unnamed")); +} + +#[test] +fn a_scene_names_its_addressee_as_a_participant_unless_it_is_the_speaker() { + let d = scene_bed(); + add_line(d.path(), "nibu", "nibu", "scene-1", "Mind the hatch."); + add_line(d.path(), "nibu", "nibu", "scene-2", "Mind the ladder."); + ok(scene_put(d.path(), "scene-1", "entity:brokkr")); + let dump = ok(weksa(d.path(), &["scene", "dump"])); + assert!(dump.contains("scene:scene-1@1 rev:3"), "{dump}"); + assert!(dump.contains("[entity:nibu,entity:brokkr]"), "{dump}"); + // A speaker addressing themself is a one-participant scene, whose + // relation to a named party is refused as the store reads it. + let own = scene_put(d.path(), "scene-2", "entity:nibu"); + assert!(!own.status.success()); + assert!( + out(&own).contains("a relation's named to is another participant"), + "{}", + out(&own) + ); + assert_eq!(ok(weksa(d.path(), &["scene", "dump"])), dump); +} + +#[test] +fn relower_refuses_malformed_arm_and_floor_labels_before_reading_the_run() { + let d = tempfile::tempdir().unwrap(); + let relower = |arm: &str, floor: &[&str]| { + let mut args = vec![ + "loop", + "relower", + "no-such-run", + "--round", + "1", + "--arm", + arm, + "--lowerer", + "m", + ]; + for f in floor { + args.extend(["--floor", f]); + } + weksa(d.path(), &args) + }; + for (arm, floor) in [ + ("BAD", vec!["b"]), + ("a", vec!["Bad"]), + ("a", vec!["a", "b"]), + ("a", vec!["b", "b"]), + ] { + let o = relower(arm, &floor); + assert!(!o.status.success()); + assert!( + out(&o).contains("arm and floor labels must match"), + "{arm} {floor:?}: {}", + out(&o) + ); + } + // Well-formed, distinct labels get past the label check (to the missing run). + let o = relower("a", &["b", "c"]); + assert!(!o.status.success()); + assert!(out(&o).contains("the root holds no run"), "{}", out(&o)); +} diff --git a/crates/weksa/tests/mutants_c.rs b/crates/weksa/tests/mutants_c.rs new file mode 100644 index 0000000..70c499d --- /dev/null +++ b/crates/weksa/tests/mutants_c.rs @@ -0,0 +1,136 @@ +//! Pins for behaviour of `context.rs`, `encode.rs` and `run.rs` found unpinned +//! by mutation testing. Each test states what the code does in domain terms. + +mod common; + +use std::path::{Path, PathBuf}; + +use weksa::context::Prompt; +use weksa::entity::{Entity, EntityOrigin, LanguageForms}; +use weksa::packet::{EntityId, proof_packet}; +use weksa::{CorpusItemId, CorpusStore, EntityForms, EntityStore, InventoryStore, PacketStore}; + +fn data(sub: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../data") + .join(sub) +} + +struct Held(Vec); + +impl EntityForms for Held { + fn entity(&self, id: &EntityId) -> anyhow::Result> { + Ok(self.0.iter().find(|e| &e.id == id).cloned()) + } +} + +fn spoken(slug: &str, form: &str) -> Entity { + Entity { + id: EntityId::parse(&format!("entity:{slug}")).unwrap(), + forms: vec![LanguageForms { + language: "en-US".into(), + forms: vec![form.into()], + pronunciation: None, + }], + origin: EntityOrigin::Cast { + persona: slug.into(), + fixture: "f.yaml".into(), + }, + } +} + +/// A speaker whom no referent cites contributes no entry to the lowering +/// prompt, even when another entity is cited: only the cited entity is listed. +#[test] +fn an_uncited_speaker_is_absent_from_the_lowering_prompt() { + let cast_inv = InventoryStore::open(&data("interlingua/inventory.cc")).unwrap(); + let entities = EntityStore::open(&data("entity/entity.cc")).unwrap(); + let corpus = CorpusStore::open(&data("corpus/corpus.cc")).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let view = PacketStore::open(&dir.path().join("p.cc")) + .unwrap() + .admit(proof_packet(), &cast_inv, &corpus, &entities) + .unwrap(); + let state = weksa::SpeakerStateStore::open(&dir.path().join("s.cc")).unwrap(); + let view = view + .with_scene(&cast_inv.inventory().unwrap(), &state, &corpus) + .unwrap(); + let held = Held(vec![ + spoken("nibu", "Nibu"), + spoken("other", "SPEAKERCANARY"), + ]); + let item = CorpusItemId::parse("item:nibu:1").unwrap(); + let speaker = EntityId::parse("entity:other").unwrap(); + let prompt = Prompt::lower( + &view, + &item, + &speaker, + "en-US", + &cast_inv.inventory().unwrap(), + &held, + &weksa::Ontology::empty(), + ) + .unwrap(); + let text = prompt.text(); + assert!( + text.contains("\n\nentities: {\"entity:nibu\":[\"Nibu\"]}\n\n"), + "{text}" + ); + assert!(!text.contains("SPEAKERCANARY"), "{text}"); +} + +/// A run refuses a panel language that lacks a live Train or a live Validation +/// item: one no item carries, one with only a Test item, one with only Train. +#[test] +fn a_panel_language_no_item_carries_is_refused_at_open() { + let dir = tempfile::tempdir().unwrap(); + let d = dir.path(); + // Builds the corpus, entity and ontology stores the run reads. + drop(common::run_over(d)); + let entities = common::entities_in(d); + let mut corpus = CorpusStore::open(&d.join("c.cc")).unwrap(); + for (key, language, bucket) in [("item:jp:1", "ja-JP", 90), ("item:pt:1", "pt-BR", 10)] { + let mut item = common::item_in( + key, + &format!("line of {key}"), + &common::group_in_bucket(bucket), + common::cast(), + ); + item.language = language.into(); + corpus.add(item, &entities).unwrap(); + } + drop(corpus); + let create = |panel: &[&str]| { + weksa::run::RunStores::create( + &d.join(format!("run-{}", panel.join("-"))), + "t", + weksa::run::RunPaths { + corpus: &d.join("c.cc"), + inventory: &d.join("i.cc"), + entities: &d.join("entities.cc"), + ontology: &d.join("ontology.cc"), + speaker_state: &common::speaker_state_beside(&d.join("ontology.cc")), + }, + "fake-model", + "bag", + weksa::run::LowererReach::ToolLessRelay, + weksa::corpus::ModelFamily::Unrecorded, + &panel.iter().map(|t| (*t).to_string()).collect::>(), + ) + }; + assert!(create(&["en-US"]).is_ok()); + for panel in [&["ja-JP"][..], &["pt-BR"], &["xx-XX"], &["en-US", "ja-JP"]] { + assert!( + matches!(create(panel), Err(weksa::run::RunError::Panel)), + "{panel:?}" + ); + } +} + +/// A run keeps its recall probes in `recall` under its own root. +#[test] +fn a_run_keeps_its_recall_under_its_root() { + let dir = tempfile::tempdir().unwrap(); + let run = common::run_over(dir.path()); + assert_eq!(run.recall_dir(), dir.path().join("run").join("recall")); +} diff --git a/crates/weksa/tests/scene.rs b/crates/weksa/tests/scene.rs index 5974883..e4841e6 100644 --- a/crates/weksa/tests/scene.rs +++ b/crates/weksa/tests/scene.rs @@ -11,7 +11,7 @@ use weksa::packet::{FeatValue, FeatureValue, Packet, proof_packet}; use weksa::speaker::{Annotator, PairRelation, Party, Scene}; use weksa::{ CorpusStore, EntityStore, Inventory, InventoryStore, LoweringInput, NameSpan, PacketStore, - Refusal, SceneRead, SpeakerStateStore, StoreError, lower, + Refusal, SceneRead, SpeakerStateStore, StoreError, }; struct W { @@ -158,7 +158,7 @@ fn selected(rev: u32, features: Vec) -> SceneRead { } #[test] -fn a_scene_relation_reaches_the_lowering_view_and_changes_no_english() { +fn a_scene_relation_reaches_the_lowering_view() { let mut w = world(); put(&mut w, standard("g", 1)); let plain = view_of(&w, 3, Some("n3")); @@ -170,10 +170,6 @@ fn a_scene_relation_reaches_the_lowering_view_and_changes_no_english() { .unwrap(); assert_eq!(with.scene(), status().as_slice()); assert_eq!(with.scene_read(), &selected(1, status())); - assert_eq!( - lower(&with, &inv(&w), &w.entities).unwrap(), - lower(&plain, &inv(&w), &w.entities).unwrap() - ); } #[test] diff --git a/crates/weksa/tests/split_schema.rs b/crates/weksa/tests/split_schema.rs index bb6bc78..19749de 100644 --- a/crates/weksa/tests/split_schema.rs +++ b/crates/weksa/tests/split_schema.rs @@ -9,6 +9,7 @@ fn item(group: &str, source: Source) -> CorpusItem { language: "en-US".into(), speaker: weksa::EntityId::parse("entity:s").unwrap(), names: vec![], + names_by: None, group: group.into(), prior_turns: vec![], source, diff --git a/crates/weksa/tests/validate.rs b/crates/weksa/tests/validate.rs index b46cd85..057e92f 100644 --- a/crates/weksa/tests/validate.rs +++ b/crates/weksa/tests/validate.rs @@ -740,6 +740,53 @@ fn a_source_that_only_read_wording_cannot_set_consent() { } } +// V18: a request frame is licensed by the packet's own consent record. + +/// The committed inventory, where `frame:3` is `request_intervention`. +fn cast_refusals(p: &Packet) -> Vec { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/interlingua/inventory.cc"); + let inv = weksa::InventoryStore::open(&path).unwrap().inventory().unwrap(); + validate(p, &inv).err().unwrap_or_default() +} + +fn requesting() -> Packet { + let mut p = proof_packet(); + p.predications[0].frame = iv("frame:3"); + p.discourse.addressee_consent = Some(nd("n4")); + p +} + +#[test] +fn a_request_frame_needs_consent_naming_its_predication() { + assert_eq!(cast_refusals(&requesting()), vec![]); + let mut p = requesting(); + p.discourse.addressee_consent = None; + assert_eq!( + cast_refusals(&p), + vec![Refusal::V18RequestWithoutConsent { predication: nd("n4") }] + ); + p.discourse.addressee_consent = Some(nd("n5")); + assert_eq!( + cast_refusals(&p), + vec![Refusal::V18RequestWithoutConsent { predication: nd("n4") }] + ); + let mut p = requesting(); + p.provenance.source = ProvenanceSource::GameTelemetry; + assert_eq!(cast_refusals(&p), vec![]); +} + +#[test] +fn model_source_consent_is_v15_alone() { + let mut p = requesting(); + p.provenance.source = ProvenanceSource::ModelDecomposition; + assert_eq!( + cast_refusals(&p), + vec![Refusal::V15ConsentSource { + source: ProvenanceSource::ModelDecomposition + }] + ); +} + // V17: the addressee is someone spoken to, a referent. #[test] diff --git a/docs/english-pronunciation-lowering.md b/docs/english-pronunciation-lowering.md index 0e2f7cc..acd5e37 100644 --- a/docs/english-pronunciation-lowering.md +++ b/docs/english-pronunciation-lowering.md @@ -85,8 +85,8 @@ thing. An entity's citation form in a language may carry a ruled pronunciation (`LanguageForms.pronunciation`): bare IPA with stress marks, who ruled it, the date, the source, and an optional list of rejected variants kept so nobody -re-proposes them. `weksa lower --ipa` prints the stored IPA of each -entity the packet cites, one per line, and refuses with `NoPronunciation` for an +re-proposes them. `weksa pronounce ` prints the stored IPA of each +entity the packet cites, one per line (`--phones` prints the phones line), and refuses with `NoPronunciation` for an entity with none; rejected variants are never read. This is not the plan below: it carries no timing, contour, length or invented prosody, and Weksa has no English G2P, so an unruled name has no IPA. diff --git a/docs/interlingua-standard.md b/docs/interlingua-standard.md index f8e7937..ca8b860 100644 --- a/docs/interlingua-standard.md +++ b/docs/interlingua-standard.md @@ -164,6 +164,7 @@ names the rule. `GameTelemetry`; a decomposition of text or imported corpus text cannot mint it. - V17: an `addressee` names a referent, not a predication. +- V18: a predication whose frame is the inventory's `request_intervention` is named by `addressee_consent`. V9 is retired and not renumbered: the form of a key or a grade is its type, and an extension key's family is V6. `PacketStore::admit` returns the lowering view of a @@ -196,10 +197,24 @@ and `features` carry source-side nuance on content (hedge, certainty, irony and whatever the inventory grows). This is source-side communicative pressure, not target-language register. `addressee_consent` is no feature: it names the predication the addressee asked for or agreed to, a fact about what the -addressee did. The English lowerer's request row (`frame:3`, "{agent} asked -{patient} to stop") reads it: the consent must name that predication and its -agent must be the addressee, else `ConsentMissing` or `AskerNotAddressee`. A -consent naming any other predication is refused as unrendered. +addressee did. `V14` and `V15` check the field: it may be set only by authoring or game +telemetry, and it must name a predication of the packet. `V18` refuses a predication whose frame resolves, at the packet's revision, to +the gloss `request_intervention` (`frame:3` in the committed inventories) +unless `addressee_consent` names that predication. The encoder can never set +consent (`V15`), so an encoder-produced `request_intervention` predication (`frame:3`) is always refused. +Limits, tracked by `weksa-lowering:follow_up:request-frame-consent-residual`: +`V18` keys on the gloss `request_intervention` and sees no other frame or +concept, so a request-shaped meaning built from another frame or concept (a +`frame:12` communication with `concept:6` request validates) or from a renamed +or newly minted frame is not seen by `validate`, which does not read meaning. +The inventory carries no marker for request shape, so classifying it would be +an inventory-shape decision for Imagination and language-ontology. `addressee_consent` +is one field, so a packet carries one request predication, and a compound +request is two packets until a packet schema revision makes consent a set. The +asker is not checked to be the addressee: a consented `frame:3` whose agent is +not the addressee validates, and no code outside `validate` and the encoder +prompt text reads `addressee_consent`. That check is an owed obligation with no +owner yet. ### `stance` diff --git a/docs/name-span-annotation.md b/docs/name-span-annotation.md new file mode 100644 index 0000000..546d57b --- /dev/null +++ b/docs/name-span-annotation.md @@ -0,0 +1,52 @@ +# Name-span annotation + +How the name spans of an imported public-domain work are placed. A span has a +committed rule that produced it; this file is that rule. + +## Why + +- `pd-annotation-model-then-attest`: a labelled model places the spans of a + public-domain work; the operator may attest them afterwards. +- `leak-owned-by-flow-and-scorer-3`: spans are fixed with the text at intake and + written by no loop writer. The offsets are derived from markup the annotator + writes, never typed by a model (inline marking, as in GPT-NER, Wang et al. + 2023); `CorpusStore::add` judges range, grapheme boundary, form equality and + overlap. + +The work file (`weksa corpus import `) is a transcription of a work in +hand: provenance, cast forms, lines, scene groups and the speaker each speech +prefix names. It holds no distributor header, licence or trademark text, and the +binary fetches nothing. + +## Rubric r1 + +You are given a work's cast: each member's entity id and the forms the work spells that member with. For each numbered line, copy the line exactly, character for character, and wrap each place where the line names a cast member by one of that member's listed forms as [[|]]. Mark a form only where it refers to that cast member; a listed form used as an ordinary word, or naming someone else, stays unmarked. Do not mark pronouns, titles or descriptions unless the exact text is a listed form. When a form is listed for two members, mark the one the line refers to; when you cannot tell, leave it unmarked. Answer one JSON line per input line, in the order given, and nothing else: {"line": N, "marked": "..."}. + +## Procedure + +- One fresh model agent per work. Its brief is rubric r1, the cast as + `: , ` lines and the work's lines as ``; + nothing else (no store, no loop, no other answer). +- Import with `--model-family` and `--generator ' (name-span + rubric r1, docs/name-span-annotation.md)'`. An operator-written or + operator-reviewed answer imports with `--operator`. +- Import is all or nothing: every entity and item passes the real store doors on + scratch copies before the first write, so a refusal (a parse failure, a span + the store judges, a work that repeats a line under case or width folding) + leaves the stores untouched. Re-draw the whole answer once, log it, and import + again. A work file repeating a line's text is refused naming both lines; leave + that line out of the answer file (the rest imports with its numbering intact), + since a repeated line cannot be two corpus items. +- A line the answer does not cover is not imported by that run; a later run over + other lines of the work does not collide (keys are `item::`). +- A line whose own text holds `[[` or `]]` cannot be marked and is refused; it + can still enter by hand through `weksa corpus add --name`. + +## Label scope + +`names_by` labels the spans only, not the line or its speaker. One label per +item: a later attestation would be a superseding item with the same text and +group, but no command writes one yet; question `pd-name-span-attestation` owns +that door. +`None` means unrecorded (the cast seeds, `weksa corpus add`, the Dao De Jing +bed), not attested. diff --git a/docs/research/2026-10-29-golf-loop-prior-art.md b/docs/research/2026-10-29-golf-loop-prior-art.md index e2a24ae..7fb5510 100644 --- a/docs/research/2026-10-29-golf-loop-prior-art.md +++ b/docs/research/2026-10-29-golf-loop-prior-art.md @@ -65,7 +65,7 @@ Verified licence facts are few. Discord data excluded by policy; not read. - PersonaChat (Zhang et al. 2018): ParlAI project page, archived 2020-08-01, states no licence, only "You may continue to use the data without issue". [V] https://parl.ai/projects/personachat/ ; paper https://arxiv.org/abs/1801.07243 says data released open source via ParlAI; licence text [GAP]. Crowdworker-written, persona-statement-conditioned chit-chat [B]. - LIGHT: ParlAI page states no licence; contents: 663 locations, 3,462 objects, 1,755 character types, 11,000 episodes; WILD 41,131 training episodes from game deployment (human players, so a different provenance than crowdwork). [V] https://parl.ai/projects/light/ . Licence [GAP: check ParlAI repo]. - Cornell Movie-Dialogs Corpus: licence/terms page fetch failed (certificate rejected) https://www.cs.cornell.edu/~cristian/Cornell_Movie-Dialogs_Corpus.html . [GAP] Underlying scripts are copyrighted film text [B]; dataset licence unverified. -- Project Gutenberg: "vast majority" of eBooks public domain in the US, so no one can grant or withhold permission for uses; PG cannot sublicense copyrighted items; subject to its Terms of Use, which the permission page links and does not summarise; no statement on ML/bulk use on that page. [V] https://www.gutenberg.org/policy/permission.html . Terms-of-Use text on bulk/automated access and the trademark licence: [GAP, not read]. Items public domain in the US may not be elsewhere (same page). +- Project Gutenberg: "vast majority" of eBooks public domain in the US, so no one can grant or withhold permission for uses; PG cannot sublicense copyrighted items; subject to its Terms of Use, which the permission page links and does not summarise; no statement on ML/bulk use on that page. [V] https://www.gutenberg.org/policy/permission.html . Terms-of-Use text on bulk/automated access and the trademark licence: read 2026-11-13, see `2026-11-13-gutenberg-terms.md`. Items public domain in the US may not be elsewhere (same page). - Google Synthetic-Persona-Chat (model-generated, persona-chat-style): CC BY-NC? listed as cc-by-4.0 on HF. [V search-only] https://huggingface.co/datasets/google/Synthetic-Persona-Chat - Operator-authored or model-generated corpus: no licence question beyond provider terms [B, unread this pass]. Problems with evaluating on model-generated text: see 2 (self-bias, preference leakage, stylistic homogeneity: Xu et al. 2025; Li et al. 2025). The same Xu et al. finding says it extends to open-ended generation. - Expressive-register-annotated English corpora (hedging, humour, affect, politeness labelled), with permissive terms: [GAP] not searched beyond the list above (e.g. Stanford Politeness Corpus, GYAFC formality, Enron etc. are [B] candidates, unverified, GYAFC is known to carry Yahoo Answers terms [B]). @@ -83,6 +83,6 @@ Verified licence facts are few. Discord data excluded by policy; not read. - No per-metric human-correlation numbers for hedging, humour, affect, stance, persona voice; nearest is formality (Briakou 2021). COMET, BLEURT, MAUVE correlations not obtained. - No source on surface-form smuggling through an LLM bottleneck or its defences (vocab restriction, paraphrase invariance). - No source on schema/inventory growth from generation failures, or its overfitting controls. -- Cornell, LIGHT, PersonaChat licence texts not obtained; Gutenberg ToU on bulk use not read. +- Cornell, LIGHT, PersonaChat licence texts not obtained. - No realiser or evaluation protocol for constructed languages from a meaning representation. - Several arXiv PDFs (Briakou, Zheng, STEL, LUAR/StyleDistance) fetched as binary; numbers above come from the ar5iv/arXiv-HTML versions where marked, otherwise search summaries. diff --git a/docs/research/2026-11-13-gutenberg-terms.md b/docs/research/2026-11-13-gutenberg-terms.md new file mode 100644 index 0000000..1d2f596 --- /dev/null +++ b/docs/research/2026-11-13-gutenberg-terms.md @@ -0,0 +1,59 @@ +# Project Gutenberg terms: bulk access and header stripping + +Read 2026-11-13 for follow-up `gutenberg-terms` (campaign `weksa-lowering`), +ahead of cut `pd-importer`. Pages: gutenberg.org `policy/permission.html`, +`policy/terms_of_use.html`, `policy/robot_access.html`, `policy/license.html`. +The quotes were checked against the live HTML of each page by a second reader. +`pd-importer`'s Imagination re-reads the pages before it fixes a fetch path, +since they change. + +## Findings + +- **Automated access to the main site is refused.** Terms of Use: "This website + is intended for human users only"; "Any perceived use of automated tools to + access this website will result in a temporary or permanent block". OPDS + feeds: no more requests than a browser user would make; heavy requesters are + blocked by IP. +- **Bulk has two sanctioned paths.** Terms of Use: take bulk downloads "from + one of our mirrors, not from the main site". The robot-access page names the + `robot/harvest` wget endpoint (`-w 2`, filterable by file type or language) + and recommends a private mirror for a full local copy; it suggests checking + periodically for changes. Catalog data are offered as + machine-readable offline catalogs, "granted to the public domain". +- **Header and licence stripping.** Licence section 1.C: for a work not + protected by US copyright, with the user in the US, the Foundation does "not + claim a right to prevent you from copying, distributing, performing, + displaying or creating derivative works", "as long as all references to + Project Gutenberg are removed", the condition holding for a user in the US. + Section 1.E opens "Unless you have removed all references to Project + Gutenberg", and 1.E.1 to 1.E.7 (including 1.E.4, do not detach the licence) + attach to a work on which the phrase "Project Gutenberg" appears or with which + it is associated, so a text with every reference removed is outside 1.E. + 1.E.2 is not that grant: it says a PG eBook derived from texts not restricted + by US copyright may be copied and distributed in the US without fee. The + trademark is registered; non-commercial use of the name needs no permission, + commercial use does. +- **No statement on machine learning or AI use** on any of the four pages. +- **US public domain only.** The Foundation makes no claim about copyright + outside the US (1.D). A work free in the US may not be free elsewhere, and + Gutenberg's own catalog is not a per-work clearance. + +## What this decides for `pd-importer` + +- Fetch from a mirror or an offline dump. The Terms of Use footnote counts "many + books" as more than about 100 a day and sends those to mirrors, while + `robot/harvest` is a `www.gutenberg.org` URL: use it (with `-w 2`) only for a + handful of works. Never crawl `gutenberg.org` pages or OPDS from the loop host. +- Strip the whole header and footer, every Project Gutenberg reference and the + licence text from the stored text; keep provenance as the ebook number and + edition in the item's source field, which is Weksa's own record and not a use + of the mark. This is the path `docs/targets/dao-de-jing-test-bed.md` already + takes. The remaining open question there, what counts as a "reference", stays + open: the safe reading is that the ebook number in a provenance field is a + catalogue identifier, not a reference inside the work, but nothing read here + rules on it. +- Per-work check stays with the importer: the work's author and first + publication date against US public-domain rules, recorded on the item. The + Foundation's catalogue does not supply it. A work Gutenberg posts with the + copyright holder's permission (1.E.3) is out of scope. +- No importer code in this note. diff --git a/docs/targets/dao-de-jing-test-bed.md b/docs/targets/dao-de-jing-test-bed.md index bd2ebbf..0af4831 100644 --- a/docs/targets/dao-de-jing-test-bed.md +++ b/docs/targets/dao-de-jing-test-bed.md @@ -23,9 +23,10 @@ plus a separately recorded chosen gloss per lowering). `corpus add ... --language en-US --source public-domain` per item, then `packet dao `. - Source: Project Gutenberg eBook 216, "The Tao Teh King, or the Tao and its Characteristics", Laozi, tr. James Legge (Sacred Books of the East 39, 1891). - Gutenberg's licence (1.E.2) lets a work derived from texts not protected by US - copyright be used with no fee provided every Project Gutenberg reference is - removed from the work. The items hold only Legge's words, one numbered section + Gutenberg's licence (1.C) does not claim a right to prevent derivative works of + a work not protected by US copyright, for a user in the US, provided every + Project Gutenberg reference is removed (see + `docs/research/2026-11-13-gutenberg-terms.md`). The items hold only Legge's words, one numbered section or sentence each; no Gutenberg header, licence text or trademark is stored, and the eBook file itself is not committed. `edition_url` names the page the text was retrieved from, as a citation of provenance and not as part of any @@ -80,6 +81,8 @@ ceiling of an author who saw the reference: frame minted from Train, scored lowest (chrF 44.18). The scores rise with how bespoke the frame is, the reverse of generalisation. The Validation and Test numbers measure the lowerer table, not what the interlingua can carry. + That table is deleted: the Dao numbers below were measured on it, and the + loop now lowers through the model (`Prompt::lower`). - n = 5, one translation as reference, one embedding model, no variance. ## What the scorer cannot see diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index f9cf105..2a54223 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -38,7 +38,7 @@ means no store holds it and nothing writes it. | Community (cut `language-ontology`, follow-up `language-ontology-communities`) | Key `community:`; type `weksa.community`, built fresh when the cut is mapped: `data/cultural-ontology/` (draft YAML profiles, read by no code) is deleted from main (cut 5e `delete-cultural-ontology`) and stays in git history before that cut's commit | Revised by commit; a speaker names the communities it belongs to by id | The cultural ontology's author (owner-attested evidence only; Discord-only content stays deleted, `discord-only-content-deleted`) | | Speaker (cut `speaker-scene`) | Key `speaker:@`; type `weksa.speaker`: stable axes as feature values of slot `Speaker` (origin, socioeconomic standing, age cohort, education and whatever the loop adds), at an inventory revision, and an `Annotator` | A new revision appends, never edits; a corpus item and every prior turn name a speaker slug, and a round records the speaker revisions it read | The annotator it names: `PersonaSnapshot` (a projection of a `gamecult.persona_state.v0` document, naming its source document and `updatedAt`), `Operator`, `WorkMetadata` (a public-domain work's own cast list and narration), or `Model { family, generator }` (the 2b `ModelFamily`) | | Scene (cut `speaker-scene`) | Key `scene:@`, the item's split group; type `weksa.scene`: participants (speaker slugs) and, per ordered pair, relation features of slot `Scene` (relative status, intimacy, in-group, audience and whatever the loop adds), at an inventory revision (provenance; a run reads the features at its own head), and an `Annotator` | As Speaker. Keyed by the group, so a scene never straddles the split. `data/original/speaker-state.cc` holds Model-annotated keeper scenes, one per Train or Validation group of both languages (`weksa scene put|dump`); a Test group has none and reads `NoScene` | As Speaker; a model-annotated scene is never Test evidence | -| Corpus item | Key `item::`, `n` canonical decimal >= 1 (no sign, no leading zero, from cut 2b); type `weksa.corpus_item` with text, language, speaker, split group, prior turns, source. Source `ModelWritten` carries a `ModelFamily` enum, the generator string as the fixture wrote it, and a prompt digest (cut 2b). The item carries no speaker state of its own: its speaker slug and group name the Speaker and Scene documents | Admitted once; the whole item is immutable (re-adding an identical item is a no-op, any difference is refused; a correction is a new item that `supersedes` the old); text already held under another key is refused unless the new item supersedes that key (cut 2b); withdrawn, never deleted, so old scores stay explainable | Importers and `weksa corpus add`, under `corpus-cast-plus-public-domain`; the source enum has no variant for Discord or any training-forbidden source (`corpus-clean`) | +| Corpus item | Key `item::`, `n` canonical decimal >= 1 (no sign, no leading zero, from cut 2b); type `weksa.corpus_item` with text, language, speaker, split group, prior turns, source and who placed its name spans (`names_by`, an `Annotator`, absent when unrecorded; cut `pd-name-spans`). Source `ModelWritten` carries a `ModelFamily` enum, the generator string as the fixture wrote it, and a prompt digest (cut 2b). The item carries no speaker state of its own: its speaker slug and group name the Speaker and Scene documents | Admitted once; the whole item is immutable (re-adding an identical item is a no-op, any difference is refused; a correction is a new item that `supersedes` the old); text already held under another key is refused unless the new item supersedes that key (cut 2b); withdrawn, never deleted, so old scores stay explainable | Importers and `weksa corpus add`, under `corpus-cast-plus-public-domain`; the source enum has no variant for Discord or any training-forbidden source (`corpus-clean`) | | Split assignment | Derived: `split(scheme, item.group)` | Fixed for a scheme; a new scheme is a new constant, never a reassignment. A model-written item whose group derives test is in no split; a superseding item keeps its predecessor's group | The hash function. Nobody assigns by hand, so there is no writer to forbid | | Loop round | Key `run: