From 5eb58c31001bf40a8e15dd71389f85d86a3e016d Mon Sep 17 00:00:00 2001 From: Weksa Date: Sat, 7 Nov 2026 02:10:10 +0000 Subject: [PATCH 001/175] Dao De Jing test bed: PublicDomain provenance fields, Legge slice, first golf-loop number Source::PublicDomain now requires licence, jurisdiction, translator and edition_year; the store refuses an item missing any. data/daodejing holds five Legge items (Gutenberg 216, no Gutenberg matter stored), inventory revision 3, five packets with stance, and the lowerer rows to render them. docs/targets/dao-de-jing-test-bed.md reports the first closeness number and what the scorer cannot see. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/corpus.rs | 9 + crates/weksa/src/daodejing.rs | 325 ++++++++++++++++++ crates/weksa/src/lib.rs | 1 + crates/weksa/src/lower.rs | 47 +++ crates/weksa/src/main.rs | 55 ++- crates/weksa/src/store.rs | 12 + crates/weksa/tests/cli.rs | 88 +++++ crates/weksa/tests/daodejing.rs | 71 ++++ crates/weksa/tests/stores.rs | 77 ++++- data/daodejing/corpus.cc | 1 + data/daodejing/entity.cc | 1 + data/daodejing/inventory.cc | Bin 0 -> 10361 bytes data/daodejing/packets.cc | 1 + docs/targets/dao-de-jing-test-bed.md | 116 +++++++ schemas/cultnet/weksa.corpus_item.schema.json | 22 +- 15 files changed, 823 insertions(+), 3 deletions(-) create mode 100644 crates/weksa/src/daodejing.rs create mode 100644 crates/weksa/tests/daodejing.rs create mode 100644 data/daodejing/corpus.cc create mode 100644 data/daodejing/entity.cc create mode 100644 data/daodejing/inventory.cc create mode 100644 data/daodejing/packets.cc create mode 100644 docs/targets/dao-de-jing-test-bed.md diff --git a/crates/weksa/src/corpus.rs b/crates/weksa/src/corpus.rs index d67cc60..0beaa17 100644 --- a/crates/weksa/src/corpus.rs +++ b/crates/weksa/src/corpus.rs @@ -29,10 +29,19 @@ pub enum Source { OperatorApproved { note: String, }, + /// A public-domain work, with everything a reader needs to check the claim: + /// the edition's licence and the jurisdiction the public-domain status is + /// asserted in, who translated it (the translation is the text's copyright, + /// not the original's) and when the edition appeared. A work missing any + /// is refused at admission. PublicDomain { work: String, author: String, edition_url: String, + licence: String, + jurisdiction: String, + translator: String, + edition_year: u16, }, /// Labelled model output; never in the test split, never judged by its own family. /// `family` is `Unrecorded` when nothing records one. `generator` is the diff --git a/crates/weksa/src/daodejing.rs b/crates/weksa/src/daodejing.rs new file mode 100644 index 0000000..acd45f3 --- /dev/null +++ b/crates/weksa/src/daodejing.rs @@ -0,0 +1,325 @@ +//! The Dao De Jing test bed: inventory revision 3 of the `data/daodejing` +//! stores and the packets for the slice of James Legge's chapters in its corpus. +//! It follows `proof_packet`'s precedent: a hand-authored packet the stores +//! admit through the same doors as any other. +//! +//! The ids carry the leak labels the golf loop needs. A frame minted while +//! reading a Train chapter is generic evidence; one minted for a Validation or +//! Test chapter was written with that chapter's reference in view, so a score +//! on it measures the lowerer table's reach and not the interlingua's. + +use std::num::NonZeroU32; + +use crate::corpus::CorpusItemId; +use crate::inventory::{Change, Family, InvId, InventoryEntry, InventoryRevision}; +use crate::packet::{ + Constraints, Context, Discourse, EncoderId, FeatValue, FeatureValue, Grade, NodeId, Packet, + PacketId, PacketKind, Predication, Provenance, ProvenanceSource, Referent, RoleFill, + SCHEMA_VERSION, Stance, Trace, +}; + +const CONCEPTS: &[&str] = &[ + "sage", + "mind", + "belly", + "will", + "bone", + "empty", + "fill", + "weaken", + "strengthen", + "heaven", + "earth", + "is_long_enduring", + "continues_long", + "highest_excellence", + "water", + "true_words", + "seem_paradoxical", + "space_between_heaven_and_earth", + "bellows", + "compare", + "resemble", +]; + +/// Frame glosses, from `frame:3`. 3 is minted from a Train chapter, 4 and 5 +/// from Validation chapters, 6 and 7 from Test chapters. +const FRAMES: &[&str] = &[ + "act", + "intransitive", + "likeness", + "seems_paradoxical", + "may_be_compared", +]; + +/// Revision 3 of the Dao De Jing stores, after the seed and the proof revision. +pub fn revision() -> (InventoryRevision, Vec) { + let mut entries = Vec::new(); + for (family, glosses) in [(Family::Concept, CONCEPTS), (Family::Frame, FRAMES)] { + let first = if family == Family::Concept { 6 } else { 3 }; + for (i, gloss) in glosses.iter().enumerate() { + entries.push(InventoryEntry { + id: family.id(NonZeroU32::new(first + i as u32).expect("nonzero")), + gloss: (*gloss).to_string(), + value_kind: None, + applies_to: vec![], + introduced_rev: 3, + retired_rev: None, + }); + } + } + let changes = entries.iter().map(|e| Change::Add(e.id)).collect(); + ( + InventoryRevision { + n: 3, + parent: Some(2), + changes, + frozen: false, + }, + entries, + ) +} + +fn concept(gloss: &str) -> &'static str { + let at = CONCEPTS + .iter() + .position(|c| *c == gloss) + .expect("known concept"); + Box::leak(format!("concept:{}", 6 + at).into_boxed_str()) +} + +fn frame(gloss: &str) -> &'static str { + let at = FRAMES + .iter() + .position(|c| *c == gloss) + .expect("known frame"); + Box::leak(format!("frame:{}", 3 + at).into_boxed_str()) +} + +fn hex8(ascii: &str) -> String { + assert_eq!(ascii.len(), 8); + ascii.bytes().map(|b| format!("{b:02x}")).collect() +} + +struct Spec { + /// Eight ASCII characters; the packet id is their hex. + id: &'static str, + item: &'static str, + speech_act: &'static str, + /// `(node, referent kind id, concept gloss)`. + referents: &'static [(&'static str, &'static str, &'static str)], + /// `(node, concept gloss, frame gloss, [(role id, filler node)])`. + predications: &'static [( + &'static str, + &'static str, + &'static str, + &'static [(&'static str, &'static str)], + )], + /// The author's reading of the speaker's felt state: `(feature gloss, permille)`. + stance: &'static [(&'static str, u16)], + note: &'static str, +} + +const AGENT: &str = "role:1"; +const PATIENT: &str = "role:2"; +const EXPERIENCER: &str = "role:3"; +const GOAL: &str = "role:6"; +const PERSON: &str = "referent_kind:1"; +const ORGANISM: &str = "referent_kind:3"; +const PLACE: &str = "referent_kind:4"; +const ABSTRACT: &str = "referent_kind:11"; + +/// Feature ids of the seed: restraint is the tenth stance feature. +fn stance_feature(gloss: &str) -> InvId { + let n = [ + "anger", + "fear", + "sadness", + "joy", + "disgust", + "surprise", + "contempt", + "affection", + "shame", + "restraint", + ] + .iter() + .position(|g| *g == gloss) + .expect("stance feature"); + // Features before the stance block: polarity, tense, aspect, modality, + // evidentiality, hedge, certainty, irony. + InvId::parse(&format!("feature:{}", 9 + n)).expect("feature id") +} + +const SPECS: &[Spec] = &[ + // Chapter 3 section 2. Train. + Spec { + id: "dao-0302", + item: "item:daodejing:1", + speech_act: "speech_act:1", + referents: &[ + ("n1", PERSON, "sage"), + ("n2", ABSTRACT, "mind"), + ("n3", ORGANISM, "belly"), + ("n4", ABSTRACT, "will"), + ("n5", ORGANISM, "bone"), + ], + predications: &[ + ("n6", "empty", "act", &[(AGENT, "n1"), (PATIENT, "n2")]), + ("n7", "fill", "act", &[(AGENT, "n1"), (PATIENT, "n3")]), + ("n8", "weaken", "act", &[(AGENT, "n1"), (PATIENT, "n4")]), + ("n9", "strengthen", "act", &[(AGENT, "n1"), (PATIENT, "n5")]), + ], + stance: &[("restraint", 800)], + note: "Four acts of one agent; 'in the exercise of his government' and 'their' are not carried.", + }, + // Chapter 7 section 1, first sentence. Validation. + Spec { + id: "dao-0701", + item: "item:daodejing:2", + speech_act: "speech_act:1", + referents: &[("n1", PLACE, "heaven"), ("n2", PLACE, "earth")], + predications: &[ + ( + "n3", + "is_long_enduring", + "intransitive", + &[(EXPERIENCER, "n1")], + ), + ( + "n4", + "continues_long", + "intransitive", + &[(EXPERIENCER, "n2")], + ), + ], + stance: &[("restraint", 800)], + note: "Heaven and earth as places; whether they are entities is a ruling the ontology owes.", + }, + // Chapter 8 section 1, first sentence. Validation. + Spec { + id: "dao-0801", + item: "item:daodejing:3", + speech_act: "speech_act:1", + referents: &[ + ("n1", ABSTRACT, "highest_excellence"), + ("n2", ABSTRACT, "water"), + ], + predications: &[( + "n3", + "resemble", + "likeness", + &[(EXPERIENCER, "n1"), (GOAL, "n2")], + )], + stance: &[("restraint", 800), ("affection", 300)], + note: "The metaphor is a plain likeness predication; the tenor and vehicle are not marked.", + }, + // Chapter 78 section 4. Test. + Spec { + id: "dao-7804", + item: "item:daodejing:4", + speech_act: "speech_act:1", + referents: &[("n1", ABSTRACT, "true_words")], + predications: &[( + "n2", + "seem_paradoxical", + "seems_paradoxical", + &[(EXPERIENCER, "n1")], + )], + stance: &[("restraint", 800)], + note: "The whole paradox is in the referent's concept and the frame; the packet holds no more than the surface does.", + }, + // Chapter 5 section 2, first sentence. Test. + Spec { + id: "dao-0502", + item: "item:daodejing:5", + speech_act: "speech_act:2", + referents: &[ + ("n1", PLACE, "space_between_heaven_and_earth"), + ("n2", ARTIFACT, "bellows"), + ], + predications: &[( + "n3", + "compare", + "may_be_compared", + &[(EXPERIENCER, "n1"), (GOAL, "n2")], + )], + stance: &[("restraint", 800)], + note: "A rhetorical question; the speech act is ask and the negated 'may not' is dropped.", + }, +]; + +const ARTIFACT: &str = "referent_kind:5"; + +/// The packets of the slice, each minted for its corpus item. +pub fn packets() -> Vec { + SPECS.iter().map(build).collect() +} + +fn build(spec: &Spec) -> Packet { + let id = |s: &str| InvId::parse(s).expect("ids are well formed"); + let node = |s: &str| NodeId::parse(s).expect("nodes are well formed"); + Packet { + id: PacketId::parse(&format!("packet:{}", hex8(spec.id))).expect("packet id"), + schema_version: SCHEMA_VERSION, + inventory_rev: 3, + kind: PacketKind::CorpusExample, + provenance: Provenance { + source: ProvenanceSource::Authored, + corpus_item: CorpusItemId::parse(spec.item).expect("item key"), + encoder: EncoderId::parse(&format!("encoder:{}", hex8("claude-s"))).expect("encoder"), + confidence: Grade::new(600).expect("grade"), + }, + context: Context::default(), + discourse: Discourse { + speech_act: Some(id(spec.speech_act)), + addressee: None, + features: vec![], + }, + referents: spec + .referents + .iter() + .map(|(n, kind, gloss)| Referent { + id: node(n), + kind: id(kind), + concept: id(concept(gloss)), + entity: None, + features: vec![], + }) + .collect(), + predications: spec + .predications + .iter() + .map(|(n, c, f, roles)| Predication { + id: node(n), + concept: id(concept(c)), + frame: id(frame(f)), + roles: roles + .iter() + .map(|(role, filler)| RoleFill { + role: id(role), + filler: node(filler), + }) + .collect(), + features: vec![], + }) + .collect(), + relations: vec![], + constraints: Constraints::default(), + extensions: vec![], + trace: Trace { + notes: vec![spec.note.to_string()], + rejected_readings: vec![], + }, + stance: Stance { + features: spec + .stance + .iter() + .map(|(g, v)| FeatureValue { + feature: stance_feature(g), + value: FeatValue::Grade(Grade::new(*v).expect("grade")), + }) + .collect(), + }, + } +} diff --git a/crates/weksa/src/lib.rs b/crates/weksa/src/lib.rs index 0bfdb8c..0fdb1b0 100644 --- a/crates/weksa/src/lib.rs +++ b/crates/weksa/src/lib.rs @@ -4,6 +4,7 @@ //! (chrF, meaning cosine, leak diagnostics). pub mod corpus; +pub mod daodejing; pub mod entity; pub mod form; pub mod inventory; diff --git a/crates/weksa/src/lower.rs b/crates/weksa/src/lower.rs index d6fcdfb..1084ca8 100644 --- a/crates/weksa/src/lower.rs +++ b/crates/weksa/src/lower.rs @@ -26,6 +26,33 @@ const CONCEPTS: &[Row] = &[ ("concept:3", "player", "player"), ("concept:4", "control", "control"), ("concept:5", "ship_mind", "ship mind"), + // The Dao De Jing test bed (`daodejing.rs`). Nouns, then the verbs a + // `{verb}` frame speaks. + ("concept:6", "sage", "sage"), + ("concept:7", "mind", "minds"), + ("concept:8", "belly", "bellies"), + ("concept:9", "will", "wills"), + ("concept:10", "bone", "bones"), + ("concept:11", "empty", "empties"), + ("concept:12", "fill", "fills"), + ("concept:13", "weaken", "weakens"), + ("concept:14", "strengthen", "strengthens"), + ("concept:15", "heaven", "heaven"), + ("concept:16", "earth", "earth"), + ("concept:17", "is_long_enduring", "is long-enduring"), + ("concept:18", "continues_long", "continues long"), + ("concept:19", "highest_excellence", "highest excellence"), + ("concept:20", "water", "water"), + ("concept:21", "true_words", "words that are strictly true"), + ("concept:22", "seem_paradoxical", "seem paradoxical"), + ( + "concept:23", + "space_between_heaven_and_earth", + "space between heaven and earth", + ), + ("concept:24", "bellows", "bellows"), + ("concept:25", "compare", "compare"), + ("concept:26", "resemble", "resemble"), ]; /// Frames: the clause a predication of that frame says. `{gloss}` is the filler @@ -41,11 +68,27 @@ const FRAMES: &[Row] = &[ "intervention", "{agent} will prevent {patient} to protect {beneficiary}", ), + // The Dao De Jing test bed. Frame 3 was written from a Train chapter, 4 and + // 5 from Validation chapters, 6 and 7 from Test chapters. + ("frame:3", "act", "{agent} {verb} {patient}"), + ("frame:4", "intransitive", "{experiencer} {verb}"), + ("frame:5", "likeness", "{experiencer} is like {goal}"), + ( + "frame:6", + "seems_paradoxical", + "{experiencer} seem paradoxical", + ), + ( + "frame:7", + "may_be_compared", + "{experiencer} may be compared to {goal}", + ), ]; /// Speech acts: the opener of the first sentence. const SPEECH_ACTS: &[Row] = &[ ("speech_act:1", "assert", ""), + ("speech_act:2", "ask", ""), ("speech_act:5", "warn", "Warning: "), ]; @@ -156,6 +199,10 @@ fn clause(input: &LoweringInput, inv: &Inventory, p: &Predication) -> Result, #[arg(long)] edition_url: Option, + /// The edition's licence, for `public-domain`. + #[arg(long)] + licence: Option, + /// Where the public-domain status is asserted, for `public-domain`. + #[arg(long)] + jurisdiction: Option, + #[arg(long)] + translator: Option, + #[arg(long)] + edition_year: Option, #[arg(long = "model-family", value_enum)] family: Option, /// The generator string the fixture or run wrote, verbatim. @@ -265,7 +279,15 @@ impl AddArgs { SourceKind::CastAuthored => &["persona", "fixture", "author"], SourceKind::OperatorWritten => &[], SourceKind::OperatorApproved => &["note"], - SourceKind::PublicDomain => &["work", "author", "edition-url"], + SourceKind::PublicDomain => &[ + "work", + "author", + "edition-url", + "licence", + "jurisdiction", + "translator", + "edition-year", + ], SourceKind::ModelWritten => &["model-family", "generator", "prompt-digest"], }; let given = [ @@ -275,6 +297,10 @@ impl AddArgs { ("note", self.note.is_some()), ("work", self.work.is_some()), ("edition-url", self.edition_url.is_some()), + ("licence", self.licence.is_some()), + ("jurisdiction", self.jurisdiction.is_some()), + ("translator", self.translator.is_some()), + ("edition-year", self.edition_year.is_some()), ("model-family", self.family.is_some()), ("generator", self.generator.is_some()), ("prompt-digest", self.prompt_digest.is_some()), @@ -301,6 +327,12 @@ impl AddArgs { work: get(&self.work, "work")?, author: get(&self.author, "author")?, edition_url: get(&self.edition_url, "edition-url")?, + licence: get(&self.licence, "licence")?, + jurisdiction: get(&self.jurisdiction, "jurisdiction")?, + translator: get(&self.translator, "translator")?, + edition_year: self + .edition_year + .context("this source needs --edition-year")?, }, SourceKind::ModelWritten => Source::ModelWritten { family: self @@ -472,6 +504,27 @@ fn main() -> Result<()> { .map_err(refused)?; println!("admitted rev:2 with {count} entries"); } + Command::Inventory(InventoryCommand::Dao) => { + let (rev, entries) = weksa::daodejing::revision(); + let count = entries.len(); + InventoryStore::open(&cli.inventory)? + .admit_revision(rev, entries) + .map_err(refused)?; + println!("admitted rev:3 with {count} entries"); + } + Command::Packet(PacketCommand::Dao { store }) => { + let mut packets = PacketStore::open(&store)?; + let inventory = InventoryStore::open(&cli.inventory)?; + let corpus = CorpusStore::open(&cli.corpus)?; + let entities = EntityStore::open(&cli.entities)?; + for packet in weksa::daodejing::packets() { + let key = packet.id.to_string(); + packets + .admit(packet, &inventory, &corpus, &entities) + .map_err(refused)?; + println!("admitted {key}"); + } + } Command::Packet(PacketCommand::Proof { store }) => { let packet = proof_packet(); let key = packet.id.to_string(); diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index 1caa6ac..6032db6 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -402,6 +402,18 @@ fn empty_attestation(source: &Source) -> Option<&'static str> { Source::PublicDomain { edition_url, .. } if empty(edition_url) => { Some("a public-domain source names its edition url") } + Source::PublicDomain { licence, .. } if empty(licence) => { + Some("a public-domain source names its licence") + } + Source::PublicDomain { jurisdiction, .. } if empty(jurisdiction) => { + Some("a public-domain source names its jurisdiction") + } + Source::PublicDomain { translator, .. } if empty(translator) => { + Some("a public-domain source names its translator") + } + Source::PublicDomain { + edition_year: 0, .. + } => Some("a public-domain source names its edition year"), Source::ModelWritten { generator, .. } if empty_some(generator) => { Some("a model-written source names a non-empty generator") } diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index e05b24a..c9be4fb 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -210,6 +210,14 @@ fn model_written_and_public_domain_sources_name_their_provenance() { "a", "--edition-url", "https://example.org/w", + "--licence", + "Public domain", + "--jurisdiction", + "US", + "--translator", + "T. Translator", + "--edition-year", + "1891", ]); assert!(pd.status.success(), "{}", out(&pd)); let bad_turn = run(&[ @@ -566,6 +574,14 @@ fn source_flags_must_match_the_source() { "a", "--edition-url", "u", + "--licence", + "l", + "--jurisdiction", + "j", + "--translator", + "t", + "--edition-year", + "1891", "--note", "n", ], @@ -618,6 +634,78 @@ fn source_flags_must_match_the_source() { &["--work", "w", "--edition-url", "u"], "--author", ), + ( + "public-domain", + &[ + "--work", + "w", + "--author", + "a", + "--edition-url", + "u", + "--jurisdiction", + "j", + "--translator", + "t", + "--edition-year", + "1891", + ], + "--licence", + ), + ( + "public-domain", + &[ + "--work", + "w", + "--author", + "a", + "--edition-url", + "u", + "--licence", + "l", + "--translator", + "t", + "--edition-year", + "1891", + ], + "--jurisdiction", + ), + ( + "public-domain", + &[ + "--work", + "w", + "--author", + "a", + "--edition-url", + "u", + "--licence", + "l", + "--jurisdiction", + "j", + "--edition-year", + "1891", + ], + "--translator", + ), + ( + "public-domain", + &[ + "--work", + "w", + "--author", + "a", + "--edition-url", + "u", + "--licence", + "l", + "--jurisdiction", + "j", + "--translator", + "t", + ], + "--edition-year", + ), // Empty provenance strings. ( "model-written", diff --git a/crates/weksa/tests/daodejing.rs b/crates/weksa/tests/daodejing.rs new file mode 100644 index 0000000..8138ebd --- /dev/null +++ b/crates/weksa/tests/daodejing.rs @@ -0,0 +1,71 @@ +//! The Dao De Jing test bed (`data/daodejing`): its Legge items carry full +//! public-domain provenance, and its packets are admitted and lower to the +//! English the first golf-loop run scored (`docs/targets/dao-de-jing-test-bed.md`). + +use std::path::Path; + +use weksa::daodejing::packets; +use weksa::{CorpusStore, EntityStore, InventoryStore, PacketStore, Source, lower}; + +fn root() -> std::path::PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/daodejing") +} + +#[test] +fn every_legge_item_names_licence_jurisdiction_translator_and_edition_year() { + let items = CorpusStore::open(&root().join("corpus.cc")) + .unwrap() + .all() + .unwrap(); + assert_eq!(items.len(), 5); + for item in items { + let Source::PublicDomain { + licence, + jurisdiction, + translator, + edition_year, + .. + } = item.source + else { + panic!("{} is not a public-domain item", item.id); + }; + assert!( + !licence.is_empty() && !jurisdiction.is_empty(), + "{}", + item.id + ); + assert_eq!((translator.as_str(), edition_year), ("James Legge", 1891)); + } +} + +#[test] +fn the_slice_packets_are_admitted_and_lower_to_the_scored_english() { + let inventory = InventoryStore::open(&root().join("inventory.cc")).unwrap(); + let corpus = CorpusStore::open(&root().join("corpus.cc")).unwrap(); + let entities = EntityStore::open(&root().join("entity.cc")).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let mut store = PacketStore::open(&dir.path().join("p.cc")).unwrap(); + let inv = inventory.inventory().unwrap(); + let lowered: Vec = packets() + .into_iter() + .map(|p| { + let view = store.admit(p, &inventory, &corpus, &entities).unwrap(); + lower(&view, &inv).unwrap() + }) + .collect(); + assert_eq!( + lowered, + [ + "The sage empties the minds. The sage fills the bellies. The sage weakens the wills. The sage strengthens the bones.", + "The heaven is long-enduring. The earth continues long.", + "The highest excellence is like the water.", + "The words that are strictly true seem paradoxical.", + "The space between heaven and earth may be compared to the bellows.", + ] + ); + // The committed packet store holds exactly what the module builds. + let stored = PacketStore::open(&root().join("packets.cc")).unwrap(); + for p in packets() { + assert_eq!(stored.get(&p.id.to_string()).unwrap(), Some(p)); + } +} diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs index 07cbe0c..2a9889c 100644 --- a/crates/weksa/tests/stores.rs +++ b/crates/weksa/tests/stores.rs @@ -371,7 +371,7 @@ fn an_attestation_holds_something_readable() { let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); let s = |x: &str| x.to_string(); type Build = fn(String) -> Source; - let builders: [(Build, &'static str); 9] = [ + let builders: [(Build, &'static str); 12] = [ ( |v| Source::CastAuthored { persona: v, @@ -405,6 +405,10 @@ fn an_attestation_holds_something_readable() { work: v, author: "a".into(), edition_url: "u".into(), + licence: "l".into(), + jurisdiction: "j".into(), + translator: "t".into(), + edition_year: 1891, }, "a public-domain source names its work", ), @@ -413,6 +417,10 @@ fn an_attestation_holds_something_readable() { work: "w".into(), author: v, edition_url: "u".into(), + licence: "l".into(), + jurisdiction: "j".into(), + translator: "t".into(), + edition_year: 1891, }, "a public-domain source names its author", ), @@ -421,9 +429,49 @@ fn an_attestation_holds_something_readable() { work: "w".into(), author: "a".into(), edition_url: v, + licence: "l".into(), + jurisdiction: "j".into(), + translator: "t".into(), + edition_year: 1891, }, "a public-domain source names its edition url", ), + ( + |v| Source::PublicDomain { + work: "w".into(), + author: "a".into(), + edition_url: "u".into(), + licence: v, + jurisdiction: "j".into(), + translator: "t".into(), + edition_year: 1891, + }, + "a public-domain source names its licence", + ), + ( + |v| Source::PublicDomain { + work: "w".into(), + author: "a".into(), + edition_url: "u".into(), + licence: "l".into(), + jurisdiction: v, + translator: "t".into(), + edition_year: 1891, + }, + "a public-domain source names its jurisdiction", + ), + ( + |v| Source::PublicDomain { + work: "w".into(), + author: "a".into(), + edition_url: "u".into(), + licence: "l".into(), + jurisdiction: "j".into(), + translator: v, + edition_year: 1891, + }, + "a public-domain source names its translator", + ), ( |v| Source::ModelWritten { family: ModelFamily::Unrecorded, @@ -528,6 +576,10 @@ fn an_item_source_attests_with_non_empty_strings() { work: s(work), author: s(author), edition_url: s(url), + licence: s("l"), + jurisdiction: s("j"), + translator: s("t"), + edition_year: 1891, }; let cases: [(Source, &'static str); 9] = [ ( @@ -1292,3 +1344,26 @@ fn seed_reseeded() { replay.add(i, &entities).unwrap(); } } + +#[test] +fn a_public_domain_source_without_an_edition_year_is_refused() { + let d = dir(); + let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + let key = "item:pd:1"; + let source = |edition_year| Source::PublicDomain { + work: "w".into(), + author: "a".into(), + edition_url: "u".into(), + licence: "l".into(), + jurisdiction: "j".into(), + translator: "t".into(), + edition_year, + }; + assert_eq!( + refusal(store.add(item_in(key, "line", "scene-1", source(0)), &ents(&d))), + malformed(key, "a public-domain source names its edition year") + ); + store + .add(item_in(key, "line", "scene-1", source(1891)), &ents(&d)) + .unwrap(); +} diff --git a/data/daodejing/corpus.cc b/data/daodejing/corpus.cc new file mode 100644 index 0000000..45bcda5 --- /dev/null +++ b/data/daodejing/corpus.cc @@ -0,0 +1 @@ +“²cultcache.store.v1‘—±weksa.corpus_item±weksa.corpus_item´weksa.corpus_item.v1±weksa.corpus_itemÙV{"schemaName":"weksa.corpus_item","schemaVersion":"weksa.corpus_item.v1","members":[]}‘±weksa.corpus_item•”°item:daodejing:1±weksa.corpus_item´2026-11-07T02:09:00ZÅ4›°item:daodejing:1Ù’Therefore the sage, in the exercise of his government, empties their minds, fills their bellies, weakens their wills, and strengthens their bones.¥en-US¬entity:laozi³daodejing-legge-ch3¬PublicDomain—ÙBThe Tao Teh King, or the Tao and its Characteristics (Dao De Jing)¥LaoziÙ$https://www.gutenberg.org/ebooks/216ÙPublic domain (author d. c. 5th century BCE; translator d. 1897); Project Gutenberg eBook 216 text used without its trademark or licence matterÙ7United States and life+70 jurisdictions (Legge d. 1897)«James LeggeÍcª2026-11-07À”°item:daodejing:2±weksa.corpus_item´2026-11-07T02:09:00ZÅÓ›°item:daodejing:2Ù1Heaven is long-enduring and earth continues long.¥en-US¬entity:laozi³daodejing-legge-ch7¬PublicDomain—ÙBThe Tao Teh King, or the Tao and its Characteristics (Dao De Jing)¥LaoziÙ$https://www.gutenberg.org/ebooks/216ÙPublic domain (author d. c. 5th century BCE; translator d. 1897); Project Gutenberg eBook 216 text used without its trademark or licence matterÙ7United States and life+70 jurisdictions (Legge d. 1897)«James LeggeÍcª2026-11-07À”°item:daodejing:3±weksa.corpus_item´2026-11-07T02:09:00ZÅÑ›°item:daodejing:3Ù/The highest excellence is like (that of) water.¥en-US¬entity:laozi³daodejing-legge-ch8¬PublicDomain—ÙBThe Tao Teh King, or the Tao and its Characteristics (Dao De Jing)¥LaoziÙ$https://www.gutenberg.org/ebooks/216ÙPublic domain (author d. c. 5th century BCE; translator d. 1897); Project Gutenberg eBook 216 text used without its trademark or licence matterÙ7United States and life+70 jurisdictions (Legge d. 1897)«James LeggeÍcª2026-11-07À”°item:daodejing:4±weksa.corpus_item´2026-11-07T02:09:00ZÅ×›°item:daodejing:4Ù4Words that are strictly true seem to be paradoxical.¥en-US¬entity:laozi´daodejing-legge-ch78¬PublicDomain—ÙBThe Tao Teh King, or the Tao and its Characteristics (Dao De Jing)¥LaoziÙ$https://www.gutenberg.org/ebooks/216ÙPublic domain (author d. c. 5th century BCE; translator d. 1897); Project Gutenberg eBook 216 text used without its trademark or licence matterÙ7United States and life+70 jurisdictions (Legge d. 1897)«James LeggeÍcª2026-11-07À”°item:daodejing:5±weksa.corpus_item´2026-11-07T02:09:00ZÅæ›°item:daodejing:5ÙDMay not the space between heaven and earth be compared to a bellows?¥en-US¬entity:laozi³daodejing-legge-ch5¬PublicDomain—ÙBThe Tao Teh King, or the Tao and its Characteristics (Dao De Jing)¥LaoziÙ$https://www.gutenberg.org/ebooks/216ÙPublic domain (author d. c. 5th century BCE; translator d. 1897); Project Gutenberg eBook 216 text used without its trademark or licence matterÙ7United States and life+70 jurisdictions (Legge d. 1897)«James LeggeÍcª2026-11-07À \ No newline at end of file diff --git a/data/daodejing/entity.cc b/data/daodejing/entity.cc new file mode 100644 index 0000000..c11c6df --- /dev/null +++ b/data/daodejing/entity.cc @@ -0,0 +1 @@ +“²cultcache.store.v1‘—¬weksa.entity¬weksa.entity¯weksa.entity.v1¬weksa.entityÙL{"schemaName":"weksa.entity","schemaVersion":"weksa.entity.v1","members":[]}‘¬weksa.entity‘”¬entity:laozi¬weksa.entity´2026-11-07T02:09:00ZÄb“¬entity:laozi‘’¥en-US‘¥Laozi¬PublicDomain‘Ù4The Tao Teh King, or the Tao and its Characteristics \ No newline at end of file diff --git a/data/daodejing/inventory.cc b/data/daodejing/inventory.cc new file mode 100644 index 0000000000000000000000000000000000000000..0d2377ff42d9d08028c301dc25757a2c0443d6ed GIT binary patch literal 10361 zcmbVSTaOz_70%@sENo_y*@0|^%M+Wm+t(QhAtc1U0n#EKKw!$ z+!1}qgR#{a(HPUaj-XnsZh=J*(Az z{l(p-9=D~TN5UaxuO8oewYXGxwQBcP1KwF016Tbhq^};|HqlV@4h*zS~;m|_vVvz3}$lXsMmt#MiQenWtJ;mUD z$G|=PY~&9|l*c{#$fZF*p#U>%Ra25i6&b%t)M8wpoh~ zsuqp(bn2Heo6_DaVjj0JQByYJ?>H|xrQ_a&Fycjze3w|o>7a`9eH|yK@gF`RF75Sc zJfk%1i8APs(CbxXu!AbvyM~?SrVm*V&1{#TD%J#SZQN)K4zYTugDTm32E+E2Q4T9- zVC5;wwFf#+8|z6x=5obQS&wp{k@ZcOE^Im?2s|u63J&GIdzxHp$4CC8H}*rX2#9ju z+ZwRF;$&!f4%O}qtjXc3JlfN3}UCNS-vjr29(58MsZt; zA1(E`?Ia2a^W*u)o1af&*m&`$pyGb)U(jtxha~;+>t84LYi#zi)7F@5o?4-8A#Afkzej zi2>h9_y~?X*Fd9c-elh0K`W0L@x!uXQ~{zg@dI7BP7>3QQ&qK#)r9ak-IQ?Hl&lBS zv@L97e@p`?1-pQic~rjsUYl>M37NV6PAJRbbU|PEV=Y`wSTqDyJZ&f=()Z&j z2{!WiUS6nFcQvo(b{u&mF#FkZP2AMxZW4jZVxI(QgX2CdvAdAEz>Ki@#-rSa9=D77 zfe%fOdV8s%>f}RBt@SrJ+IoV4$#1dtz_J@}{cu1TaDW}aBw|2eLpYy~uc3k}s+y+K zPDa!l%A|j~`-#upXv#RX+PumVRCP7AozBjsEKUnR_RKf1g(|oAwXJUAvnZUeej1SB z8<;^Yz7MpS(C7+=h#8Hmuya1HhD|I~ zS=`UD+J8?^H@tMbkrPJMz_)VH&ZlEChht4o9ur`n3;W4W8HG}Z*!0d}>x)eqd-NKb zDNln8`=^%{GHQQzyeth*YUxKc0Nk3g3)3gF~Yf`AJe`jfrPKGR+PO5-W z>HlpTxUr4ckc2)TS7D=SqC~oxz)3B=o+x_*RU9SKtu0&v`GY;4Mit8g+ac{MKjbm2 z>7vb)sUyqA3S?(P-6)8LaB?a`qgINNySoXbNL=Y|)Y@=tyX15YH{WnQpHaSKG{{a( z*djLuUb&I1YMC8Z8WdIEM@w_$7WRti`T(sHd}};G750INSzkr7u!M)IE&bcUe<1h-!*{ooQ5T1zh)Sb(Sq9T|X)o%m{0hD5~(EV#d3YEHt60CWdF02fWe}x&MQudn?tF`8EypxNE@DwUp-;y}()i6r$ z{FT2^DSAhubhcSRX2%kRgv!p`;oYk~?EMG6>%$2?qxAwVC8#^=d{{SD%$}CX{R|IXs4N^3Qi^399ROTHPn6_5w zxweFe%E7w@#5Qd3DJ{<&m5@cqP7;quSdI`XGmi@_yE_86>(MZGX963pv;F_1Q2Dw3 z%iI55+=A&gX6)lTdXjx1^%YwVHJ+5W64TWH6C|!Vw#1fmm!-PF%MHCESef&*VG8>rbciuHG+w$ z5q#8*qwuK`1rsF-8cGx#lqd)&QCL@^Fs($(u86krnJRONmSr}o&sZ`Wl`NR)_XRaA z3vOB#tzUu0dw+54|Wm0jbDj~2IXKFheP&yCWh`9BRwm*%2$X)a2a=Axi2 zZ^8tS6dXygfg||k&BfyOt=Bh>iX3IEaXTh_&F9o$KvN${2Ys)BlyX`. +- Source: Project Gutenberg eBook 216, "The Tao Teh King, or the Tao and its + Characteristics", Laozi, tr. James Legge (Sacred Books of the East 39, 1891). + Gutenberg's licence (1.E.2) lets a work derived from texts not protected by US + copyright be used with no fee provided every Project Gutenberg reference is + removed from the work. The items hold only Legge's words, one numbered section + or sentence each; no Gutenberg header, licence text or trademark is stored, + and the eBook file itself is not committed. `edition_url` names the page the + text was retrieved from, as a citation of provenance and not as part of any + item's text. The legal-root question of whether a citation counts as a + "reference" is not ruled; if Gutenberg says it does, the field moves to the + Sacred Books of the East print edition. +- `Source::PublicDomain` now carries `licence`, `jurisdiction`, `translator` and + `edition_year`, and the store refuses an item missing any of them (an empty + string, or year 0). + +## The slice + +Corpus groups are chapters, so the derived split spreads the slice: + +| item | Legge | split | +|---|---|---| +| `item:daodejing:1` | ch. 3 s. 2 | Train | +| `item:daodejing:2` | ch. 7 s. 1, first sentence | Validation | +| `item:daodejing:3` | ch. 8 s. 1, first sentence | Validation | +| `item:daodejing:4` | ch. 78 s. 4 | Test | +| `item:daodejing:5` | ch. 5 s. 2, first sentence | Test | + +Each packet states neutral facts (referents, predications, a speech act) and a +stance (restraint 800 on all; affection 300 on the water likeness). The stance is +my reading of the sage's voice, not data. + +## First closeness number + +Scored with `weksa score` (chrF, and meaning as cosine under `qwen3-embedding:0.6b` +through the local Ollama), one lowered output against its own item. + +| item | split | chrF | meaning | lowered | +|---|---|---|---|---| +| 1 | Train | 44.18 | 0.831 | The sage empties the minds. The sage fills the bellies. The sage weakens the wills. The sage strengthens the bones. | +| 2 | Validation | 82.12 | 0.976 | The heaven is long-enduring. The earth continues long. | +| 3 | Validation | 74.99 | 0.974 | The highest excellence is like the water. | +| 4 | Test | 82.50 | 0.949 | The words that are strictly true seem paradoxical. | +| 5 | Test | 75.99 | 0.894 | The space between heaven and earth may be compared to the bellows. | + +Mean over the five: chrF 71.96, meaning 0.925. Mismatched baseline (each output +against the other four references, 20 pairs): chrF 14.1, meaning 0.458. So the +scorer separates the right reference from a wrong one by a wide margin; that is +all it shows here. + +**Do not read 72 as a measure of the interlingua.** The five outputs are the +ceiling of an author who saw the reference: + +- One author wrote the packets, the inventory entries and the lowerer rows with + Legge's sentence open. Frame 3 (`act`) was written from the Train item. Frames + 4 and 5 came from Validation items and frames 6 and 7 from Test items, each + written while reading that item. The Train item, the only one lowered by a + frame minted from Train, scored lowest (chrF 44.18). The scores rise with + how bespoke the frame is, the reverse of generalisation. The Validation and + Test numbers measure the lowerer table, not what the interlingua can carry. +- n = 5, one translation as reference, one embedding model, no variance. + +## What the scorer cannot see + +- **Stance.** The stance features are in the packets and the lowerer does not + render them, so the number is blind to the packet's voice entirely. It would + not change if every stance were deleted. +- **Dropped meaning.** Item 5 is a negated rhetorical question ("May not ... be + compared"); the lowering is a plain statement with no question mark and no + "not". Item 1 loses "Therefore", "in the exercise of his government" and "their". + Item 4 loses "to be". chrF scores the overlap and cannot say that the speech + act changed. The embedding cosine of 0.894 for item 5 is not much lower than + for the statement-for-statement items. +- **Table defects it forgives.** "The heaven" and "the water" are wrong English + (the lowerer prefixes "the" to every referent and has no proper-noun, + mass-noun or entity rendering). chrF takes a small penalty only. +- **Register and meaning of the Chinese.** The packets were read off Legge's + English, not the Chinese, so this is a round trip English to interlingua to + English. It measures how well the packet kept Legge's wording, not whether Legge + or the packet is right about the Dao. The term "Tao" never appears in the slice. + The multi-reading term modelling Meta described is untested. +- **Leak.** `weksa score` scores the whole output against the whole reference + with no mask, as the leak ruling says. Nothing here enforces that the + author did not use the reference; only the labels above say so. + +## What the run showed the interlingua needs + +- Negation and question force: `polarity` is refused by the lowerer ("features") + and `ask` has no punctuation row, so two of Laozi's habitual moves, the negated + claim and the rhetorical question, cannot be lowered at all. +- A frame per sentence shape is the lowerer's cost model today: each new shape + is one more table row and one more inventory frame. The Dao's aphorisms are + mostly copular and paradoxical, which suggests the next frames to mint + (attribution, contrast, negated identity) before more chapters are added. diff --git a/schemas/cultnet/weksa.corpus_item.schema.json b/schemas/cultnet/weksa.corpus_item.schema.json index 7b613be..c82c4da 100644 --- a/schemas/cultnet/weksa.corpus_item.schema.json +++ b/schemas/cultnet/weksa.corpus_item.schema.json @@ -170,6 +170,7 @@ ] }, { + "description": "A public-domain work, with everything a reader needs to check the claim:\nthe edition's licence and the jurisdiction the public-domain status is\nasserted in, who translated it (the translation is the text's copyright,\nnot the original's) and when the edition appeared. A work missing any\nis refused at admission.", "type": "object", "properties": { "PublicDomain": { @@ -181,6 +182,21 @@ "edition_url": { "type": "string" }, + "edition_year": { + "type": "integer", + "format": "uint16", + "maximum": 65535, + "minimum": 0 + }, + "jurisdiction": { + "type": "string" + }, + "licence": { + "type": "string" + }, + "translator": { + "type": "string" + }, "work": { "type": "string" } @@ -188,7 +204,11 @@ "required": [ "work", "author", - "edition_url" + "edition_url", + "licence", + "jurisdiction", + "translator", + "edition_year" ] } }, From e7a3406b8924ed473c09f7b8a6e3ad283255a253 Mon Sep 17 00:00:00 2001 From: Weksa Date: Sat, 7 Nov 2026 12:04:41 +0000 Subject: [PATCH 002/175] Entity forms carry a ruled pronunciation; lower --ipa emits it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LanguageForms gains an optional Pronunciation: bare stress-marked IPA, ruled_by, date, source and a rejected list. The store checks its shape and freezes a ruling once set. pronounce() and `weksa lower --ipa` print the stored IPA of each cited entity and refuse with NoPronunciation / NoEntityReferent; rejected variants are never read. Nibu is the first entry: ˈnibu, rejected ˈnɪbu. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/entity.rs | 31 +++++++++ crates/weksa/src/lib.rs | 4 +- crates/weksa/src/lower.rs | 47 +++++++++++--- crates/weksa/src/main.rs | 62 +++++++++++++++--- crates/weksa/src/store.rs | 59 ++++++++++++++++- crates/weksa/tests/cli.rs | 41 ++++++++++++ crates/weksa/tests/common/mod.rs | 1 + crates/weksa/tests/entities.rs | 75 ++++++++++++++++++++++ crates/weksa/tests/entity_door.rs | 4 ++ crates/weksa/tests/lower.rs | 80 +++++++++++++++++++++++- data/entity/entity.cc | 2 +- docs/english-pronunciation-lowering.md | 11 ++++ schemas/cultnet/weksa.entity.schema.json | 48 ++++++++++++++ 13 files changed, 444 insertions(+), 21 deletions(-) diff --git a/crates/weksa/src/entity.rs b/crates/weksa/src/entity.rs index 33d22e1..5862913 100644 --- a/crates/weksa/src/entity.rs +++ b/crates/weksa/src/entity.rs @@ -16,6 +16,29 @@ pub struct LanguageForms { /// 1 to 64 bytes each, no control or invisible format character, no edge /// whitespace, none repeated. pub forms: Vec, + /// How the citation form (`forms[0]`) is said, when someone with the + /// authority to rule on it has. Absent means unruled, never "spelled as + /// read". + #[serde(default)] + pub pronunciation: Option, +} + +/// A ruled pronunciation of an entity's citation form in one language. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct Pronunciation { + /// Broad IPA with stress marks, bare: no enclosing `/` or `[ ]`. 1 to 64 + /// bytes, NFC. + pub ipa: String, + /// Who ruled: the entity itself, a writer, an operator. + pub ruled_by: String, + /// `YYYY-MM-DD`. + pub date: String, + /// Where the ruling was made: a room, a document, a run. + pub source: String, + /// Pronunciations that were considered and refused, same shape as `ipa`. + /// Kept so nobody re-proposes one; never emitted. + #[serde(default)] + pub rejected: Vec, } /// Where an entity was minted. @@ -50,4 +73,12 @@ impl Entity { .find(|f| f.language == language) .map(|f| f.forms.as_slice()) } + + /// The ruled pronunciation of the citation form in `language`, if any. + pub fn pronunciation_in(&self, language: &str) -> Option<&Pronunciation> { + self.forms + .iter() + .find(|f| f.language == language) + .and_then(|f| f.pronunciation.as_ref()) + } } diff --git a/crates/weksa/src/lib.rs b/crates/weksa/src/lib.rs index 8595226..3f6b43c 100644 --- a/crates/weksa/src/lib.rs +++ b/crates/weksa/src/lib.rs @@ -16,13 +16,13 @@ pub mod store; pub mod validate; pub use corpus::{CorpusItem, CorpusItemId, NameSpan, Source, Turn}; -pub use entity::{Entity, EntityOrigin, LanguageForms}; +pub use entity::{Entity, EntityOrigin, LanguageForms, Pronunciation}; pub use form::FormError; pub use inventory::{ Change, Family, InvId, Inventory, InventoryEntry, InventoryRevision, OptionLabel, OptionSet, Slot, ValueKind, }; -pub use lower::{EntityForms, LowerError, TARGET_LANGUAGE, lower}; +pub use lower::{EntityForms, LowerError, TARGET_LANGUAGE, lower, pronounce}; pub use packet::{ EncoderId, EntityId, Grade, NodeId, Packet, PacketId, PacketKind, SCHEMA_VERSION, Stance, }; diff --git a/crates/weksa/src/lower.rs b/crates/weksa/src/lower.rs index 93f0074..43a8752 100644 --- a/crates/weksa/src/lower.rs +++ b/crates/weksa/src/lower.rs @@ -90,6 +90,11 @@ pub enum LowerError { UnknownEntity { entity: EntityId }, /// The entity has no form in [`TARGET_LANGUAGE`]. NoForm { entity: EntityId }, + /// IPA was requested for an entity whose [`TARGET_LANGUAGE`] citation form + /// has no ruled pronunciation. Spelling is not a pronunciation. + NoPronunciation { entity: EntityId }, + /// IPA was requested for a packet that cites no entity. + NoEntityReferent, /// The entity port failed to read. EntityRead { entity: EntityId, why: String }, } @@ -180,6 +185,29 @@ pub fn lower( Ok(out) } +/// The ruled IPA of every entity a packet cites, one per entity referent in +/// the packet's referent order, as stored: bare, stress-marked, nothing +/// derived. A rejected pronunciation is never read. An entity the port does not +/// hold, or one with no ruled pronunciation in [`TARGET_LANGUAGE`], is refused: +/// the lowerer has no grapheme-to-phoneme guess to offer in its place. +pub fn pronounce( + input: &LoweringInput, + entities: &dyn EntityForms, +) -> Result, LowerError> { + let mut out = Vec::new(); + for id in input.referents().iter().filter_map(|r| r.entity.as_ref()) { + let entity = entity(entities, id)?; + let p = entity + .pronunciation_in(TARGET_LANGUAGE) + .ok_or_else(|| LowerError::NoPronunciation { entity: id.clone() })?; + out.push(p.ipa.clone()); + } + if out.is_empty() { + return Err(LowerError::NoEntityReferent); + } + Ok(out) +} + fn clause( input: &LoweringInput, inv: &Inventory, @@ -254,16 +282,19 @@ fn filler( /// The entity's citation form in [`TARGET_LANGUAGE`]. fn name(entities: &dyn EntityForms, id: &EntityId) -> Result { - let entity = entities - .entity(id) - .map_err(|e| LowerError::EntityRead { - entity: id.clone(), - why: e.to_string(), - })? - .ok_or_else(|| LowerError::UnknownEntity { entity: id.clone() })?; - entity + entity(entities, id)? .forms_in(TARGET_LANGUAGE) .and_then(<[String]>::first) .cloned() .ok_or_else(|| LowerError::NoForm { entity: id.clone() }) } + +fn entity(entities: &dyn EntityForms, id: &EntityId) -> Result { + entities + .entity(id) + .map_err(|e| LowerError::EntityRead { + entity: id.clone(), + why: e.to_string(), + })? + .ok_or_else(|| LowerError::UnknownEntity { entity: id.clone() }) +} diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 0b52bb3..5518823 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -7,13 +7,13 @@ use anyhow::{Context, Result, bail}; use clap::{Args, Parser, Subcommand, ValueEnum}; use unicode_normalization::UnicodeNormalization; use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn}; -use weksa::entity::{Entity, EntityOrigin, LanguageForms}; +use weksa::entity::{Entity, EntityOrigin, LanguageForms, Pronunciation}; use weksa::inventory::{proof_revision, seed}; use weksa::packet::proof_packet; use weksa::score::ScoreInput; use weksa::{ CorpusStore, EntityId, EntityStore, InventoryStore, OllamaEmbedder, PacketStore, Scorer, - SplitScheme, StoreError, lower, validate, + SplitScheme, StoreError, lower, pronounce, validate, }; #[derive(Parser)] @@ -48,7 +48,14 @@ enum Command { #[command(subcommand)] Packet(PacketCommand), /// Lower a stored packet to English text. - Lower { store: PathBuf, key: String }, + Lower { + store: PathBuf, + key: String, + /// Print the ruled IPA of each entity the packet cites, one per line, + /// instead of the text; refused for an entity with no ruled pronunciation. + #[arg(long)] + ipa: bool, + }, /// Score output text (stdin, or --output) against corpus items, one embed call. Score(ScoreArgs), /// Print the split of a corpus item. @@ -127,6 +134,22 @@ struct PutArgs { /// The work of a `public-domain` origin; refused with any other origin. #[arg(long)] work: Option, + /// `language=ipa`: how that language's citation form is said, bare IPA with + /// stress marks. Needs `--ruled-by`, `--ruled-on` and `--ruling-source`. + #[arg(long, requires_all = ["ruled_by", "ruled_on", "ruling_source"])] + pronunciation: Option, + /// Who ruled the pronunciation. + #[arg(long)] + ruled_by: Option, + /// `YYYY-MM-DD`. + #[arg(long)] + ruled_on: Option, + /// Where it was ruled: a room, a document, a run. + #[arg(long)] + ruling_source: Option, + /// A refused pronunciation, bare IPA, repeated. + #[arg(long = "rejected", requires = "pronunciation")] + rejected: Vec, } #[derive(Clone, Copy, ValueEnum)] @@ -146,9 +169,28 @@ impl PutArgs { None => forms.push(LanguageForms { language: language.to_string(), forms: vec![form.nfc().collect()], + pronunciation: None, }), } } + if let Some(p) = &self.pronunciation { + let (language, ipa) = p + .split_once('=') + .context("--pronunciation is not language=ipa")?; + let lf = forms + .iter_mut() + .find(|lf| lf.language == language) + .with_context(|| { + format!("--pronunciation names {language}, which has no --form") + })?; + lf.pronunciation = Some(Pronunciation { + ipa: ipa.nfc().collect(), + ruled_by: need(self.ruled_by.clone(), "ruled-by")?, + date: need(self.ruled_on.clone(), "ruled-on")?, + source: need(self.ruling_source.clone(), "ruling-source")?, + rejected: self.rejected.iter().map(|r| r.nfc().collect()).collect(), + }); + } let origin = match (self.origin, self.persona, self.fixture, self.work) { (OriginKind::Cast, persona, fixture, None) => EntityOrigin::Cast { persona: need(persona, "persona")?, @@ -485,7 +527,7 @@ fn main() -> Result<()> { .map_err(refused)?; println!("admitted {key}"); } - Command::Lower { store, key } => { + Command::Lower { store, key, ipa } => { let inventory_store = InventoryStore::open(&cli.inventory)?; let inventory = inventory_store.inventory()?; let mut packets = PacketStore::open(&store)?; @@ -502,10 +544,14 @@ fn main() -> Result<()> { &EntityStore::open(&cli.entities)?, ) .map_err(refused)?; - println!( - "{}", - lower(&view, &inventory, &EntityStore::open(&cli.entities)?)? - ); + let entities = EntityStore::open(&cli.entities)?; + if ipa { + for line in pronounce(&view, &entities)? { + println!("{line}"); + } + } else { + println!("{}", lower(&view, &inventory, &entities)?); + } } Command::Score(args) => { let output = match &args.output { diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index 1caa6ac..764a7be 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -12,7 +12,7 @@ use unicode_normalization::{UnicodeNormalization, char::is_combining_mark, is_nf use unicode_segmentation::UnicodeSegmentation; use crate::corpus::{CorpusItem, CorpusItemId, Source, item_key, key_slug}; -use crate::entity::{Entity, EntityOrigin}; +use crate::entity::{Entity, EntityOrigin, Pronunciation}; use crate::form::{is_attestation, is_joining_only, is_language_tag, is_refused_format}; use crate::inventory::{Change, InvId, Inventory, InventoryEntry, InventoryRevision, revision_key}; use crate::packet::{EntityId, Packet}; @@ -453,6 +453,7 @@ impl EntityStore { o.language == n.language && o.forms.len() <= n.forms.len() && n.forms.starts_with(&o.forms) + && (o.pronunciation.is_none() || o.pronunciation == n.pronunciation) }); if !grows { return Err(StoreRefusal::EntityNotAppendOnly { id: entity.id }.into()); @@ -506,6 +507,9 @@ fn check_entity_shape(entity: &Entity) -> Result<(), StoreError> { if lf.forms.len() > EntityBound::MAX_FORMS { return Err(bound(EntityBound::Forms)); } + if let Some(p) = &lf.pronunciation { + check_pronunciation(entity, p)?; + } let mut seen = BTreeSet::new(); for f in &lf.forms { if f.len() > EntityBound::MAX_FORM_BYTES { @@ -536,6 +540,59 @@ fn check_entity_shape(entity: &Entity) -> Result<(), StoreError> { Ok(()) } +/// A pronunciation is bare IPA, NFC, with no brackets, control character or +/// edge whitespace; every rejected variant is the same, differs from the ruled +/// one and is named once; the ruling names its author, a `YYYY-MM-DD` date and +/// its source. +fn check_pronunciation(entity: &Entity, p: &Pronunciation) -> Result<(), StoreError> { + let bad = |why| malformed(entity.id.as_str(), why); + let ipa_ok = |s: &str| { + !s.is_empty() + && s.len() <= EntityBound::MAX_FORM_BYTES + && is_nfc(s) + && s.trim() == s + && !s + .chars() + .any(|c| c.is_control() || is_refused_format(c) || "/[]".contains(c)) + }; + if !ipa_ok(&p.ipa) { + return Err(bad( + "a pronunciation is bare NFC IPA with no brackets or slashes", + )); + } + if !is_attestation(&p.ruled_by) || !is_attestation(&p.source) { + return Err(bad("a pronunciation names who ruled it and where")); + } + let b = p.date.as_bytes(); + let date_ok = b.len() == 10 + && b.iter().enumerate().all(|(i, c)| { + if i == 4 || i == 7 { + *c == b'-' + } else { + c.is_ascii_digit() + } + }) + && (1..=12).contains(&p.date[5..7].parse::().unwrap_or(0)) + && (1..=31).contains(&p.date[8..].parse::().unwrap_or(0)); + if !date_ok { + return Err(bad("a pronunciation is dated YYYY-MM-DD")); + } + if p.rejected.len() > EntityBound::MAX_FORMS { + return Err(StoreRefusal::EntityBounds { + id: entity.id.clone(), + bound: EntityBound::Forms, + } + .into()); + } + let mut seen = BTreeSet::from([&p.ipa]); + if !p.rejected.iter().all(|r| ipa_ok(r) && seen.insert(r)) { + return Err(bad( + "a rejected pronunciation is well-formed, named once and not the ruled one", + )); + } + Ok(()) +} + pub struct InventoryStore { cache: CultCache, } diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 29d8a4b..72ad140 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -838,6 +838,47 @@ fn proof_packet_lowers_and_the_lowering_scores_against_items() { let missing = weksa(d.path(), &["lower", "./p.cc", "packet:0000000000000000"]); assert!(!missing.status.success() && out(&missing).contains("no packet")); + // IPA is the ruled pronunciation or nothing: spelling is not one. + let args = ["lower", "./p.cc", "packet:6e6962752d303031", "--ipa"]; + let unruled = weksa(d.path(), &args); + assert!( + !unruled.status.success() && out(&unruled).contains("NoPronunciation"), + "{}", + out(&unruled) + ); + let ruled = weksa( + d.path(), + &[ + "entity", + "put", + "entity:nibu", + "--form", + "en-US=NIBU", + "--origin", + "cast", + "--persona", + "nibu", + "--fixture", + "f.yaml", + "--pronunciation", + "en-US=ˈnibu", + "--ruled-by", + "Nibu", + "--ruled-on", + "2026-11-07", + "--ruling-source", + "test", + "--rejected", + "ˈnɪbu", + ], + ); + assert!(ruled.status.success(), "{}", out(&ruled)); + let ipa = weksa(d.path(), &args); + assert!(ipa.status.success(), "{}", out(&ipa)); + assert_eq!(String::from_utf8(ipa.stdout).unwrap(), "ˈnibu\n"); + let again = weksa(d.path(), &["lower", "./p.cc", "packet:6e6962752d303031"]); + assert_eq!(String::from_utf8(again.stdout).unwrap(), text); + std::fs::write(d.path().join("out.txt"), &text).unwrap(); let url = fake_embedder(); let scored = weksa( diff --git a/crates/weksa/tests/common/mod.rs b/crates/weksa/tests/common/mod.rs index 07e95ee..d866904 100644 --- a/crates/weksa/tests/common/mod.rs +++ b/crates/weksa/tests/common/mod.rs @@ -93,6 +93,7 @@ pub fn ent(slug: &str, language: &str, forms: &[&str]) -> Entity { forms: vec![LanguageForms { language: language.into(), forms: forms.iter().map(|f| f.to_string()).collect(), + pronunciation: None, }], origin: EntityOrigin::Cast { persona: slug.into(), diff --git a/crates/weksa/tests/entities.rs b/crates/weksa/tests/entities.rs index 46aeb71..5493943 100644 --- a/crates/weksa/tests/entities.rs +++ b/crates/weksa/tests/entities.rs @@ -88,6 +88,7 @@ fn entity_forms_only_grow() { more.forms.push(weksa::LanguageForms { language: "tr".into(), forms: vec!["Kay".into()], + pronunciation: None, }); assert_eq!(store.put(more.clone()).unwrap(), AddOutcome::Added); // The store read back from disk holds the growth. @@ -131,6 +132,7 @@ fn entity_forms_only_grow() { let lf = |language: &str, forms: &[&str]| weksa::LanguageForms { language: language.into(), forms: forms.iter().map(|f| f.to_string()).collect(), + pronunciation: None, }; for bad in [ vec![], @@ -427,3 +429,76 @@ fn packet_schema_v2() { "array" ); } + +#[test] +fn a_pronunciation_is_added_once_and_then_frozen() { + let d = tempfile::tempdir().unwrap(); + let mut store = EntityStore::open(&d.path().join("e.cc")).unwrap(); + let kay = ent("kay", "en-US", &["Kay"]); + store.put(kay.clone()).unwrap(); + let said = |ipa: &str, rejected: &[&str]| { + let mut e = kay.clone(); + e.forms[0].pronunciation = Some(weksa::Pronunciation { + ipa: ipa.into(), + ruled_by: "Kay".into(), + date: "2026-11-07".into(), + source: "test".into(), + rejected: rejected.iter().map(|r| r.to_string()).collect(), + }); + e + }; + // An unruled entity gains a ruling. + let ruled = said("ˈkeɪ", &["ˈkaɪ"]); + assert_eq!(store.put(ruled.clone()).unwrap(), AddOutcome::Added); + let mut store = EntityStore::open(&d.path().join("e.cc")).unwrap(); + assert_eq!(store.get(&eid("kay")).unwrap(), Some(ruled.clone())); + // A ruling is neither changed nor dropped; a changed ruling is a new decision. + for e in [said("ˈkaɪ", &[]), said("ˈkeɪ", &[]), kay.clone()] { + assert_eq!( + refusal(store.put(e)), + StoreRefusal::EntityNotAppendOnly { id: eid("kay") } + ); + } + assert_eq!(store.get(&eid("kay")).unwrap(), Some(ruled)); +} + +#[test] +fn a_malformed_pronunciation_is_refused() { + let d = tempfile::tempdir().unwrap(); + let mut store = EntityStore::open(&d.path().join("e.cc")).unwrap(); + let with = |edit: fn(&mut weksa::Pronunciation)| { + let mut e = ent("kay", "en-US", &["Kay"]); + let mut p = weksa::Pronunciation { + ipa: "ˈkeɪ".into(), + ruled_by: "Kay".into(), + date: "2026-11-07".into(), + source: "test".into(), + rejected: vec!["ˈkaɪ".into()], + }; + edit(&mut p); + e.forms[0].pronunciation = Some(p); + e + }; + let edits: [fn(&mut weksa::Pronunciation); 9] = [ + |p| p.ipa = "".into(), + |p| p.ipa = "/ˈkeɪ/".into(), + |p| p.ipa = "[ˈkeɪ]".into(), + |p| p.ipa = " ˈkeɪ".into(), + |p| p.ruled_by = "".into(), + |p| p.date = "07-11-2026".into(), + |p| p.date = "2026-13-07".into(), + |p| p.rejected = vec!["ˈkeɪ".into()], + |p| p.rejected = vec!["ˈkaɪ".into(), "ˈkaɪ".into()], + ]; + for edit in edits { + assert!( + matches!( + store.put(with(edit)), + Err(StoreError::Refused(StoreRefusal::Malformed { .. })) + ), + "{:?}", + with(edit) + ); + } + assert_eq!(store.get(&eid("kay")).unwrap(), None); +} diff --git a/crates/weksa/tests/entity_door.rs b/crates/weksa/tests/entity_door.rs index bcd53e8..191cb74 100644 --- a/crates/weksa/tests/entity_door.rs +++ b/crates/weksa/tests/entity_door.rs @@ -158,6 +158,7 @@ fn entity_capacity_is_bounded() { let lf = |language: &str, forms: Vec| LanguageForms { language: language.into(), forms, + pronunciation: None, }; let one = |f: &str| vec![f.to_string()]; let make = |forms: Vec| Entity { @@ -331,6 +332,7 @@ fn entities_are_append_only_and_replay_needs_them() { LanguageForms { language: "fr".into(), forms: vec!["Z\u{e8}de".into()], + pronunciation: None, }, ); assert_eq!(refusal(w.entities.put(prepend)), not_append); @@ -344,6 +346,7 @@ fn entities_are_append_only_and_replay_needs_them() { duplicate.forms.push(LanguageForms { language: "en-US".into(), forms: vec!["Zeddy".into()], + pronunciation: None, }); assert!(matches!( refusal(w.entities.put(duplicate)), @@ -371,6 +374,7 @@ fn with_form(slug: &str, form: &str) -> Entity { forms: vec![LanguageForms { language: "en-US".into(), forms: vec![form.into()], + pronunciation: None, }], origin: EntityOrigin::Live { persona: "p".into(), diff --git a/crates/weksa/tests/lower.rs b/crates/weksa/tests/lower.rs index ff90fd7..4f1691c 100644 --- a/crates/weksa/tests/lower.rs +++ b/crates/weksa/tests/lower.rs @@ -11,7 +11,7 @@ use weksa::inventory::proof_revision; use weksa::packet::{EntityId, FeatValue, FeatureValue, Packet, proof_packet}; use weksa::{ CorpusStore, EntityForms, EntityStore, Inventory, InventoryStore, LowerError, LoweringInput, - PacketStore, lower, validate, + PacketStore, Pronunciation, lower, pronounce, validate, }; const PROOF: &str = "Warning: you may endanger yourself by touching the control. \ @@ -245,6 +245,7 @@ fn nibu(forms: &[(&str, &[&str])]) -> Entity { .map(|(language, forms)| LanguageForms { language: (*language).into(), forms: forms.iter().map(|f| (*f).into()).collect(), + pronunciation: None, }) .collect(), origin: EntityOrigin::Cast { @@ -356,3 +357,80 @@ fn a_purpose_the_frame_does_not_say_is_refused_not_dropped() { }) ); } + +fn ruled(ipa: &str, rejected: &[&str]) -> Pronunciation { + Pronunciation { + ipa: ipa.into(), + ruled_by: "Nibu".into(), + date: "2026-11-07".into(), + source: "test".into(), + rejected: rejected.iter().map(|r| (*r).into()).collect(), + } +} + +fn say(entities: &dyn EntityForms) -> Result, LowerError> { + let (inv, _dir) = committed(); + pronounce(&view(&proof_packet(), &inv), entities) +} + +#[test] +fn nibu_round_trips_from_the_committed_seed_to_its_ruled_ipa() { + let seed = committed_entities(); + let nibu = seed + .get(&EntityId::parse("entity:nibu").unwrap()) + .unwrap() + .unwrap(); + let p = nibu.pronunciation_in("en-US").unwrap(); + assert_eq!( + (p.ipa.as_str(), p.ruled_by.as_str(), p.date.as_str()), + ("ˈnibu", "Nibu", "2026-11-07") + ); + assert_eq!(say(&seed).unwrap(), ["ˈnibu"]); +} + +#[test] +fn a_rejected_pronunciation_is_never_emitted() { + let mut e = Entities::default(); + let mut n = nibu(&[("en-US", &["Nibu"])]); + n.forms[0].pronunciation = Some(ruled("ˈnibu", &["ˈnɪbu", "ˈnaɪbu"])); + e.held.push(n); + assert_eq!(say(&e).unwrap(), ["ˈnibu"]); + // The committed seed carries the same rejection and still says only the ruling. + let seed = say(&committed_entities()).unwrap().concat(); + assert!(!seed.contains('ɪ'), "{seed}"); +} + +#[test] +fn an_entity_with_no_ruled_pronunciation_is_refused_not_spelled_out() { + let id = EntityId::parse("entity:nibu").unwrap(); + // No pronunciation at all. + let mut e = Entities::default(); + e.held.push(nibu(&[("en-US", &["Nibu"])])); + assert_eq!( + say(&e), + Err(LowerError::NoPronunciation { entity: id.clone() }) + ); + // A pronunciation under another language is not the target language's. + let mut n = nibu(&[("sa-Latn", &["Nibuwa"]), ("en-US", &["Nibu"])]); + n.forms[0].pronunciation = Some(ruled("ˈnibuwa", &[])); + let mut e = Entities::default(); + e.held.push(n); + assert_eq!(say(&e), Err(LowerError::NoPronunciation { entity: id })); +} + +#[test] +fn ipa_needs_an_entity_the_port_holds_and_a_packet_that_cites_one() { + assert_eq!( + say(&Entities::default()), + Err(LowerError::UnknownEntity { + entity: EntityId::parse("entity:nibu").unwrap() + }) + ); + let (inv, _dir) = committed(); + let mut p = proof_packet(); + p.referents[2].entity = None; + assert_eq!( + pronounce(&view(&p, &inv), &committed_entities()), + Err(LowerError::NoEntityReferent) + ); +} diff --git a/data/entity/entity.cc b/data/entity/entity.cc index 8f9a198..1338530 100644 --- a/data/entity/entity.cc +++ b/data/entity/entity.cc @@ -1 +1 @@ -“²cultcache.store.v1‘—¬weksa.entity¬weksa.entity¯weksa.entity.v1¬weksa.entityÙL{"schemaName":"weksa.entity","schemaVersion":"weksa.entity.v1","members":[]}‘¬weksa.entity’”­entity:brokkr¬weksa.entity´2026-10-29T16:43:28ZÄn“­entity:brokkr‘’¥en-US‘¦Brokkr¤Cast’¦brokkrÙ?examples/persona-lowering/brokkr-edda-voice/lowered-output.yaml”«entity:nibu¬weksa.entity´2026-10-29T16:43:28ZÄe“«entity:nibu‘’¥en-US‘¤Nibu¤Cast’¤nibuÙ --ipa` prints the stored IPA of each +entity the packet cites, one per line, and refuses with `NoPronunciation` for an +entity with none; rejected variants are never read. This is not the plan below: +it carries no timing, contour, length or invented prosody, and Weksa has no +English G2P, so an unruled name has no IPA. + ## Pronunciation Plan Shape ```yaml diff --git a/schemas/cultnet/weksa.entity.schema.json b/schemas/cultnet/weksa.entity.schema.json index b2b14f2..7298534 100644 --- a/schemas/cultnet/weksa.entity.schema.json +++ b/schemas/cultnet/weksa.entity.schema.json @@ -112,12 +112,60 @@ }, "language": { "type": "string" + }, + "pronunciation": { + "description": "How the citation form (`forms[0]`) is said, when someone with the\nauthority to rule on it has. Absent means unruled, never \"spelled as\nread\".", + "anyOf": [ + { + "$ref": "#/$defs/Pronunciation" + }, + { + "type": "null" + } + ], + "default": null } }, "required": [ "language", "forms" ] + }, + "Pronunciation": { + "description": "A ruled pronunciation of an entity's citation form in one language.", + "type": "object", + "properties": { + "date": { + "description": "`YYYY-MM-DD`.", + "type": "string" + }, + "ipa": { + "description": "Broad IPA with stress marks, bare: no enclosing `/` or `[ ]`. 1 to 64\nbytes, NFC.", + "type": "string" + }, + "rejected": { + "description": "Pronunciations that were considered and refused, same shape as `ipa`.\nKept so nobody re-proposes one; never emitted.", + "type": "array", + "default": [], + "items": { + "type": "string" + } + }, + "ruled_by": { + "description": "Who ruled: the entity itself, a writer, an operator.", + "type": "string" + }, + "source": { + "description": "Where the ruling was made: a room, a document, a run.", + "type": "string" + } + }, + "required": [ + "ipa", + "ruled_by", + "date", + "source" + ] } } } From 81572b1a5425e76ac89fc9da20faa0d0581582f5 Mon Sep 17 00:00:00 2001 From: Weksa Date: Sat, 7 Nov 2026 22:04:59 +0000 Subject: [PATCH 003/175] Packet carries an addressee-consent fact (schema v5, V14, V15) Discourse.addressee_consent names the predication the addressee asked for or agreed to. V4 checks it resolves, V14 needs an addressee and a predication, V15 allows only Authored or GameTelemetry sources. The English lowerer renders no consent and refuses a packet carrying one. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/lower.rs | 3 + crates/weksa/src/packet.rs | 10 +++- crates/weksa/src/validate.rs | 41 ++++++++++++- crates/weksa/tests/common/mod.rs | 1 + crates/weksa/tests/entities.rs | 2 +- crates/weksa/tests/lower.rs | 2 + crates/weksa/tests/stores.rs | 4 +- crates/weksa/tests/validate.rs | 59 +++++++++++++++++++ docs/interlingua-standard.md | 16 +++-- .../weksa.interlingua_packet.schema.json | 11 ++++ 10 files changed, 139 insertions(+), 10 deletions(-) diff --git a/crates/weksa/src/lower.rs b/crates/weksa/src/lower.rs index 43a8752..8f7b856 100644 --- a/crates/weksa/src/lower.rs +++ b/crates/weksa/src/lower.rs @@ -139,6 +139,9 @@ pub fn lower( { return unrendered("features"); } + if input.discourse().addressee_consent.is_some() { + return unrendered("addressee consent"); + } if !input.relations().is_empty() { return unrendered("relations"); } diff --git a/crates/weksa/src/packet.rs b/crates/weksa/src/packet.rs index f2f06de..9d6c582 100644 --- a/crates/weksa/src/packet.rs +++ b/crates/weksa/src/packet.rs @@ -16,7 +16,7 @@ pub use crate::inventory::InvId; use crate::inventory::OptionLabel; /// Bumped by a commit that changes these types; the published schema moves with it. -pub const SCHEMA_VERSION: u32 = 4; +pub const SCHEMA_VERSION: u32 = 5; /// Id of a referent or predication within one packet: `n`, `k` a canonical /// decimal 1..=9999. @@ -261,6 +261,13 @@ pub struct Discourse { /// Speech-act id. pub speech_act: Option, pub addressee: Option, + /// The predication the addressee asked for or agreed to. A fact about what + /// the addressee did, never about how the line sounds, so it is no feature: + /// only a source that saw the request may set it (V15), it needs an + /// addressee (V14) and names a predication (V14), and a lowering that + /// speaks for the addressee's request ("you asked her to") reads it here + /// and refuses without it. + pub addressee_consent: Option, pub features: Vec, } @@ -385,6 +392,7 @@ pub fn proof_packet() -> Packet { discourse: Discourse { speech_act: Some(id("speech_act:5")), addressee: Some(node("n1")), + addressee_consent: None, features: vec![], }, referents: vec![ diff --git a/crates/weksa/src/validate.rs b/crates/weksa/src/validate.rs index 6630430..e8aaf2f 100644 --- a/crates/weksa/src/validate.rs +++ b/crates/weksa/src/validate.rs @@ -3,7 +3,7 @@ //! returns `Ok(())` or the refusals. [`LoweringInput`], the only thing a //! lowerer sees, comes from `PacketStore::admit` alone, which builds it through //! the crate-private `admissible`. The stance is checked like every feature -//! list, in slot `Stance`. The rules are V1..V8 and V10..V13: the shape of an id +//! list, in slot `Stance`. The rules are V1..V8 and V10..V15: the shape of an id //! or a grade is its type's parse, not a rule, so V9 is retired and not //! renumbered. V13 guards the one other way features reach a lowering view: //! [`LoweringInput::with_scene`]. @@ -29,7 +29,7 @@ use std::fmt; use crate::inventory::{Family, InvId, Inventory, InventoryEntry, Slot, ValueKind}; use crate::packet::{ Constraints, Context, Discourse, Extension, FeatValue, FeatureValue, NodeId, Packet, - PacketKind, Predication, Referent, Relation, SCHEMA_VERSION, Stance, + PacketKind, Predication, ProvenanceSource, Referent, Relation, SCHEMA_VERSION, Stance, }; /// A broken rule, named. @@ -71,6 +71,15 @@ pub enum Refusal { /// state the speaker's relation to the addressee, so with none they /// describe nobody. V13SceneWithoutAddressee, + /// V14: an addressee consent that no addressee can have given, or that + /// names a node that is not a predication. It is what the addressee asked + /// for or agreed to, an act, so a referent cannot be its object. + V14ConsentWithoutAddressee, + V14ConsentNotAPredication { id: NodeId }, + /// V15: an addressee consent set by a source that only read wording. + /// A decomposition of text, or imported corpus text, sees "you asked her + /// to" as words; only authoring or game telemetry saw the request. + V15ConsentSource { source: ProvenanceSource }, } impl Refusal { @@ -90,6 +99,10 @@ impl Refusal { | Refusal::V11DuplicateRole { .. } => "V11", Refusal::V12TooManyNodes { .. } => "V12", Refusal::V13SceneWithoutAddressee => "V13", + Refusal::V14ConsentWithoutAddressee | Refusal::V14ConsentNotAPredication { .. } => { + "V14" + } + Refusal::V15ConsentSource { .. } => "V15", } } } @@ -156,6 +169,7 @@ pub(crate) fn admissible( let nodes = check_nodes(packet, &mut out); check_refs(packet, &nodes, &mut out); check_repeats(packet, &mut out); + check_consent(packet, &nodes, &mut out); if inventory.has_revision(packet.inventory_rev) { Inventoried { inv: inventory, @@ -208,6 +222,9 @@ fn check_refs(packet: &Packet, nodes: &BTreeSet, out: &mut Vec) if let Some(a) = &packet.discourse.addressee { need(a, "addressee"); } + if let Some(c) = &packet.discourse.addressee_consent { + need(c, "addressee consent"); + } for (list, at) in node_lists(packet) { for id in list { need(id, at); @@ -215,6 +232,26 @@ fn check_refs(packet: &Packet, nodes: &BTreeSet, out: &mut Vec) } } +/// V14 and V15: the addressee's consent is a typed fact with one origin. A +/// dangling id is V4's, so V14 judges only a node that exists. +fn check_consent(packet: &Packet, nodes: &BTreeSet, out: &mut Vec) { + let Some(consent) = &packet.discourse.addressee_consent else { + return; + }; + if packet.discourse.addressee.is_none() { + out.push(Refusal::V14ConsentWithoutAddressee); + } + if nodes.contains(consent) && !packet.predications.iter().any(|p| &p.id == consent) { + out.push(Refusal::V14ConsentNotAPredication { + id: consent.clone(), + }); + } + match packet.provenance.source { + ProvenanceSource::Authored | ProvenanceSource::GameTelemetry => {} + source => out.push(Refusal::V15ConsentSource { source }), + } +} + fn node_lists(packet: &Packet) -> [(&Vec, &'static str); 3] { [ (&packet.context.participants, "participant"), diff --git a/crates/weksa/tests/common/mod.rs b/crates/weksa/tests/common/mod.rs index d866904..60ba544 100644 --- a/crates/weksa/tests/common/mod.rs +++ b/crates/weksa/tests/common/mod.rs @@ -266,6 +266,7 @@ pub fn good() -> Packet { discourse: Discourse { speech_act: Some(iv(WARN)), addressee: Some(nd("n1")), + addressee_consent: None, features: vec![FeatureValue { feature: iv(HEDGE), value: FeatValue::Grade(grade(300)), diff --git a/crates/weksa/tests/entities.rs b/crates/weksa/tests/entities.rs index 5493943..5f1f7cf 100644 --- a/crates/weksa/tests/entities.rs +++ b/crates/weksa/tests/entities.rs @@ -397,7 +397,7 @@ fn packet_entities_resolve() { #[test] fn packet_schema_v2() { - assert_eq!(SCHEMA_VERSION, 4); + assert_eq!(SCHEMA_VERSION, 5); let mut p = good(); p.schema_version = 2; let Err(refused) = validate(&p, &inventory()) else { diff --git a/crates/weksa/tests/lower.rs b/crates/weksa/tests/lower.rs index 4f1691c..1d8301d 100644 --- a/crates/weksa/tests/lower.rs +++ b/crates/weksa/tests/lower.rs @@ -142,6 +142,7 @@ fn a_packet_carrying_what_the_lowerer_does_not_render_is_refused() { filler, }) }; + let consent = |p: &mut Packet| p.discourse.addressee_consent = Some(nd("n5")); let lone_referent = |p: &mut Packet| { let mut r = p.referents[0].clone(); r.id = nd("n99"); @@ -150,6 +151,7 @@ fn a_packet_carrying_what_the_lowerer_does_not_render_is_refused() { for (what, edit) in [ ("features", &discourse as &dyn Fn(&mut Packet)), ("features", &predication), + ("addressee consent", &consent), ("relations", &relation), ("extensions", &extension), ("no predication", &no_predication), diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs index 07cbe0c..cf5ba4a 100644 --- a/crates/weksa/tests/stores.rs +++ b/crates/weksa/tests/stores.rs @@ -989,14 +989,14 @@ fn packet_admission_goes_through_the_validator() { let mut w = world(); let mut store = PacketStore::open(&w.dir.path().join("p.cc")).unwrap(); let mut bad = good(); - bad.schema_version = 5; + bad.schema_version = 6; match store.admit(bad, &w.inventory, &w.corpus, &w.entities) { Err(e @ StoreError::Invalid(_)) => { assert!(e.to_string().contains("V1"), "{e}"); let StoreError::Invalid(r) = e else { unreachable!() }; - assert_eq!(r, vec![Refusal::V1SchemaVersion { found: 5 }]); + assert_eq!(r, vec![Refusal::V1SchemaVersion { found: 6 }]); } other => panic!("{other:?}"), } diff --git a/crates/weksa/tests/validate.rs b/crates/weksa/tests/validate.rs index 3453376..aaa87bd 100644 --- a/crates/weksa/tests/validate.rs +++ b/crates/weksa/tests/validate.rs @@ -683,3 +683,62 @@ fn v11_role_repeat_in_one_predication_refused() { ok.predications.push(second); assert_eq!(refusals(&ok), vec![]); } + +// The addressee's consent: which predication the addressee asked for, set only +// by a source that saw the request. + +fn consenting() -> Packet { + let mut p = good(); + p.discourse.addressee_consent = Some(nd("n2")); + p +} + +#[test] +fn consent_to_a_predication_by_an_authored_or_telemetry_source_passes() { + assert_eq!(refusals(&consenting()), vec![]); + let mut p = consenting(); + p.provenance.source = ProvenanceSource::GameTelemetry; + assert_eq!(refusals(&p), vec![]); +} + +#[test] +fn consent_needs_an_addressee() { + let mut p = consenting(); + p.discourse.addressee = None; + assert_eq!(refusals(&p), vec![Refusal::V14ConsentWithoutAddressee]); +} + +#[test] +fn consent_names_a_predication_not_a_referent() { + let mut p = consenting(); + p.discourse.addressee_consent = Some(nd("n1")); + assert_eq!( + refusals(&p), + vec![Refusal::V14ConsentNotAPredication { id: nd("n1") }] + ); +} + +#[test] +fn consent_to_nothing_is_dangling_only() { + let mut p = consenting(); + p.discourse.addressee_consent = Some(nd("n99")); + assert_eq!( + refusals(&p), + vec![Refusal::V4DanglingRef { + id: nd("n99"), + at: "addressee consent" + }] + ); +} + +#[test] +fn a_source_that_only_read_wording_cannot_set_consent() { + for source in [ + ProvenanceSource::ModelDecomposition, + ProvenanceSource::ImportedCorpus, + ] { + let mut p = consenting(); + p.provenance.source = source; + assert_eq!(refusals(&p), vec![Refusal::V15ConsentSource { source }]); + } +} diff --git a/docs/interlingua-standard.md b/docs/interlingua-standard.md index 6a835dd..ca392d1 100644 --- a/docs/interlingua-standard.md +++ b/docs/interlingua-standard.md @@ -120,7 +120,7 @@ Packet id, schema_version, inventory_rev, kind provenance { source, corpus_item, encoder, confidence } context { setting, medium, participants, audience } - discourse { speech_act, addressee, features } + discourse { speech_act, addressee, addressee_consent, features } referents [ { id, kind, concept, entity, features } ] predications [ { id, concept, frame, roles: [ { role, filler } ], features } ] relations [ { relation, from, to, features } ] @@ -144,8 +144,8 @@ names the rule. - V1: `schema_version` is the version these types define. - V2: at least one of `referents`, `predications`, `relations` is present. - V3: a node id appears once across referents and predications. -- V4: every role filler, relation end, addressee, participant, audience member - and preserved node names a node. +- V4: every role filler, relation end, addressee, addressee consent, + participant, audience member and preserved node names a node. - V5: every inventory id resolves at the packet's `inventory_rev`: introduced at or before it, and not retired at or before it. - V6: an inventory id is of the family its slot takes. @@ -159,6 +159,10 @@ names the rule. predication's roles. - V12: a packet has at most 64 referents, predications and relations together (a calibration constant, capacity and not meaning). +- V14: an `addressee_consent` needs an `addressee` and names a predication. +- V15: an `addressee_consent` is set only by provenance `Authored` or + `GameTelemetry`; a decomposition of text or imported corpus text cannot mint + it. V9 is retired and not renumbered: the form of a key or a grade is its type, and an extension key's family is V6. `PacketStore::admit` returns the lowering view of a @@ -189,7 +193,11 @@ ids. Why the packet is said: `speech_act` is a speech-act id, `addressee` a node id, and `features` carry source-side nuance on content (hedge, certainty, irony and whatever the inventory grows). This is source-side communicative pressure, not -target-language register. +target-language register. `addressee_consent` is no feature: it names the +predication the addressee asked for or agreed to, a fact about what the +addressee did. The English lowerer renders no consent and refuses a packet that +carries one; a row that speaks for the request ("you asked her to") reads it +and refuses without it. ### `stance` diff --git a/schemas/cultnet/weksa.interlingua_packet.schema.json b/schemas/cultnet/weksa.interlingua_packet.schema.json index d839203..97442f9 100644 --- a/schemas/cultnet/weksa.interlingua_packet.schema.json +++ b/schemas/cultnet/weksa.interlingua_packet.schema.json @@ -164,6 +164,17 @@ } ] }, + "addressee_consent": { + "description": "The predication the addressee asked for or agreed to. A fact about what\nthe addressee did, never about how the line sounds, so it is no feature:\nonly a source that saw the request may set it (V15), it needs an\naddressee (V14) and names a predication (V14), and a lowering that\nspeaks for the addressee's request (\"you asked her to\") reads it here\nand refuses without it.", + "anyOf": [ + { + "$ref": "#/$defs/NodeId" + }, + { + "type": "null" + } + ] + }, "features": { "type": "array", "items": { From 17f44144a5372ec80eefd318c31a8def35986b27 Mon Sep 17 00:00:00 2001 From: Weksa Date: Sun, 8 Nov 2026 00:50:39 +0000 Subject: [PATCH 004/175] Show her working: static page of the lowering chain from the real lowerer and scorer Co-Authored-By: Claude Sonnet 5.5 --- .gitignore | 1 + crates/weksa/examples/show_working.rs | 373 ++++++++++++++++++++++++++ scripts/publish-show-working.sh | 16 ++ tools/show-working/README.md | 9 + tools/show-working/build.py | 203 ++++++++++++++ 5 files changed, 602 insertions(+) create mode 100644 crates/weksa/examples/show_working.rs create mode 100755 scripts/publish-show-working.sh create mode 100644 tools/show-working/README.md create mode 100755 tools/show-working/build.py diff --git a/.gitignore b/.gitignore index 1428a57..e00572b 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ build/ .venv/ venv/ target/ +tools/show-working/out/ diff --git a/crates/weksa/examples/show_working.rs b/crates/weksa/examples/show_working.rs new file mode 100644 index 0000000..cf24b64 --- /dev/null +++ b/crates/weksa/examples/show_working.rs @@ -0,0 +1,373 @@ +//! Build-time generator for the "Show her working" page. Runs the real +//! admission door, lowerer and IPA reader over the committed proof packet and +//! variants of it, and prints one JSON document on stdout: the typed slots, the +//! inventory entries the packet chose, and the outcome, which is lowered text or +//! a named refusal. Scoring is not here: `tools/show-working/build.py` calls the +//! `weksa score` CLI on the text this emits. +//! +//! Run from the repository root: `cargo run --release --example show_working`. +//! Every variant is a mutation of `proof_packet()`, spelled in its `source`. + +use std::path::Path; + +use serde_json::{Value, json}; +use weksa::packet::{ + EntityId, FeatValue, FeatureValue, Grade, NodeId, ProvenanceSource, RoleFill, proof_packet, +}; +use weksa::{ + CorpusStore, EntityStore, InvId, Inventory, InventoryStore, LowerError, Packet, PacketStore, + StoreError, lower, pronounce, +}; + +struct Fixture { + id: &'static str, + title: &'static str, + note: &'static str, + source: &'static str, + packet: Packet, +} + +fn id(s: &str) -> InvId { + s.parse().expect("inventory id") +} + +fn node(s: &str) -> NodeId { + NodeId::parse(s).expect("node id") +} + +fn grade(feature: &str, v: u16) -> FeatureValue { + FeatureValue { + feature: id(feature), + value: FeatValue::Grade(Grade::new(v).expect("grade")), + } +} + +fn fixtures() -> Vec { + let proof = proof_packet(); + let mut out = vec![Fixture { + id: "proof", + title: "Nibu warns the player off the control", + note: "The committed proof packet, unchanged.", + source: "proof_packet() in crates/weksa/src/packet.rs", + packet: proof.clone(), + }]; + let mut add = |id, title, note, source, f: &dyn Fn(&mut Packet)| { + let mut packet = proof.clone(); + f(&mut packet); + out.push(Fixture { id, title, note, source, packet }); + }; + add( + "no-addressee", + "Same meaning, nobody addressed", + "With the addressee unset the player is a third party: no \"you\", no reflexive.", + "proof_packet() with discourse.addressee = None", + &|p| p.discourse.addressee = None, + ); + add( + "stance-held-back", + "Anger held back, restraint high", + "Stance is the speaker's felt affect. The lowerer has no row that reads it, so the text is the proof's text.", + "proof_packet() with stance anger=300, restraint=700", + &|p| p.stance.features = vec![grade("feature:9", 300), grade("feature:18", 700)], + ); + add( + "hedged", + "Hedged and uncertain warning", + "Hedge and certainty are meaning the lowerer cannot yet say. It refuses rather than drop them.", + "proof_packet() with discourse hedge=600, certainty=400", + &|p| p.discourse.features = vec![grade("feature:6", 600), grade("feature:7", 400)], + ); + add( + "negated", + "Negated intervention (polarity: negative)", + "Dropping the negation would say the opposite, so the lowerer refuses. This is the shape of a negated Dao line awaiting polarity.", + "proof_packet() with predication n5 polarity=negative", + &|p| { + p.predications[1].features = vec![FeatureValue { + feature: id("feature:1"), + value: FeatValue::Option("negative".parse().expect("label")), + }] + }, + ); + add( + "consent-authored", + "Addressee consent, authored source", + "Someone who saw the request says the addressee asked for it. No lowering speaks for that request yet.", + "proof_packet() with discourse.addressee_consent = n4 (source Authored)", + &|p| p.discourse.addressee_consent = Some(node("n4")), + ); + add( + "consent-imported", + "Addressee consent, imported-corpus source", + "A reader of wording cannot have seen a request, so the packet is refused at admission (V15), before any lowering.", + "proof_packet() with addressee_consent = n4 and provenance.source = ImportedCorpus", + &|p| { + p.discourse.addressee_consent = Some(node("n4")); + p.provenance.source = ProvenanceSource::ImportedCorpus; + }, + ); + add( + "relation", + "A causal relation between the predications", + "Relations are not rendered; the lowerer refuses instead of asserting a different meaning.", + "proof_packet() with relation:8 (causes) n4 -> n5", + &|p| { + p.relations = vec![weksa::packet::Relation { + relation: id("relation:8"), + from: node("n4"), + to: node("n5"), + features: vec![], + }] + }, + ); + add( + "dangling-filler", + "A role filled by a node that does not exist", + "Refused at admission (V4): the packet is not a packet.", + "proof_packet() with predication n5 agent filler n9", + &|p| { + p.predications[1].roles[0] = RoleFill { role: id("role:1"), filler: node("n9") } + }, + ); + add( + "unknown-entity", + "A referent naming an entity nobody holds", + "The entity store is the only owner of names; a packet cannot invent one.", + "proof_packet() with referent n3 entity = entity:nobody", + &|p| p.referents[2].entity = Some(EntityId::parse("entity:nobody").expect("entity")), + ); + add( + "common-noun-agent", + "The ship mind as a common noun", + "No entity: the referent lowers as \"the ship mind\", and there is no name to pronounce.", + "proof_packet() with referent n3 entity = None", + &|p| p.referents[2].entity = None, + ); + out +} + +fn gloss(inv: &Inventory, rev: u32, i: &InvId) -> Value { + match inv.resolve(i, rev) { + Some(e) => json!({ "id": i.to_string(), "gloss": e.gloss }), + None => json!({ "id": i.to_string(), "gloss": null }), + } +} + +fn feature_json(inv: &Inventory, rev: u32, slot: &str, at: &str, f: &FeatureValue) -> Value { + let mut g = gloss(inv, rev, &f.feature); + g["slot"] = json!(slot); + g["at"] = json!(at); + g["value"] = serde_json::to_value(&f.value).expect("value"); + g +} + +/// Every feature the packet carries, with where it sits. +fn features(p: &Packet, inv: &Inventory) -> Vec { + let rev = p.inventory_rev; + let mut v = Vec::new(); + for f in &p.discourse.features { + v.push(feature_json(inv, rev, "discourse", "discourse", f)); + } + for f in &p.stance.features { + v.push(feature_json(inv, rev, "stance", "stance", f)); + } + for r in &p.referents { + for f in &r.features { + v.push(feature_json(inv, rev, "referent", r.id.as_str(), f)); + } + } + for pr in &p.predications { + for f in &pr.features { + v.push(feature_json(inv, rev, "predication", pr.id.as_str(), f)); + } + } + v +} + +/// Ids the packet chose, once each, with the inventory's gloss at its revision. +fn chosen(p: &Packet, inv: &Inventory) -> Vec { + let rev = p.inventory_rev; + let mut ids: Vec = Vec::new(); + let mut push = |i: &InvId| { + if !ids.contains(i) { + ids.push(*i) + } + }; + p.context.setting.iter().for_each(&mut push); + p.context.medium.iter().for_each(&mut push); + p.discourse.speech_act.iter().for_each(&mut push); + for r in &p.referents { + push(&r.kind); + push(&r.concept); + } + for pr in &p.predications { + push(&pr.concept); + push(&pr.frame); + pr.roles.iter().for_each(|f| push(&f.role)); + } + p.relations.iter().for_each(|r| push(&r.relation)); + features(p, inv) + .iter() + .filter_map(|f| f["id"].as_str().and_then(|s| s.parse().ok())) + .for_each(|i: InvId| push(&i)); + ids.iter().map(|i| gloss(inv, rev, i)).collect() +} + +fn slots(p: &Packet) -> Value { + let j = |v: &dyn erased::Ser| v.to_value(); + json!({ + "meaning": { + "referents": p.referents.iter().map(|r| j(r)).collect::>(), + "predications": p.predications.iter().map(|r| j(r)).collect::>(), + "relations": p.relations.iter().map(|r| j(r)).collect::>(), + }, + "stance": j(&p.stance), + "discourse": j(&p.discourse), + "context": j(&p.context), + "extensions": p.extensions.iter().map(|r| j(r)).collect::>(), + "constraints": j(&p.constraints), + "provenance": { "source": j(&p.provenance.source), "corpus_item": p.provenance.corpus_item.to_string(), "confidence": p.provenance.confidence.get() }, + "kind": j(&p.kind), + "inventory_rev": p.inventory_rev, + }) +} + +mod erased { + pub trait Ser { + fn to_value(&self) -> serde_json::Value; + } + impl Ser for T { + fn to_value(&self) -> serde_json::Value { + serde_json::to_value(self).expect("serialize") + } + } +} + +fn lower_error(e: &LowerError) -> Value { + let (variant, reason) = match e { + LowerError::Unrendered { what } => ("Unrendered", format!("the packet carries {what}, which the lowerer does not render")), + LowerError::NoRow { id } => ("NoRow", format!("no table row for {id}")), + LowerError::GlossMismatch { id, row } => ("GlossMismatch", format!("{id} is not the {row} the row speaks for")), + LowerError::UnknownRole { frame, role } => ("UnknownRole", format!("{frame} names role {role}, which the inventory lacks")), + LowerError::MissingRole { predication, role } => ("MissingRole", format!("{predication} fills no {role} role")), + LowerError::UnknownEntity { entity } => ("UnknownEntity", format!("the entity store does not hold {entity}")), + LowerError::NoForm { entity } => ("NoForm", format!("{entity} has no en-US form")), + LowerError::NoPronunciation { entity } => ("NoPronunciation", format!("{entity} has no ruled en-US pronunciation")), + LowerError::NoEntityReferent => ("NoEntityReferent", "the packet cites no entity".to_string()), + LowerError::EntityRead { entity, why } => ("EntityRead", format!("reading {entity}: {why}")), + }; + json!({ "variant": variant, "reason": reason, "debug": format!("{e:?}") }) +} + +struct World { + inventory: InventoryStore, + corpus: CorpusStore, + entities: EntityStore, + inv: Inventory, +} + +/// Admit `packet` through a fresh packet store and lower it. `Ok` carries the +/// text and the IPA outcome; `Err` is the refusal, as JSON. +fn run(w: &World, dir: &Path, packet: &Packet) -> Result<(String, Value), Value> { + let mut store = PacketStore::open(&dir.join("p.cc")).expect("packet store"); + let view = match store.admit(packet.clone(), &w.inventory, &w.corpus, &w.entities) { + Ok(v) => v, + Err(StoreError::Invalid(rs)) => { + return Err(json!({ + "kind": "admission_refused", + "refusals": rs.iter().map(|r| json!({ "rule": r.rule(), "detail": format!("{r:?}") })).collect::>(), + })); + } + Err(e) => { + return Err(json!({ "kind": "admission_refused", "refusals": [{ "rule": "store", "detail": format!("{e:?}") }] })); + } + }; + let text = lower(&view, &w.inv, &w.entities).map_err(|e| { + json!({ "kind": "lower_error", "error": lower_error(&e) }) + })?; + let ipa = match pronounce(&view, &w.entities) { + Ok(v) => json!({ "ok": v }), + Err(e) => json!({ "error": lower_error(&e) }), + }; + Ok((text, ipa)) +} + +fn main() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let inventory = InventoryStore::open(&root.join("data/interlingua/inventory.cc")).expect("inventory"); + let inv = inventory.inventory().expect("inventory view"); + let w = World { + corpus: CorpusStore::open(&root.join("data/corpus/corpus.cc")).expect("corpus"), + entities: EntityStore::open(&root.join("data/entity/entity.cc")).expect("entities"), + inventory, + inv, + }; + let dir = std::env::temp_dir().join(format!("show-working-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("scratch dir"); + let mut out = Vec::new(); + for fx in fixtures() { + let p = &fx.packet; + let first = run(&w, &dir, p); + let outcome = match first { + Err(e) if e["kind"] == "admission_refused" => e, + first => { + // Ablation: run again with each feature removed, against the + // baseline outcome. It is how the page says whether the lowerer + // read a feature, instead of the page asserting it. + let base = first.as_ref().ok().map(|(t, _)| t.clone()); + let n_disc = p.discourse.features.len(); + let n_stance = p.stance.features.len(); + let total = features(p, &w.inv).len(); + let mut effects = Vec::new(); + for k in 0..total { + let mut q = p.clone(); + if k < n_disc { + q.discourse.features.remove(k); + } else if k < n_disc + n_stance { + q.stance.features.remove(k - n_disc); + } else { + let mut at = k - n_disc - n_stance; + 'o: { + for r in &mut q.referents { + if at < r.features.len() { r.features.remove(at); break 'o; } + at -= r.features.len(); + } + for pr in &mut q.predications { + if at < pr.features.len() { pr.features.remove(at); break 'o; } + at -= pr.features.len(); + } + } + } + let _ = std::fs::remove_file(dir.join("p.cc")); + effects.push(match (&base, run(&w, &dir, &q)) { + (Some(t), Ok((u, _))) if *t == u => "Without it the text is identical: the lowerer did not read it.", + (Some(_), Ok(_)) => "Without it the text changes: the lowerer read it.", + (Some(_), Err(_)) => "Without it the packet is refused.", + (None, Ok(_)) => "Without it the packet lowers: this feature is what the lowerer refuses.", + (None, Err(_)) => "Still refused without this feature alone.", + }); + } + match first { + Ok((text, ipa)) => json!({ "kind": "lowered", "text": text, "ipa": ipa, "feature_effects": effects }), + Err(e) => { let mut e = e; e["feature_effects"] = json!(effects); e } + } + } + }; + let _ = std::fs::remove_file(dir.join("p.cc")); + let mut slots = slots(p); + slots["features"] = json!(features(p, &w.inv)); + out.push(json!({ + "id": fx.id, + "title": fx.title, + "note": fx.note, + "source": fx.source, + "packet_id": p.id.to_string(), + "corpus_item": p.provenance.corpus_item.to_string(), + "slots": slots, + "chosen": chosen(p, &w.inv), + "outcome": outcome, + })); + } + let _ = std::fs::remove_dir_all(&dir); + println!("{}", serde_json::to_string_pretty(&json!({ "fixtures": out })).expect("json")); +} diff --git a/scripts/publish-show-working.sh b/scripts/publish-show-working.sh new file mode 100755 index 0000000..3af3194 --- /dev/null +++ b/scripts/publish-show-working.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Build the "Show her working" page from the real lowerer and scorer, then publish it as a Mind preview. +# Run from the repository root, in a Mind's Self run (it writes only /srv/minds-web//). +# +# scripts/publish-show-working.sh [--no-publish] +# +# Needs cargo, python3 and an Ollama embedder at 127.0.0.1:11434 (qwen3-embedding:0.6b): the scorer is not stubbed. +# publish-mind-preview.sh lives in the gamecult-ops checkout: set GAMECULT_OPS to it (default ../gamecult-ops). +set -euo pipefail +[ -f Cargo.toml ] && [ -d tools/show-working ] || { echo "run from the Weksa repository root" >&2; exit 2; } +out=${SHOW_WORKING_OUT:-tools/show-working/out} +rm -rf "$out" +python3 -I tools/show-working/build.py "$out" +[ "${1:-}" = --no-publish ] && { echo "built $out"; exit 0; } +ops=${GAMECULT_OPS:-../gamecult-ops} +exec "$ops/scripts/publish-mind-preview.sh" "$out" weksa-show-working diff --git a/tools/show-working/README.md b/tools/show-working/README.md new file mode 100644 index 0000000..5e2a320 --- /dev/null +++ b/tools/show-working/README.md @@ -0,0 +1,9 @@ +# Show her working + +A static page that shows the lowering chain for committed fixture packets: typed slots, inventory entries chosen, lowered English, scorer closeness to the corpus target, with hedge, certainty and restraint annotated. Refusals (admission rules, `LowerError`, addressee consent, negation awaiting polarity) are results, not omissions. + +- `crates/weksa/examples/show_working.rs`: runs the real admission door, `lower` and `pronounce` over `proof_packet()` and its variants and prints JSON. Each feature is removed in turn and the packet re-run, so "the lowerer read this" is measured. +- `build.py`: runs that, scores each lowered text with the `weksa score` CLI (needs Ollama with `qwen3-embedding:0.6b`), writes `out/data.json` and one dependency-free `out/index.html`. Fails if anything cannot run; there is no fallback text. +- `scripts/publish-show-working.sh`: build, then publish through gamecult-ops `publish-mind-preview.sh`. + +Adding a fixture: add an entry in `fixtures()` in the example. A packet from another store needs a loader there; none exists yet. diff --git a/tools/show-working/build.py b/tools/show-working/build.py new file mode 100755 index 0000000..269032e --- /dev/null +++ b/tools/show-working/build.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""Build the "Show her working" page from the real lowerer and scorer. + +usage: build.py [out_dir] (run from the repository root) + +1. `cargo run --example show_working` admits each fixture packet through the + packet store and lowers it with the English lowerer (crates/weksa/examples). +2. `weksa score` scores each lowered text against its corpus item (and the + other committed Nibu reference) through the Ollama embedder. +3. out_dir/data.json holds the joined result; out_dir/index.html is one file + with no scripts to fetch and no network calls. + +Nothing on the page is typed by hand except the layout and the reading notes. +If cargo or the scorer cannot run, the build fails: there is no fallback text. +Standard library only. +""" +import json +import pathlib +import re +import subprocess +import sys +import time +from html import escape as e + +root = pathlib.Path.cwd() +out_dir = pathlib.Path(sys.argv[1] if len(sys.argv) > 1 else "tools/show-working/out") +weksa_bin = None + + +def run(cmd, **kw): + r = subprocess.run(cmd, cwd=root, capture_output=True, text=True, **kw) + if r.returncode != 0: + sys.exit(f"{' '.join(map(str, cmd))} failed:\n{r.stderr}") + return r.stdout + + +def cargo_target(): + meta = json.loads(run(["cargo", "metadata", "--format-version", "1", "--no-deps"])) + return pathlib.Path(meta["target_directory"]) + + +# -- 1. the real lowerer ------------------------------------------------------ +run(["cargo", "build", "--release", "--bin", "weksa", "--example", "show_working"]) +target = cargo_target() / "release" +weksa_bin = target / "weksa" +data = json.loads(run([str(target / "examples" / "show_working")])) + +# -- 2. the real scorer ------------------------------------------------------- +corpus = {} +for m in re.finditer(r'^(item:\S+) \[(.*?)\]\n "(.*)"$', run([str(weksa_bin), "corpus", "dump"]), re.M): + corpus[m.group(1)] = {"text": m.group(3), "meta": m.group(2)} +REFS_EXTRA = ["item:nibu:2"] +model = "qwen3-embedding:0.6b" + + +def score(text, items): + out = run([str(weksa_bin), "score", *items], input=text) + got = {} + for m in re.finditer(r"^(item:\S+) chrf=([\d.]+) meaning=(-?[\d.]+)", out, re.M): + got[m.group(1)] = {"chrf": float(m.group(2)), "meaning": float(m.group(3))} + if set(got) != set(items): + sys.exit(f"scorer answered {sorted(got)} for {items}:\n{out}") + return got + + +for fx in data["fixtures"]: + o = fx["outcome"] + if o["kind"] == "lowered": + refs = [fx["corpus_item"]] + [r for r in REFS_EXTRA if r != fx["corpus_item"]] + o["scores"] = [{"item": k, "target": k == fx["corpus_item"], **v} for k, v in score(o["text"], refs).items()] + +data["corpus"] = corpus +data["build"] = { + "commit": run(["git", "rev-parse", "HEAD"]).strip(), + "embedder": model, + "built_unix": int(time.time()), +} +out_dir.mkdir(parents=True, exist_ok=True) +(out_dir / "data.json").write_text(json.dumps(data, indent=1, ensure_ascii=False), encoding="utf-8") + +# -- 3. the page -------------------------------------------------------------- +CSS = """ +:root{--ink:#1c1b19;--mute:#5d5a54;--line:#cfc9bd;--bg:#f6f3ec;--card:#fff;--ok:#0b6b4b;--warn:#9a4a00;--bad:#9b1c2c;--bar:#365f9c} +*{box-sizing:border-box}body{font:16px/1.55 system-ui,sans-serif;max-width:62rem;margin:2rem auto;padding:0 1rem;color:var(--ink);background:var(--bg)} +h1{margin:0 0 .2rem}h2{margin:0 0 .4rem}h3{margin:1.2rem 0 .3rem;font-size:.85rem;text-transform:uppercase;letter-spacing:.06em;color:var(--mute)} +code,.mono{font:13px/1.4 ui-monospace,Menlo,monospace}code{background:#ece8dd;padding:0 .25rem;border-radius:3px} +.lede{max-width:46rem}.meta{color:var(--mute);font-size:.88rem} +nav.pick{display:flex;flex-wrap:wrap;gap:.4rem;margin:1.2rem 0} +nav.pick a{border:1px solid var(--line);background:var(--card);padding:.25rem .6rem;border-radius:999px;text-decoration:none;color:var(--ink);font-size:.9rem} +nav.pick a.refused{border-color:var(--bad)}nav.pick a.lowered{border-color:var(--ok)} +article{background:var(--card);border:1px solid var(--line);border-radius:8px;padding:1rem 1.3rem;margin:1.4rem 0} +.chain{display:grid;grid-template-columns:repeat(4,1fr);gap:.6rem;margin:.6rem 0 1rem} +.chain div{border:1px solid var(--line);border-radius:6px;padding:.3rem .6rem;font-size:.82rem;color:var(--mute)} +.chain b{display:block;color:var(--ink)} +table{border-collapse:collapse;width:100%;font-size:.9rem}th,td{text-align:left;padding:.25rem .5rem;border-bottom:1px solid #e6e1d5;vertical-align:top}th{color:var(--mute);font-weight:600} +.out{font-size:1.15rem;padding:.6rem .9rem;border-left:4px solid var(--ok);background:#eef6f1} +.refusal{padding:.6rem .9rem;border-left:4px solid var(--bad);background:#faeef0}.refusal b{color:var(--bad)} +.tag{display:inline-block;font-size:.75rem;padding:0 .4rem;border-radius:3px;border:1px solid var(--line);color:var(--mute)} +.tag.read{border-color:var(--ok);color:var(--ok)}.tag.unread{border-color:var(--warn);color:var(--warn)}.tag.refuse{border-color:var(--bad);color:var(--bad)} +.bar{height:.7rem;background:#e4dfd2;border-radius:3px;min-width:8rem;position:relative}.bar i{display:block;height:100%;background:var(--bar);border-radius:3px} +.ref{font-size:.88rem;color:var(--mute);margin:.2rem 0 .6rem}.ref q{color:var(--ink)} +details{margin:.5rem 0}summary{cursor:pointer;color:var(--mute);font-size:.88rem} +section.gaps li{margin:.3rem 0} +@media(max-width:700px){.chain{grid-template-columns:1fr 1fr}} +""" + +PICK_JS = """(function(){var a=[].slice.call(document.querySelectorAll('article[data-fx]'));function show(){var h=location.hash.slice(1);var any=a.some(function(x){return x.id===h});a.forEach(function(x){x.hidden=any&&x.id!==h})} +window.addEventListener('hashchange',show);show()})()""" + + +def g(chosen, i): + return next((c["gloss"] for c in chosen if c["id"] == i), "?") + + +def val(v): + (k, x), = v.items() + return f"{x}‰" if k == "Grade" else str(x) + + +def slots_html(fx): + s, ch = fx["slots"], fx["chosen"] + rows = [] + for r in s["meaning"]["referents"]: + ent = f" · entity {e(r['entity'])}" if r["entity"] else "" + rows.append(f"{e(r['id'])}referent{e(g(ch, r['kind']))} / {e(g(ch, r['concept']))}{ent}") + for p in s["meaning"]["predications"]: + roles = ", ".join(f"{e(g(ch, x['role']))}={e(x['filler'])}" for x in p["roles"]) + rows.append(f"{e(p['id'])}predication{e(g(ch, p['concept']))} in frame {e(g(ch, p['frame']))}: {roles}") + for r in s["meaning"]["relations"]: + rows.append(f"relation{e(g(ch, r['relation']))}: {e(r['from'])} → {e(r['to'])}") + meaning = "" + "".join(rows) + "
nodekindcontent
" + d = s["discourse"] + drows = [] + if d["speech_act"]: + drows.append(f"speech act{e(g(ch, d['speech_act']))}") + if d["addressee"]: + drows.append(f"addressee{e(d['addressee'])}") + if d["addressee_consent"]: + drows.append(f"addressee consentpredication {e(d['addressee_consent'])} (what the addressee asked for)") + drows.append(f"source{e(s['provenance']['source'])}, kind {e(s['kind'])}") + return (f"

Slot: meaning

{meaning}

Slot: discourse

{''.join(drows)}
" + + stance_register(fx)) + + +def stance_register(fx): + s = fx["slots"] + eff = fx["outcome"].get("feature_effects") or [] + rows = [] + for i, f in enumerate(s["features"]): + note = eff[i] if i < len(eff) else "" + cls = "read" if "read it" in note else "unread" if "did not read" in note else "refuse" if "refus" in note else "" + rows.append(f"{e(f['gloss'])} {e(f['id'])}{e(f['slot'])} @ {e(f['at'])}{e(val(f['value']))}{e(note)}") + feats = ("" + "".join(rows) + "
featureslotvaluemeasured effect on the lowerer
") if rows else "

No features: no hedge, certainty, polarity, irony or stance set.

" + return ("

Slot: stance, hedge, certainty, restraint (features)

" + feats + + "

Register: the packet has no register slot. Register and address belong to a target-language profile, not to the packet; this lowerer reads none.

") + + +def outcome_html(fx): + o = fx["outcome"] + if o["kind"] == "lowered": + ipa = o["ipa"] + ipa_s = ("IPA of cited entities: " + e(" ".join(ipa["ok"])) + "") if "ok" in ipa else f"IPA refused: {e(ipa['error']['reason'])} {e(ipa['error']['variant'])}" + sc = [] + for s in o["scores"]: + ref = corpus[s["item"]]["text"] + sc.append(f"
{'Target' if s['target'] else 'Other reference'} {e(s['item'])}: {e(ref)}
" + f"" + f"
chrF {s['chrf']:.1f}
meaning cosine {s['meaning']:.3f}
") + return (f"

Lowered English

{e(o['text'])}

{ipa_s}

" + f"

Closeness to the corpus ({e(data['build']['embedder'])})

{''.join(sc)}") + if o["kind"] == "lower_error": + er = o["error"] + return f"

Refused by the lowerer

LowerError::{e(er['variant'])}: {e(er['reason'])}
{e(er['debug'])}

No text, so no score: nothing was invented to score.

" + items = "".join(f"
  • {e(r['rule'])} {e(r['detail'])}
    • " for r in o["refusals"]) + return f"

    Refused at admission

      {items}

    The packet never became a lowering input, so the lowerer never saw it.

    " + + +def card(fx): + o = fx["outcome"] + ch = "".join(f"{e(c['id'])}{e(c['gloss'] or '(unresolved)')}" for c in fx["chosen"]) + steps = [("1 Typed slots", "meaning, stance, discourse"), ("2 Inventory entries", f"{len(fx['chosen'])} ids, rev {fx['slots']['inventory_rev']}"), + ("3 Lowered English", {"lowered": "text", "lower_error": "refused", "admission_refused": "never reached"}[o["kind"]]), + ("4 Closeness", "scored" if o["kind"] == "lowered" else "no text, no score")] + chain = "".join(f"
    {e(a)}{e(b)}
    " for a, b in steps) + return (f"

    {e(fx['title'])}

    {e(fx['note'])}

    " + f"

    Packet {e(fx['packet_id'])} for {e(fx['corpus_item'])}. Made from: {e(fx['source'])}.

    " + f"
    {chain}
    {slots_html(fx)}" + f"
    Inventory entries chosen ({len(fx['chosen'])}){ch}
    idgloss at this revision
    " + f"{outcome_html(fx)}
    ") + + +nav = "".join(f"{e(f['title'])}" for f in data["fixtures"]) +b = data["build"] +page = f""" +Weksa: show her working +

    Show her working

    +

    A meaning packet goes in. The English lowerer picks inventory entries, writes a sentence, and a scorer measures how close that sentence is to a human-written target. Every number and sentence below was produced by running the real Rust lowerer and scorer when this page was built. The refusals are results too: when the lowerer cannot say a meaning without changing it, it says so instead of dropping the part it cannot say.

    +

    Built from {e(b['commit'][:10])}, embedder {e(b['embedder'])}. English only. Fixtures are the committed nibu-proof-001 packet and variants of it, each variant described in its own card. Scores are against the committed corpus: chrF (0-100) and embedding cosine. Raw data: data.json.

    + +{''.join(card(f) for f in data['fixtures'])} +""" +(out_dir / "index.html").write_text(page, encoding="utf-8") +print(f"wrote {out_dir}/index.html and data.json ({len(data['fixtures'])} fixtures)") From a5c6392b4e928c675d8a4a7b50d68eecf1fb2660 Mon Sep 17 00:00:00 2001 From: Weksa Date: Sun, 8 Nov 2026 01:02:20 +0000 Subject: [PATCH 005/175] Fixture: the phones line for Nibu's name (n, i primary, b, u) The expected `weksa lower --phones` output for the proof packet, committed before the code that must produce it. Co-Authored-By: Claude Sonnet 5.5 --- data/phonology/fixtures/nibu-en-US.phones.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 data/phonology/fixtures/nibu-en-US.phones.json diff --git a/data/phonology/fixtures/nibu-en-US.phones.json b/data/phonology/fixtures/nibu-en-US.phones.json new file mode 100644 index 0000000..45e2e11 --- /dev/null +++ b/data/phonology/fixtures/nibu-en-US.phones.json @@ -0,0 +1 @@ +{"language":"en-US","source":"ˈnibu","phones":[{"ipa":"n","stress":"none"},{"ipa":"i","stress":"primary"},{"ipa":"b","stress":"none"},{"ipa":"u","stress":"none"}]} From 5508000d213b29988f6930c67b1e606571bfbb69 Mon Sep 17 00:00:00 2001 From: Weksa Date: Sun, 8 Nov 2026 01:04:03 +0000 Subject: [PATCH 006/175] Phone inventory, segmenter and `weksa lower --phones` data/phonology/en-US.phones.tsv lists the phones (tense/lax a column); the segmenter splits ruled IPA into {ipa, stress} by longest inventory match and refuses a symbol it does not list (LowerError::UnknownPhone) or a stress mark with no vowel (DanglingStress). No durations, pitch or acoustics. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/lib.rs | 4 +- crates/weksa/src/lower.rs | 53 ++++++++++- crates/weksa/src/main.rs | 13 ++- crates/weksa/src/phones.rs | 152 ++++++++++++++++++++++++++++++++ crates/weksa/tests/cli.rs | 12 +++ crates/weksa/tests/lower.rs | 139 ++++++++++++++++++++++++++++- data/phonology/en-US.phones.tsv | 46 ++++++++++ 7 files changed, 414 insertions(+), 5 deletions(-) create mode 100644 crates/weksa/src/phones.rs create mode 100644 data/phonology/en-US.phones.tsv diff --git a/crates/weksa/src/lib.rs b/crates/weksa/src/lib.rs index 3f6b43c..a0ef6c6 100644 --- a/crates/weksa/src/lib.rs +++ b/crates/weksa/src/lib.rs @@ -9,6 +9,7 @@ pub mod form; pub mod inventory; pub mod lower; pub mod packet; +pub mod phones; pub mod schema; pub mod score; pub mod split; @@ -22,7 +23,8 @@ pub use inventory::{ Change, Family, InvId, Inventory, InventoryEntry, InventoryRevision, OptionLabel, OptionSet, Slot, ValueKind, }; -pub use lower::{EntityForms, LowerError, TARGET_LANGUAGE, lower, pronounce}; +pub use lower::{EntityForms, LowerError, PhoneLine, TARGET_LANGUAGE, lower, phones, pronounce}; +pub use phones::{Phone, PhoneInventory, SegmentError, Stress}; pub use packet::{ EncoderId, EntityId, Grade, NodeId, Packet, PacketId, PacketKind, SCHEMA_VERSION, Stance, }; diff --git a/crates/weksa/src/lower.rs b/crates/weksa/src/lower.rs index 43a8752..51f5a1f 100644 --- a/crates/weksa/src/lower.rs +++ b/crates/weksa/src/lower.rs @@ -23,9 +23,12 @@ use std::fmt; +use serde::Serialize; + use crate::entity::Entity; use crate::inventory::{Family, InvId, Inventory}; use crate::packet::{EntityId, NodeId, Predication}; +use crate::phones::{Phone, PhoneInventory, SegmentError}; use crate::store::EntityStore; use crate::validate::LoweringInput; @@ -95,6 +98,13 @@ pub enum LowerError { NoPronunciation { entity: EntityId }, /// IPA was requested for a packet that cites no entity. NoEntityReferent, + /// Phones were requested for an entity whose ruled IPA holds a symbol the + /// language's phone inventory does not list (or the language has no + /// inventory). The lowerer does not guess a phone. + UnknownPhone { entity: EntityId, at: usize, text: String }, + /// Phones were requested for an entity whose ruled IPA has a stress mark + /// with no vowel after it. + DanglingStress { entity: EntityId, at: usize }, /// The entity port failed to read. EntityRead { entity: EntityId, why: String }, } @@ -194,13 +204,54 @@ pub fn pronounce( input: &LoweringInput, entities: &dyn EntityForms, ) -> Result, LowerError> { + Ok(ruled(input, entities)?.into_iter().map(|(_, ipa)| ipa).collect()) +} + +/// One entity's ruled pronunciation as the phones line a sound consumer reads: +/// the language tag, the stored IPA it came from, and the segmented phones. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct PhoneLine { + pub language: &'static str, + pub source: String, + pub phones: Vec, +} + +/// The phones line of every entity a packet cites, in [`pronounce`] order, each +/// segmented by [`TARGET_LANGUAGE`]'s phone inventory. A symbol the inventory +/// does not list refuses the whole call. +pub fn phones( + input: &LoweringInput, + entities: &dyn EntityForms, +) -> Result, LowerError> { + let inventory = PhoneInventory::for_language(TARGET_LANGUAGE); + ruled(input, entities)? + .into_iter() + .map(|(entity, source)| { + let unknown = |at, text| LowerError::UnknownPhone { entity: entity.clone(), at, text }; + let inventory = inventory.ok_or_else(|| unknown(0, source.clone()))?; + let phones = inventory.segment(&source).map_err(|e| match e { + SegmentError::UnknownPhone { at, text } => unknown(at, text), + SegmentError::DanglingStress { at } => { + LowerError::DanglingStress { entity: entity.clone(), at } + } + })?; + Ok(PhoneLine { language: TARGET_LANGUAGE, source, phones }) + }) + .collect() +} + +/// `(entity, ruled IPA)` for each entity referent, in referent order. +fn ruled( + input: &LoweringInput, + entities: &dyn EntityForms, +) -> Result, LowerError> { let mut out = Vec::new(); for id in input.referents().iter().filter_map(|r| r.entity.as_ref()) { let entity = entity(entities, id)?; let p = entity .pronunciation_in(TARGET_LANGUAGE) .ok_or_else(|| LowerError::NoPronunciation { entity: id.clone() })?; - out.push(p.ipa.clone()); + out.push((id.clone(), p.ipa.clone())); } if out.is_empty() { return Err(LowerError::NoEntityReferent); diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 5518823..bc61c7e 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -53,8 +53,13 @@ enum Command { key: String, /// Print the ruled IPA of each entity the packet cites, one per line, /// instead of the text; refused for an entity with no ruled pronunciation. - #[arg(long)] + #[arg(long, conflicts_with = "phones")] ipa: bool, + /// Print the phones line of each entity the packet cites, one JSON + /// object per line ({language, source, phones: [{ipa, stress}]}), + /// instead of the text; refused for a phone the inventory lacks. + #[arg(long)] + phones: bool, }, /// Score output text (stdin, or --output) against corpus items, one embed call. Score(ScoreArgs), @@ -527,7 +532,7 @@ fn main() -> Result<()> { .map_err(refused)?; println!("admitted {key}"); } - Command::Lower { store, key, ipa } => { + Command::Lower { store, key, ipa, phones } => { let inventory_store = InventoryStore::open(&cli.inventory)?; let inventory = inventory_store.inventory()?; let mut packets = PacketStore::open(&store)?; @@ -549,6 +554,10 @@ fn main() -> Result<()> { for line in pronounce(&view, &entities)? { println!("{line}"); } + } else if phones { + for line in weksa::phones(&view, &entities)? { + println!("{}", serde_json::to_string(&line)?); + } } else { println!("{}", lower(&view, &inventory, &entities)?); } diff --git a/crates/weksa/src/phones.rs b/crates/weksa/src/phones.rs new file mode 100644 index 0000000..1477812 --- /dev/null +++ b/crates/weksa/src/phones.rs @@ -0,0 +1,152 @@ +//! The phone inventory and the segmenter: a ruled IPA string to an ordered list of +//! phones with stress, for the consumer that turns phones into sound. The +//! inventory is data (`data/phonology/.phones.tsv`); this module reads +//! it and owns one rule, that every symbol of the string is a phone the inventory +//! lists. A symbol it does not list is refused: no nearest phone, no spelling +//! rule, no guess. Tense/lax is a column of the inventory, so /i/ and /ɪ/ are two +//! phones here, not one phone and a flag on an entity. +//! +//! Stress marks are not phones. `ˈ` (primary) and `ËŒ` (secondary) put their +//! stress on the next vowel phone, the syllable's nucleus; every other phone is +//! `none`. A mark with no vowel after it is refused. Durations, pitch and +//! acoustics are the consumer's, never read or written here. + +use std::collections::BTreeMap; +use std::sync::OnceLock; + +use serde::Serialize; + +const EN_US: &str = include_str!("../../../data/phonology/en-US.phones.tsv"); + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Stress { + None, + Primary, + Secondary, +} + +/// One phone of a segmented string: its IPA symbol as the inventory spells it +/// and its stress. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct Phone { + pub ipa: String, + pub stress: Stress, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Tenseness { + Tense, + Lax, + Reduced, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Class { + Consonant, + Vowel(Tenseness), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SegmentError { + /// The symbol (or the text from it on) matches no inventory phone. + UnknownPhone { at: usize, text: String }, + /// A stress mark (`at` is its byte offset) that finds no vowel before the + /// string ends or another mark comes. + DanglingStress { at: usize }, +} + +pub struct PhoneInventory { + phones: BTreeMap, + longest: usize, +} + +impl PhoneInventory { + /// The inventory for a language tag, if one is committed. + pub fn for_language(language: &str) -> Option<&'static PhoneInventory> { + static EN_US_INV: OnceLock = OnceLock::new(); + match language { + "en-US" => Some(EN_US_INV.get_or_init(|| { + PhoneInventory::parse(EN_US).expect("the committed en-US phone inventory parses") + })), + _ => None, + } + } + + /// Parse the tab-separated inventory. Malformed lines are errors, as are + /// repeated symbols. + pub fn parse(text: &str) -> Result { + let mut phones = BTreeMap::new(); + for (n, line) in text.lines().enumerate() { + if line.is_empty() || line.starts_with('#') { + continue; + } + let cols: Vec<&str> = line.split('\t').collect(); + let [ipa, class, tenseness, gloss] = cols[..] else { + return Err(format!("line {}: want 4 tab-separated columns", n + 1)); + }; + let class = match (class, tenseness) { + ("consonant", "-") => Class::Consonant, + ("vowel", "tense") => Class::Vowel(Tenseness::Tense), + ("vowel", "lax") => Class::Vowel(Tenseness::Lax), + ("vowel", "reduced") => Class::Vowel(Tenseness::Reduced), + _ => return Err(format!("line {}: bad class/tenseness {class}/{tenseness}", n + 1)), + }; + if ipa.is_empty() || gloss.is_empty() || ipa.chars().any(|c| matches!(c, 'ˈ' | 'ËŒ')) { + return Err(format!("line {}: empty or stress-marked entry", n + 1)); + } + if phones.insert(ipa.to_string(), class).is_some() { + return Err(format!("line {}: {ipa} repeats", n + 1)); + } + } + let longest = phones.keys().map(|k| k.chars().count()).max().unwrap_or(0); + Ok(Self { phones, longest }) + } + + pub fn class(&self, ipa: &str) -> Option<&Class> { + self.phones.get(ipa) + } + + /// Segment `ipa` into phones: longest inventory match at each position. + pub fn segment(&self, ipa: &str) -> Result, SegmentError> { + let chars: Vec<(usize, char)> = ipa.char_indices().collect(); + let mut out: Vec = Vec::new(); + let mut pending: Option<(Stress, usize)> = None; + let mut i = 0; + while i < chars.len() { + let at = chars[i].0; + let mark = match chars[i].1 { + 'ˈ' => Some(Stress::Primary), + 'ËŒ' => Some(Stress::Secondary), + _ => None, + }; + if let Some(m) = mark { + if let Some((_, was)) = pending { + return Err(SegmentError::DanglingStress { at: was }); + } + pending = Some((m, at)); + i += 1; + continue; + } + let hit = (1..=self.longest.min(chars.len() - i)).rev().find_map(|len| { + let end = chars.get(i + len).map_or(ipa.len(), |c| c.0); + let s = &ipa[at..end]; + self.phones.get(s).map(|c| (len, s, c)) + }); + let Some((len, s, class)) = hit else { + return Err(SegmentError::UnknownPhone { at, text: ipa[at..].to_string() }); + }; + let stress = if matches!(class, Class::Vowel(_)) { + pending.take().map_or(Stress::None, |(m, _)| m) + } else { + Stress::None + }; + out.push(Phone { ipa: s.to_string(), stress }); + i += len; + } + if let Some((_, at)) = pending { + return Err(SegmentError::DanglingStress { at }); + } + Ok(out) + } +} diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 72ad140..b2e5b58 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -876,6 +876,18 @@ fn proof_packet_lowers_and_the_lowering_scores_against_items() { let ipa = weksa(d.path(), &args); assert!(ipa.status.success(), "{}", out(&ipa)); assert_eq!(String::from_utf8(ipa.stdout).unwrap(), "ˈnibu\n"); + // The phones line is the committed fixture, byte for byte, and is a different + // output from --ipa, so the two flags cannot be combined. + let phones = weksa(d.path(), &["lower", "./p.cc", "packet:6e6962752d303031", "--phones"]); + assert!(phones.status.success(), "{}", out(&phones)); + let fixture = std::fs::read_to_string( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../data/phonology/fixtures/nibu-en-US.phones.json"), + ) + .unwrap(); + assert_eq!(String::from_utf8(phones.stdout).unwrap(), fixture); + let both = weksa(d.path(), &["lower", "./p.cc", "packet:6e6962752d303031", "--phones", "--ipa"]); + assert!(!both.status.success(), "{}", out(&both)); let again = weksa(d.path(), &["lower", "./p.cc", "packet:6e6962752d303031"]); assert_eq!(String::from_utf8(again.stdout).unwrap(), text); diff --git a/crates/weksa/tests/lower.rs b/crates/weksa/tests/lower.rs index 4f1691c..342b25d 100644 --- a/crates/weksa/tests/lower.rs +++ b/crates/weksa/tests/lower.rs @@ -11,7 +11,7 @@ use weksa::inventory::proof_revision; use weksa::packet::{EntityId, FeatValue, FeatureValue, Packet, proof_packet}; use weksa::{ CorpusStore, EntityForms, EntityStore, Inventory, InventoryStore, LowerError, LoweringInput, - PacketStore, Pronunciation, lower, pronounce, validate, + PacketStore, PhoneInventory, PhoneLine, Pronunciation, Stress, lower, pronounce, validate, }; const PROOF: &str = "Warning: you may endanger yourself by touching the control. \ @@ -434,3 +434,140 @@ fn ipa_needs_an_entity_the_port_holds_and_a_packet_that_cites_one() { Err(LowerError::NoEntityReferent) ); } + +// ---- the phones line: the phone inventory and the segmenter ---- + +fn phones_of(ipa: &str) -> Result, LowerError> { + let mut n = nibu(&[("en-US", &["Nibu"])]); + n.forms[0].pronunciation = Some(ruled(ipa, &[])); + let mut e = Entities::default(); + e.held.push(n); + let (inv, _dir) = committed(); + weksa::phones(&view(&proof_packet(), &inv), &e) +} + +fn seg(ipa: &str) -> Vec<(String, Stress)> { + let inv = PhoneInventory::for_language("en-US").unwrap(); + inv.segment(ipa) + .unwrap() + .into_iter() + .map(|p| (p.ipa, p.stress)) + .collect() +} + +fn pairs(v: &[(&str, Stress)]) -> Vec<(String, Stress)> { + v.iter().map(|(i, s)| ((*i).to_string(), *s)).collect() +} + +#[test] +fn nibus_committed_ruling_lowers_to_the_committed_phones_fixture() { + let (inv, _dir) = committed(); + let lines = weksa::phones(&view(&proof_packet(), &inv), &committed_entities()).unwrap(); + let fixture = std::fs::read_to_string( + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../data/phonology/fixtures/nibu-en-US.phones.json"), + ) + .unwrap(); + assert_eq!(lines.len(), 1); + assert_eq!(serde_json::to_string(&lines[0]).unwrap(), fixture.trim_end()); +} + +#[test] +fn tense_i_is_a_phone_and_the_rejected_lax_form_is_a_different_one() { + use weksa::phones::{Class, Tenseness}; + let inv = PhoneInventory::for_language("en-US").unwrap(); + assert_eq!(inv.class("i"), Some(&Class::Vowel(Tenseness::Tense))); + assert_eq!(inv.class("ɪ"), Some(&Class::Vowel(Tenseness::Lax))); + assert_eq!(inv.class("u"), Some(&Class::Vowel(Tenseness::Tense))); + let ruled_form = seg("ˈnibu"); + let lax = seg("ˈnɪbu"); + assert_ne!(ruled_form, lax); + assert_eq!(lax[1], ("ɪ".to_string(), Stress::Primary)); + // The committed seed rejects the lax form, and it is not what the seed emits. + let seed = committed_entities(); + let nibu = seed.get(&EntityId::parse("entity:nibu").unwrap()).unwrap().unwrap(); + let p = nibu.pronunciation_in("en-US").unwrap(); + assert_eq!(p.rejected, ["ˈnɪbu"]); + assert_eq!(seg(&p.ipa), ruled_form); + assert!(p.rejected.iter().all(|r| seg(r) != seg(&p.ipa))); +} + +#[test] +fn stress_lands_on_the_vowel_after_the_mark_and_nowhere_else() { + use Stress::{None as N, Primary as P, Secondary as S}; + assert_eq!(seg("ˈnibu"), pairs(&[("n", N), ("i", P), ("b", N), ("u", N)])); + assert_eq!(seg("nˈibu"), seg("ˈnibu")); + assert_eq!(seg("bˈunËŒi"), pairs(&[("b", N), ("u", P), ("n", N), ("i", S)])); + assert_eq!(seg("nibu"), pairs(&[("n", N), ("i", N), ("b", N), ("u", N)])); +} + +#[test] +fn multi_character_phones_are_one_phone_by_longest_match() { + use Stress::{None as N, Primary as P}; + assert_eq!(seg("ˈtʃeɪ"), pairs(&[("tʃ", N), ("eɪ", P)])); + assert_eq!(seg("dÊ’aÊŠ"), pairs(&[("dÊ’", N), ("aÊŠ", N)])); + // /t/ then /ʃ/ is not what a lone "t" and "ʃ" mean when the affricate is listed, + // but a symbol sequence the inventory cannot join still splits. + assert_eq!(seg("ɔɪt"), pairs(&[("ɔɪ", N), ("t", N)])); +} + +#[test] +fn a_phone_the_inventory_lacks_is_refused_with_where_and_never_guessed() { + let id = EntityId::parse("entity:nibu").unwrap(); + for (ipa, at, text) in [ + ("ˈnixu", "ˈni".len(), "xu"), + ("ˈniËbu", "ˈni".len(), "Ëbu"), + ("ˈnibu.", "ˈnibu".len(), "."), + ("ˈni bu", "ˈni".len(), " bu"), + ] { + assert_eq!( + phones_of(ipa), + Err(LowerError::UnknownPhone { entity: id.clone(), at, text: text.into() }), + "{ipa}" + ); + } +} + +#[test] +fn a_stress_mark_with_no_vowel_to_land_on_is_refused() { + let id = EntityId::parse("entity:nibu").unwrap(); + for (ipa, at) in [("nibuˈ", "nibu".len()), ("ˈˈnibu", 0), ("ˈnˈibu", 0), ("ˈnb", 0)] { + assert_eq!( + phones_of(ipa), + Err(LowerError::DanglingStress { entity: id.clone(), at }), + "{ipa}" + ); + } +} + +#[test] +fn phones_need_a_ruling_and_an_entity_exactly_as_ipa_does() { + assert_eq!( + weksa::phones(&view(&proof_packet(), &committed().0), &Entities::default()), + Err(LowerError::UnknownEntity { entity: EntityId::parse("entity:nibu").unwrap() }) + ); + let mut e = Entities::default(); + e.held.push(nibu(&[("en-US", &["Nibu"])])); + assert_eq!( + weksa::phones(&view(&proof_packet(), &committed().0), &e), + Err(LowerError::NoPronunciation { entity: EntityId::parse("entity:nibu").unwrap() }) + ); +} + +#[test] +fn the_inventory_parser_refuses_what_it_cannot_read() { + for bad in [ + "i\tvowel\ttense", + "i\tvowel\t-\tx", + "n\tconsonant\ttense\tx", + "i\tvowel\tmid\tx", + "ˈi\tvowel\ttense\tx", + "\tvowel\ttense\tx", + "i\tvowel\ttense\t", + "i\tvowel\ttense\ta\ni\tvowel\tlax\tb", + ] { + assert!(PhoneInventory::parse(bad).is_err(), "{bad:?}"); + } + assert!(PhoneInventory::parse("# c\n\ni\tvowel\ttense\ta\n").is_ok()); + assert!(PhoneInventory::for_language("sa-Latn").is_none()); +} diff --git a/data/phonology/en-US.phones.tsv b/data/phonology/en-US.phones.tsv new file mode 100644 index 0000000..b607e28 --- /dev/null +++ b/data/phonology/en-US.phones.tsv @@ -0,0 +1,46 @@ +# Weksa phone inventory, en-US (General American, broad). One phone per line, tab separated: +# ipa class tenseness gloss +# class: consonant | vowel. tenseness: tense | lax | reduced for a vowel, `-` for a consonant. A multi-character +# phone (affricate, diphthong) is one entry and one phone. The segmenter reads only this file: an IPA symbol with no +# line here is refused, never guessed. Tense/lax is the textbook classification; it is what separates /i/ (nee) from +# /ɪ/ (nih). Stress is not a phone: ˈ and ËŒ are read by the segmenter and put on the vowel that follows. +p consonant - voiceless bilabial stop +b consonant - voiced bilabial stop +t consonant - voiceless alveolar stop +d consonant - voiced alveolar stop +k consonant - voiceless velar stop +g consonant - voiced velar stop +f consonant - voiceless labiodental fricative +v consonant - voiced labiodental fricative +θ consonant - voiceless dental fricative +ð consonant - voiced dental fricative +s consonant - voiceless alveolar fricative +z consonant - voiced alveolar fricative +ʃ consonant - voiceless postalveolar fricative +Ê’ consonant - voiced postalveolar fricative +h consonant - voiceless glottal fricative +tʃ consonant - voiceless postalveolar affricate +dÊ’ consonant - voiced postalveolar affricate +m consonant - bilabial nasal +n consonant - alveolar nasal +Å‹ consonant - velar nasal +l consonant - alveolar lateral approximant +ɹ consonant - alveolar approximant +j consonant - palatal approximant +w consonant - labial-velar approximant +i vowel tense close front unrounded (fleece, nee) +ɪ vowel lax near-close front unrounded (kit, nih) +É› vowel lax open-mid front unrounded (dress) +æ vowel lax near-open front unrounded (trap) +É‘ vowel tense open back unrounded (palm) +É” vowel tense open-mid back rounded (thought) +ÊŠ vowel lax near-close back rounded (foot) +u vowel tense close back rounded (goose, boo) +ÊŒ vowel lax open-mid back unrounded (strut) +É™ vowel reduced schwa +Éš vowel reduced r-coloured schwa +eɪ vowel tense face diphthong +oÊŠ vowel tense goat diphthong +aɪ vowel tense price diphthong +aÊŠ vowel tense mouth diphthong +ɔɪ vowel tense choice diphthong From a7bc2478e53623ce572ac6d997ba058aa7707e44 Mon Sep 17 00:00:00 2001 From: Weksa Date: Sun, 8 Nov 2026 01:10:09 +0000 Subject: [PATCH 007/175] =?UTF-8?q?Phone=20inventory:=20name=20/i/=20tense?= =?UTF-8?q?,=20/=C9=AA/=20lax=20and=20cite=20Nibu's=202026-11-07=20ruling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 5.5 --- data/phonology/en-US.phones.tsv | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/data/phonology/en-US.phones.tsv b/data/phonology/en-US.phones.tsv index b607e28..0730256 100644 --- a/data/phonology/en-US.phones.tsv +++ b/data/phonology/en-US.phones.tsv @@ -3,7 +3,11 @@ # class: consonant | vowel. tenseness: tense | lax | reduced for a vowel, `-` for a consonant. A multi-character # phone (affricate, diphthong) is one entry and one phone. The segmenter reads only this file: an IPA symbol with no # line here is refused, never guessed. Tense/lax is the textbook classification; it is what separates /i/ (nee) from -# /ɪ/ (nih). Stress is not a phone: ˈ and ËŒ are read by the segmenter and put on the vowel that follows. +# /ɪ/ (nih). +# Provenance of the i/ɪ rows: /i/ is tense and /ɪ/ is lax. Nibu's own ruling of 2026-11-07 (source +# weksa-body:follow_up:chat-s3981-3983, #minds: tense vowel, two syllables, stress on the first) chose ˈnibu and rejected +# ˈnɪbu; the entity entry records it, this file is what makes the two forms two different phones. +# Stress is not a phone: ˈ and ËŒ are read by the segmenter and put on the vowel that follows. p consonant - voiceless bilabial stop b consonant - voiced bilabial stop t consonant - voiceless alveolar stop From 5e2a4a0ebbe67d7f29dea37abe0f2070f8e54bb7 Mon Sep 17 00:00:00 2001 From: Weksa Date: Mon, 9 Nov 2026 07:47:17 +0000 Subject: [PATCH 008/175] Scorer coverage: words break at any non-word character, unspaced scripts read by run; near-miss chrF pairs per script Soul scorer s3 Lows. words.rs no longer lists separators (_ and .) or defers to UAX #29, which splits Han, kana and Thai per character and made honest single-word glosses read as an n>=2 leak. A word is a run of letters, digits, marks and ZWNJ/ZWJ; a run of Han, kana, Thai, Lao, Khmer or Burmese is one word. Tests pin separators, marks, joiners, NFC, every range edge, and five sacrebleu 2.4.3 near-miss pairs (Ko, Tr, Fi, Th vowel-strip, Ja). 40 hand mutants killed. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/score/words.rs | 106 +++++++++++----- crates/weksa/tests/score.rs | 152 +++++++++++++++++++++++ docs/targets/interlingua-lowering-map.md | 9 +- 3 files changed, 236 insertions(+), 31 deletions(-) diff --git a/crates/weksa/src/score/words.rs b/crates/weksa/src/score/words.rs index 58e7c1f..4909ee9 100644 --- a/crates/weksa/src/score/words.rs +++ b/crates/weksa/src/score/words.rs @@ -1,44 +1,94 @@ -//! The one way the scorer reads words: UAX #29 word segmentation, then NFC and -//! full Unicode case folding. The novel-coverage diagnostic reads text through -//! here, so every script is treated by one rule. +//! The one way the scorer reads words, then NFC and full Unicode case folding. +//! The novel-coverage diagnostic reads text through here, so every script is +//! treated by one rule. +//! +//! A word is a run of word characters: alphanumerics, combining marks and the +//! joiners ZWNJ and ZWJ. Any other character ends it, so `kick:the:bucket`, +//! `kick·the·bucket`, `kick_the_bucket` and `kick'the'bucket` read as +//! `kick the bucket`, as does `kickTheBucket` (a lowercase-to-uppercase +//! transition also ends a word). +//! +//! Han, kana, Thai, Lao, Khmer and Burmese write no word boundaries, and the +//! scorer holds no dictionary to find them. A run of those characters is one +//! word, never one word per character: per-character words would make every +//! honest single-word gloss share bigrams with a held-out sentence, which is +//! the leak signature. A change between such a run and any other script also +//! ends a word. A leak in these scripts shows at n = 1 and in chrF; a +//! dictionary segmenter is the ja-JP panel's to add. use caseless::default_case_fold_str; use unicode_normalization::UnicodeNormalization; -use unicode_segmentation::UnicodeSegmentation; +use unicode_normalization::char::is_combining_mark; /// NFC, then full default case folding. pub(super) fn normalise(s: &str) -> String { default_case_fold_str(&s.nfc().collect::()) } -/// The pieces of one segment: split at `_` and `.` (which are dropped) and -/// at every lowercase-to-uppercase transition, so `kick_the_bucket`, -/// `kick.the.bucket` and `kickTheBucket` read as `kick the bucket`. -fn pieces(segment: &str) -> Vec<&str> { +fn is_joiner(c: char) -> bool { + matches!(c, '\u{200c}' | '\u{200d}') +} + +/// A character that belongs inside a word. Marks must stay: Thai vowels and +/// tone marks, Hindi vowel signs and Hangul jamo are not alphanumeric. +fn is_word_char(c: char) -> bool { + c.is_alphanumeric() || is_combining_mark(c) || is_joiner(c) +} + +/// Han, kana, Thai, Lao, Khmer and Burmese: scripts written without spaces. +fn is_unspaced(c: char) -> bool { + matches!( + c, + '\u{3005}' + | '\u{3007}' + | '\u{3021}'..='\u{3029}' + | '\u{3038}'..='\u{303b}' + | '\u{3041}'..='\u{30ff}' + | '\u{3400}'..='\u{4dbf}' + | '\u{4e00}'..='\u{9fff}' + | '\u{f900}'..='\u{fad9}' + | '\u{ff66}'..='\u{ff9f}' + | '\u{31f0}'..='\u{31ff}' + | '\u{20000}'..='\u{323af}' + | '\u{0e01}'..='\u{0e5b}' + | '\u{0e81}'..='\u{0edf}' + | '\u{1000}'..='\u{109f}' + | '\u{1780}'..='\u{17ff}' + ) +} + +/// The normalised words of `text` in order; runs with no alphanumeric char +/// (marks alone, say) are dropped. +pub(super) fn words(text: &str) -> Vec { let mut out = Vec::new(); let mut start = 0; let mut previous: Option = None; - for (at, c) in segment.char_indices() { - let separator = c == '_' || c == '.'; - let camel = previous.is_some_and(char::is_lowercase) && c.is_uppercase(); - if separator || camel { - out.push(&segment[start..at]); - start = if separator { at + c.len_utf8() } else { at }; + let end = |from: usize, to: usize, out: &mut Vec| { + let word = &text[from..to]; + if word.chars().any(char::is_alphanumeric) { + out.push(normalise(word)); + } + }; + for (at, c) in text.char_indices() { + if !is_word_char(c) { + end(start, at, &mut out); + start = at + c.len_utf8(); + continue; + } + // a mark or joiner belongs to the character before it + let attached = is_combining_mark(c) || is_joiner(c); + let breaks = !attached + && previous.is_some_and(|p| { + (p.is_lowercase() && c.is_uppercase()) || is_unspaced(p) != is_unspaced(c) + }); + if breaks { + end(start, at, &mut out); + start = at; + } + if !attached { + previous = Some(c); } - previous = Some(c); } - out.push(&segment[start..]); + end(start, text.len(), &mut out); out } - -/// The normalised words of `text` in order: UAX #29 segments containing an -/// alphanumeric char (digits kept), each split by `pieces`, empty or -/// non-alphanumeric pieces dropped. -pub(super) fn words(text: &str) -> Vec { - text.split_word_bounds() - .filter(|segment| segment.chars().any(char::is_alphanumeric)) - .flat_map(pieces) - .filter(|piece| piece.chars().any(char::is_alphanumeric)) - .map(normalise) - .collect() -} diff --git a/crates/weksa/tests/score.rs b/crates/weksa/tests/score.rs index 6920b93..e8c363a 100644 --- a/crates/weksa/tests/score.rs +++ b/crates/weksa/tests/score.rs @@ -690,3 +690,155 @@ fn every_script_alike() { assert_eq!(got.chrf, 100.0, "{text}"); } } + +/// (reference, hypothesis, sacrebleu 2.4.3 sentence_chrf), one near miss per +/// script: the Thai hypothesis is the reference with its vowels and tone mark +/// stripped. +const SCRIPT_PAIRS: [(&str, &str, f64); 5] = [ + ("민수가 왔다", "민수는 왔다", 26.0), + ("Brokkr'ın gemisi geldi", "Brokkr'ın gemisi gitti", 77.7548), + ("Brokkrin laiva tuli", "Brokkrin laiva meni", 73.002), + ("สมชายมาà¹à¸¥à¹‰à¸§", "สมชยมà¹à¸¥à¸§", 21.8989), + ("ブロッカーãŒæ¥ãŸã€‚", "ブロッカーãŒè¡Œã£ãŸã€‚", 54.4013), +]; + +#[test] +fn every_script_scores_a_near_miss_as_sacrebleu_does() { + let scorer = Scorer::new(Bag); + for (reference, hypothesis, want) in SCRIPT_PAIRS { + close(score_one(&scorer, reference, hypothesis).chrf, want); + } +} + +#[test] +fn coverage_splits_at_any_non_word_character() { + let items = [ + validation("item:v:1", "mind the hatch before the storm"), + train("item:t:1", UNRELATED), + ]; + let want = read(&["mind the hatch"], &items); + assert_eq!(want, [(5, 3), (5, 2), (4, 1)]); + for joiner in [ + ":", "\u{b7}", "'", "\u{2019}", "\u{ff3f}", "-", "/", "\u{2014}", "|", "+", "\u{ff1a}", + "\u{30fb}", + ] { + let glued = format!("mind{joiner}the{joiner}hatch"); + assert_eq!(read(&[&glued], &items), want, "{joiner:?}"); + } + // a mark, a joiner or a digit is part of its word, not a break + let items = [validation( + "item:v:1", + "\u{e2a}\u{e21}\u{e0a}\u{e32}\u{e22} \u{e21}\u{e32}\u{e41}\u{e25}\u{e49}\u{e27} 0400 \u{628}\u{200c}\u{628}", + )]; + assert_eq!( + read(&["\u{e21}\u{e32}\u{e41}\u{e25}\u{e49}\u{e27}"], &items)[0], + (4, 1) + ); + assert_eq!(read(&["\u{628}\u{200c}\u{628}", "0400"], &items)[0], (4, 2)); + // a joiner keeps its word whole: the minted half is not a held-out word + let items = [validation("item:v:1", "\u{628}\u{200c}\u{62a} \u{62c}")]; + assert_eq!(read(&["\u{628}\u{200c}\u{62a}"], &items)[0], (2, 1)); + // decomposed and composed spellings are one word + let items = [validation("item:v:1", "Cafe\u{301}")]; + assert_eq!(read(&["Caf\u{e9}"], &items)[0], (1, 1)); +} + +#[test] +fn coverage_reads_unspaced_scripts_by_run() { + // Han, kana and Thai write no word boundaries and the scorer holds no + // dictionary, so a run of them is one word, never one word per character. + // Honest single-word glosses then read nothing above n=1, as they do in a + // spaced script. + for (held, glosses) in [ + ( + "\u{5c71}\u{7530}\u{592a}\u{90ce}\u{304c}\u{6765}\u{305f}", + ["\u{5c71}\u{7530}", "\u{592a}\u{90ce}", "\u{6765}\u{305f}"], + ), + ( + "\u{30d6}\u{30ed}\u{30c3}\u{30ab}\u{30fc}\u{304c}\u{6765}\u{305f}\u{3002}", + ["\u{30d6}\u{30ed}", "\u{30ab}\u{30fc}", "\u{6765}"], + ), + ( + "\u{e2a}\u{e21}\u{e0a}\u{e32}\u{e22}\u{e21}\u{e32}\u{e41}\u{e25}\u{e49}\u{e27}", + [ + "\u{e2a}\u{e21}", + "\u{e0a}\u{e32}\u{e22}", + "\u{e41}\u{e25}\u{e49}", + ], + ), + ] { + let items = [validation("item:v:1", held), train("item:t:1", UNRELATED)]; + assert_eq!(read(&glosses, &items), [(1, 0), (0, 0), (0, 0)], "{held}"); + assert_eq!(read(&[held], &items), [(1, 1), (0, 0), (0, 0)], "{held}"); + } + // Delimiters still make words: spaces and punctuation between runs, and a + // change between an unspaced script and a spaced one. + let items = [ + validation( + "item:v:1", + "\u{72ac}\u{304c} \u{8d70}\u{308b}\u{3001}\u{732b}\u{3082} Brokkr\u{5c71}", + ), + train("item:t:1", UNRELATED), + ]; + assert_eq!( + read( + &["\u{72ac}\u{304c} \u{8d70}\u{308b} \u{732b}\u{3082} brokkr \u{5c71}"], + &items + ), + [(5, 5), (4, 4), (3, 3)] + ); +} + +#[test] +fn an_unspaced_script_ends_a_word_beside_any_other_script() { + // the first and last letters or digits of every range the rule names + let unspaced = [ + '\u{3005}', + '\u{3007}', + '\u{3021}', + '\u{3029}', + '\u{3038}', + '\u{303b}', + '\u{3041}', + '\u{30ff}', + '\u{31f0}', + '\u{31ff}', + '\u{3400}', + '\u{4dbf}', + '\u{4e00}', + '\u{9fff}', + '\u{f900}', + '\u{fad9}', + '\u{ff66}', + '\u{ff9f}', + '\u{20000}', + '\u{2a6df}', + '\u{31350}', + '\u{323af}', + '\u{e01}', + '\u{e59}', + '\u{e81}', + '\u{edf}', + '\u{1000}', + '\u{1049}', + '\u{1780}', + '\u{17e9}', + ]; + for c in unspaced { + assert!(c.is_alphanumeric(), "{c:?} is a letter or digit"); + let items = [validation("item:v:1", &format!("ab{c}"))]; + assert_eq!(read(&["ab", &c.to_string()], &items)[0], (2, 2), "{c:?}"); + } + // Hangul, Latin and Cyrillic are spaced scripts: no break between them + for c in ['\u{ac00}', '\u{d7a3}', '\u{e9}', '\u{44f}', '7'] { + let glued = format!("ab{c}"); + let items = [validation("item:v:1", &glued)]; + assert_eq!(read(&[&glued], &items)[0], (1, 1), "{c:?}"); + assert_eq!(read(&["ab"], &items)[0], (1, 0), "{c:?}"); + } + // a combining mark or a joiner after an unspaced character stays on it + let held = "\u{4e00}\u{301}\u{200d}ab"; + let items = [validation("item:v:1", held)]; + assert_eq!(read(&["\u{4e00}\u{301}\u{200d}", "ab"], &items)[0], (2, 2)); + assert_eq!(read(&["\u{4e00}"], &items)[0], (2, 0)); +} diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index c828266..4517320 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -202,9 +202,12 @@ operator-attested annotations alone. Who annotates is question text against held-out items; novel coverage pooled per order, for n = 1, 2 and 3 separately, the covered share of all novel n-grams (word n-grams of a held-out item that occur in no Train item) over a split's items, with the - worst item at n >= 2. Keys split on `_`, `.` and lower-to-upper case - transitions as well as UAX #29 boundaries, so a snake_case `OptionLabel` - reads as words. Honest glosses of single words saturate n = 1 (the 54 seed + worst item at n >= 2. A word is a run of letters, digits, marks and + joiners; any other character, and a lower-to-upper case transition, ends it, + so `kick:the:bucket` and a snake_case `OptionLabel` read as words. Han, kana, + Thai, Lao, Khmer and Burmese write no word boundaries and the scorer holds no + dictionary, so a run of them is one word (per-character words would give + every honest gloss shared bigrams); a leak there shows at n = 1 and in chrF. Honest glosses of single words saturate n = 1 (the 54 seed glosses read max 1.000, s2), so the leak signature is pooled n >= 2. Costs: a live persona mints before it names (one typed write, which also buys coreference and one rendering per language); public-domain intake From 0bf9499ac0bb3477193098f499c0ba856034aab7 Mon Sep 17 00:00:00 2001 From: Weksa Date: Mon, 9 Nov 2026 10:07:52 +0000 Subject: [PATCH 009/175] Lower 'You asked Nibu to stop.' behind addressee consent: request concept and frame:3, consent gate in the lowerer Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/inventory.rs | 15 ++- crates/weksa/src/lower.rs | 80 +++++++++++++++- crates/weksa/tests/lower.rs | 114 +++++++++++++++++++++++ data/interlingua/inventory.cc | Bin 7772 -> 7967 bytes docs/interlingua-standard.md | 7 +- docs/targets/interlingua-lowering-map.md | 2 +- 6 files changed, 209 insertions(+), 9 deletions(-) diff --git a/crates/weksa/src/inventory.rs b/crates/weksa/src/inventory.rs index 405b4ae..c9d69a2 100644 --- a/crates/weksa/src/inventory.rs +++ b/crates/weksa/src/inventory.rs @@ -485,9 +485,20 @@ pub fn proof_revision() -> (InventoryRevision, Vec) { for (family, glosses) in [ ( Family::Concept, - ["endanger", "prevent", "player", "control", "ship_mind"].as_slice(), + [ + "endanger", + "prevent", + "player", + "control", + "ship_mind", + "request", + ] + .as_slice(), + ), + ( + Family::Frame, + ["hazard_warning", "intervention", "request_intervention"].as_slice(), ), - (Family::Frame, ["hazard_warning", "intervention"].as_slice()), ] { for (i, gloss) in glosses.iter().enumerate() { entries.push(InventoryEntry { diff --git a/crates/weksa/src/lower.rs b/crates/weksa/src/lower.rs index 8f7b856..9eb9ec7 100644 --- a/crates/weksa/src/lower.rs +++ b/crates/weksa/src/lower.rs @@ -17,6 +17,14 @@ //! the target language, is refused: the concept word would say a different //! thing than the name the packet cites. //! +//! A request clause ("You asked Nibu to stop.") says what the addressee did, so +//! it is gated on the packet's own record of it: `Discourse.addressee_consent` +//! (set only by authoring or game telemetry, V15) must name that predication, +//! and its agent must be the addressee. Without it the lowerer refuses +//! ([`LowerError::ConsentMissing`], [`LowerError::AskerNotAddressee`]); it does +//! not read wording or guess who asked. A consent naming any other predication +//! is an unrendered slot. +//! //! The table is a temporary owner. Each row speaks for an inventory entry and //! names the gloss it speaks for, so a row cannot outlive a change of what its id //! means. `language-ontology` moves the rows into ontology documents. @@ -65,8 +73,17 @@ const FRAMES: &[Row] = &[ "{agent} may endanger {patient} by touching {instrument}", ), ("frame:2", "intervention", "{agent} will stop {patient}"), + ( + "frame:3", + "request_intervention", + "{agent} asked {patient} to stop", + ), ]; +/// The frame of "{agent} asked {patient} to stop": the one frame that reads +/// the discourse's addressee consent. +const REQUEST_FRAME: &str = "frame:3"; + /// Speech acts: the opener of the first sentence. const SPEECH_ACTS: &[Row] = &[ ("speech_act:1", "assert", ""), @@ -95,6 +112,12 @@ pub enum LowerError { NoPronunciation { entity: EntityId }, /// IPA was requested for a packet that cites no entity. NoEntityReferent, + /// A request predication the packet does not carry the addressee's consent + /// to: no consent, or a consent that names another predication. The + /// lowerer does not guess who asked. + ConsentMissing { predication: NodeId }, + /// A request predication whose agent is not the discourse addressee. + AskerNotAddressee { predication: NodeId }, /// The entity port failed to read. EntityRead { entity: EntityId, why: String }, } @@ -139,9 +162,7 @@ pub fn lower( { return unrendered("features"); } - if input.discourse().addressee_consent.is_some() { - return unrendered("addressee consent"); - } + request_gate(input, inv)?; if !input.relations().is_empty() { return unrendered("relations"); } @@ -188,6 +209,59 @@ pub fn lower( Ok(out) } +/// The one place consent is read. A request clause ([`REQUEST_FRAME`]) says what +/// the addressee did, so it needs the packet's own record of it: the discourse +/// `addressee_consent`, which validation lets only authoring or game telemetry +/// set (V15), must name that predication, and its agent must be the addressee. +/// Nothing here reads wording. A consent naming any other predication is a slot +/// the lowerer does not render. +fn request_gate(input: &LoweringInput, inv: &Inventory) -> Result<(), LowerError> { + let consent = input.discourse().addressee_consent.as_ref(); + let requests = input + .predications() + .iter() + .filter(|p| p.frame.to_string() == REQUEST_FRAME); + for p in requests { + if consent != Some(&p.id) { + return Err(LowerError::ConsentMissing { + predication: p.id.clone(), + }); + } + let agent = inv + .entries() + .find(|e| { + e.family() == Family::Role && e.gloss == "agent" && e.live_at(input.inventory_rev()) + }) + .ok_or_else(|| LowerError::UnknownRole { + frame: p.frame, + role: "agent".to_string(), + })?; + let asker = p + .roles + .iter() + .find(|r| r.role == agent.id) + .map(|r| &r.filler); + if asker.is_none() || asker != input.discourse().addressee.as_ref() { + return Err(LowerError::AskerNotAddressee { + predication: p.id.clone(), + }); + } + } + match consent { + Some(_) + if !input + .predications() + .iter() + .any(|p| p.frame.to_string() == REQUEST_FRAME) => + { + Err(LowerError::Unrendered { + what: "addressee consent", + }) + } + _ => Ok(()), + } +} + /// The ruled IPA of every entity a packet cites, one per entity referent in /// the packet's referent order, as stored: bare, stress-marked, nothing /// derived. A rejected pronunciation is never read. An entity the port does not diff --git a/crates/weksa/tests/lower.rs b/crates/weksa/tests/lower.rs index 1d8301d..89e35ed 100644 --- a/crates/weksa/tests/lower.rs +++ b/crates/weksa/tests/lower.rs @@ -436,3 +436,117 @@ fn ipa_needs_an_entity_the_port_holds_and_a_packet_that_cites_one() { Err(LowerError::NoEntityReferent) ); } + +/// `nibu-proof-001` re-cut as the request: the addressee asked Nibu to stop, +/// one frame:3 predication (n4), consent naming it, and no other claim. +fn asked() -> Packet { + let mut p = proof_packet(); + p.referents.remove(1); + let mut ask = p.predications[1].clone(); + ask.id = nd("n4"); + ask.concept = iv("concept:6"); + ask.frame = iv("frame:3"); + ask.roles = vec![ + weksa::packet::RoleFill { + role: iv("role:1"), + filler: nd("n1"), + }, + weksa::packet::RoleFill { + role: iv("role:2"), + filler: nd("n3"), + }, + ]; + p.predications = vec![ask]; + p.discourse.speech_act = Some(iv("speech_act:1")); + p.discourse.addressee_consent = Some(nd("n4")); + p.constraints.preserve = vec![nd("n1")]; + p +} + +/// The request plus a frame:2 predication (n5) the consent could wrongly name. +fn asked_and_stopped() -> Packet { + let mut p = asked(); + let mut stop = proof_packet().predications.remove(1); + stop.roles[1].filler = nd("n4"); + p.predications.push(stop); + p +} + +#[test] +fn a_consented_request_lowers_to_what_the_addressee_asked() { + let (inv, _dir) = committed(); + assert_eq!(lowered(&asked(), &inv).unwrap(), "You asked Nibu to stop."); + let mut p = asked(); + p.provenance.source = weksa::packet::ProvenanceSource::GameTelemetry; + assert_eq!(lowered(&p, &inv).unwrap(), "You asked Nibu to stop."); +} + +#[test] +fn a_request_without_its_consent_is_refused_not_invented() { + let (inv, _dir) = committed(); + let mut p = asked(); + p.discourse.addressee_consent = None; + assert_eq!( + lowered(&p, &inv), + Err(LowerError::ConsentMissing { + predication: nd("n4") + }) + ); + // Consent that names a different predication is no consent to the request. + let mut p = asked_and_stopped(); + p.discourse.addressee_consent = Some(nd("n5")); + assert_eq!( + lowered(&p, &inv), + Err(LowerError::ConsentMissing { + predication: nd("n4") + }) + ); +} + +#[test] +fn a_request_by_someone_other_than_the_addressee_is_refused() { + let (inv, _dir) = committed(); + let mut p = asked(); + p.predications[0].roles[0].filler = nd("n3"); + p.predications[0].roles[1].filler = nd("n1"); + assert_eq!( + lowered(&p, &inv), + Err(LowerError::AskerNotAddressee { + predication: nd("n4") + }) + ); +} + +#[test] +fn consent_on_a_claim_that_is_no_request_stays_unrendered() { + let (inv, _dir) = committed(); + let mut p = proof_packet(); + p.discourse.addressee_consent = Some(nd("n5")); + assert_eq!( + lowered(&p, &inv), + Err(LowerError::Unrendered { + what: "addressee consent" + }) + ); +} + +#[test] +fn a_decomposition_or_import_cannot_reach_the_lowerer_with_consent() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let inventory = InventoryStore::open(&root.join("data/interlingua/inventory.cc")).unwrap(); + let corpus = CorpusStore::open(&root.join("data/corpus/corpus.cc")).unwrap(); + let entities = EntityStore::open(&root.join("data/entity/entity.cc")).unwrap(); + for source in [ + weksa::packet::ProvenanceSource::ModelDecomposition, + weksa::packet::ProvenanceSource::ImportedCorpus, + ] { + let mut p = asked(); + p.provenance.source = source; + let dir = tempfile::tempdir().unwrap(); + let refused = PacketStore::open(&dir.path().join("p.cc")) + .unwrap() + .admit(p, &inventory, &corpus, &entities) + .unwrap_err(); + assert!(format!("{refused:?}").contains("V15"), "{source:?}"); + } +} diff --git a/data/interlingua/inventory.cc b/data/interlingua/inventory.cc index becfc92e658d8e2da2833ee1a966e2b71bf2d296..cf353dbe05112cf2edefb596b06a4e009e41515c 100644 GIT binary patch delta 1411 zcmZXUT_{6g9LC$(v5&LwowF~sof?;D64@~yV`eKKBNuFCEk=^D4|kfla3iE6i<>10 zH&WcVla`2b<4VfC63N%4l(pm8<^T6S@BjI~@6-D}xUx-0cOuHSelR>08;(w=_$FAzMWkNR2bzqe8_1gk`vjy3(A-Lw zfQ-pjfDrODkY#xY$cH=(Btiv&TvA=aR9J;iSjH7L+1_X(G8yG+_-)2&hgj?z`PGxZ zP{Mp#(F7h>3cQ7KC6YNM-uRk|=3kUf=vAxg0kWpT^qz_|UNt7OtHud_{b&vP4QhPg zcc}3LA+;Sq2DEKJ_KV23)&m;94#}1dyyupm{em0C5>HodshfXl{&ncGiSvPnz(?awbgKVnz}( yw}aQ28NFx=l2Ihbrl}B5_+H;37Qmm%7bE!;zmuJOGRkz-^ zz;Zn2G6fUu@=q39--oOIu=Y4}b$BJ)?ERbd8ERb7e z90;wN0g_Xp(^MtEajTIWspr9YQ_lerG?-P^6u@z5kAd81Pk~cXajT|`^pbeN0}`)%YNEtXG6>a&OmZ&<&W8yvJ=2LZ zRXU;0nK=q&7R}Q@uFX?GzRiV%v&!1S$fRixf`yP6^=n zZ8+hfZ33JZnbSj$gHxb+Aa^uA7r|gwj2VNhb7mOGD-!_XwvPbWvE%MfcHG_Bg%`_q z;VU;}gm#3X*M{D9)erUDN9{>OV diff --git a/docs/interlingua-standard.md b/docs/interlingua-standard.md index ca392d1..e349a90 100644 --- a/docs/interlingua-standard.md +++ b/docs/interlingua-standard.md @@ -195,9 +195,10 @@ and `features` carry source-side nuance on content (hedge, certainty, irony and whatever the inventory grows). This is source-side communicative pressure, not target-language register. `addressee_consent` is no feature: it names the predication the addressee asked for or agreed to, a fact about what the -addressee did. The English lowerer renders no consent and refuses a packet that -carries one; a row that speaks for the request ("you asked her to") reads it -and refuses without it. +addressee did. The English lowerer's request row (`frame:3`, "{agent} asked +{patient} to stop") reads it: the consent must name that predication and its +agent must be the addressee, else `ConsentMissing` or `AskerNotAddressee`. A +consent naming any other predication is refused as unrendered. ### `stance` diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 4517320..5fa3022 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -342,7 +342,7 @@ measurable baseline from a packet: `lower` gives text, `score` gives numbers. | Whether a packet exists | `validate` / `PacketStore::admit` (unchanged) | packet, inventory, corpus, entities | admitted packet or typed refusal | the lowerer | | Wording of a packet in English | `crates/weksa/src/lower.rs`: `lower(&LoweringInput, &Inventory) -> Result` | the validated lowering view and the inventory at its revision | text, or a typed refusal naming the missing template | the corpus item, the scorer, any model | | English template table (concept words, frame templates, speech-act openers) | the same module, as rows `(inventory id, gloss it speaks for, surface)` | none; it is data in code | rows the lowerer looks up | the inventory (it owns the ids and glosses, not the English) | -| The inventory's concepts and frames | an inventory revision; here `inventory::proof_revision()` (rev 2), hand-seeded as `seed()` is | none | the 5 concepts and 2 frames `nibu-proof-001` names | the lowerer | +| The inventory's concepts and frames | an inventory revision; here `inventory::proof_revision()` (rev 2), hand-seeded as `seed()` is | none | the 6 concepts and 3 frames `nibu-proof-001` and the consent-gated request packet name | the lowerer | | What a score counts | `Scorer::score` (unchanged) | output text, reference text | `ItemScore` | `weksa score`, which only reads text and calls it | | The proof packet | `packet::proof_packet()`, typed ids of rev 2, admitted by `weksa packet proof ` through `PacketStore::admit`. Not JSON: `Packet`'s serde is CultCache's positional tuple, and JSON in is the xenos boundary the cut 7 encoder owns | none | admitted packet | the CLI decides nothing about validity | From 53e54148e0cbe8999b9ba90ae7222bc039347399 Mon Sep 17 00:00:00 2001 From: Weksa Date: Mon, 9 Nov 2026 10:12:01 +0000 Subject: [PATCH 010/175] Test: consent to one of two requests leaves the other unconsented (Soul low) Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/lower.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/crates/weksa/tests/lower.rs b/crates/weksa/tests/lower.rs index 89e35ed..3d796a5 100644 --- a/crates/weksa/tests/lower.rs +++ b/crates/weksa/tests/lower.rs @@ -503,6 +503,21 @@ fn a_request_without_its_consent_is_refused_not_invented() { ); } +#[test] +fn consent_to_one_of_two_requests_leaves_the_other_unconsented() { + let (inv, _dir) = committed(); + let mut p = asked(); + let mut second = p.predications[0].clone(); + second.id = nd("n5"); + p.predications.push(second); + assert_eq!( + lowered(&p, &inv), + Err(LowerError::ConsentMissing { + predication: nd("n5") + }) + ); +} + #[test] fn a_request_by_someone_other_than_the_addressee_is_refused() { let (inv, _dir) = committed(); From 453b3d6c2fdaeddfa13a815c706c9e0d5d789579 Mon Sep 17 00:00:00 2001 From: Weksa Date: Mon, 9 Nov 2026 10:42:48 +0000 Subject: [PATCH 011/175] Corpus intake lows: S6 folds dotless i, kana and Arabic-Indic digits; CLI errors stop echoing identifiers; --admitted-on is a calendar date; marks-only texts refused FormError and StoreRefusal display no caller identifiers; clap value errors name the flag and reason only. One is_calendar_date serves admitted_on and the pronunciation date. A text with no base letter or digit (Lm or M only) is refused beside the no-alphanumeric refusal; modifier_letters.txt is the Lm table its test walks. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/form.rs | 32 +- crates/weksa/src/main.rs | 63 ++-- crates/weksa/src/store.rs | 196 +++++++++-- crates/weksa/tests/cli.rs | 228 ++++++++++++- crates/weksa/tests/forms.rs | 9 +- crates/weksa/tests/modifier_letters.txt | 411 ++++++++++++++++++++++++ crates/weksa/tests/stores.rs | 151 +++++++++ 7 files changed, 1043 insertions(+), 47 deletions(-) create mode 100644 crates/weksa/tests/modifier_letters.txt diff --git a/crates/weksa/src/form.rs b/crates/weksa/src/form.rs index 9de8eb8..3626fd1 100644 --- a/crates/weksa/src/form.rs +++ b/crates/weksa/src/form.rs @@ -22,7 +22,9 @@ impl FormError { impl fmt::Display for FormError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?} is not a {}", self.got, self.form) + // `got` stays a field for callers; the text does not echo it, because + // the value is often an identifier the caller supplied. + write!(f, "not a {}", self.form) } } @@ -61,6 +63,34 @@ macro_rules! string_form { } pub(crate) use string_form; +/// A real calendar date written `YYYY-MM-DD`: four ASCII digits, a month 1 to +/// 12 and a day that month has in that year (leap years by the Gregorian rule). +pub fn is_calendar_date(s: &str) -> bool { + let b = s.as_bytes(); + if b.len() != 10 || b[4] != b'-' || b[7] != b'-' { + return false; + } + let digits = |r: std::ops::Range| { + b[r.clone()] + .iter() + .all(u8::is_ascii_digit) + .then(|| s[r].parse::().ok()) + .flatten() + }; + let (Some(year), Some(month), Some(day)) = (digits(0..4), digits(5..7), digits(8..10)) else { + return false; + }; + let leap = year % 4 == 0 && (year % 100 != 0 || year % 400 == 0); + let days = match month { + 1 | 3 | 5 | 7 | 8 | 10 | 12 => 31, + 4 | 6 | 9 | 11 => 30, + 2 if leap => 29, + 2 => 28, + _ => return false, + }; + (1..=days).contains(&day) +} + /// A BCP 47-form language tag: 1 to 35 bytes of ASCII alphanumerics in /// hyphen-separated subtags of 1 to 8 bytes; the first is 2 to 8 letters, or the /// single letter `x` of a private-use tag. Form only: no registry is consulted. diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 5518823..fe997b3 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -4,6 +4,7 @@ use std::path::PathBuf; use anyhow::{Context, Result, bail}; +use clap::error::{ContextKind, ContextValue, ErrorKind}; use clap::{Args, Parser, Subcommand, ValueEnum}; use unicode_normalization::UnicodeNormalization; use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn}; @@ -180,9 +181,7 @@ impl PutArgs { let lf = forms .iter_mut() .find(|lf| lf.language == language) - .with_context(|| { - format!("--pronunciation names {language}, which has no --form") - })?; + .context("--pronunciation names a language that has no --form")?; lf.pronunciation = Some(Pronunciation { ipa: ipa.nfc().collect(), ruled_by: need(self.ruled_by.clone(), "ruled-by")?, @@ -290,12 +289,47 @@ struct AddArgs { generator: Option, #[arg(long)] prompt_digest: Option, - #[arg(long)] + /// `YYYY-MM-DD`, a real calendar date. + #[arg(long, value_parser = parse_date)] admitted_on: String, #[arg(long)] supersedes: Option, } +fn parse_date(s: &str) -> std::result::Result { + if weksa::form::is_calendar_date(s) { + Ok(s.to_string()) + } else { + Err("not a YYYY-MM-DD calendar date".into()) + } +} + +/// Parses the command line. A rejected value is reported by the flag that took +/// it and the reason, never by the value: values are the caller's identifiers. +fn parse_cli() -> Cli { + use std::error::Error; + match Cli::try_parse() { + Ok(cli) => cli, + Err(e) + if matches!( + e.kind(), + ErrorKind::InvalidValue | ErrorKind::ValueValidation + ) => + { + let arg = match e.get(ContextKind::InvalidArg) { + Some(ContextValue::String(a)) => { + a.split_whitespace().next().unwrap_or("").to_string() + } + _ => String::new(), + }; + let reason = e.source().map(|s| format!(": {s}")).unwrap_or_default(); + eprintln!("error: invalid value for {arg}{reason}"); + std::process::exit(2); + } + Err(e) => e.exit(), + } +} + fn need(value: Option, flag: &str) -> Result { value.with_context(|| format!("this source needs --{flag}")) } @@ -413,17 +447,13 @@ fn refused(e: StoreError) -> anyhow::Error { } fn main() -> Result<()> { - let cli = Cli::parse(); + let cli = parse_cli(); match cli.command { Command::Validate { store, key } => { let inventory = InventoryStore::open(&cli.inventory)?.inventory()?; let packets = PacketStore::open(&store)?; let packets = match key { - Some(key) => vec![ - packets - .get(&key)? - .with_context(|| format!("no packet {key}"))?, - ], + Some(key) => vec![packets.get(&key)?.context("no such packet")?], None => packets.all()?, }; let mut bad = 0; @@ -531,9 +561,7 @@ fn main() -> Result<()> { let inventory_store = InventoryStore::open(&cli.inventory)?; let inventory = inventory_store.inventory()?; let mut packets = PacketStore::open(&store)?; - let packet = packets - .get(&key)? - .with_context(|| format!("no packet {key}"))?; + let packet = packets.get(&key)?.context("no such packet")?; // Admission is the only source of a LoweringInput; re-admitting the // stored packet is a no-op write that re-runs every door check. let view = packets @@ -562,12 +590,7 @@ fn main() -> Result<()> { let references = args .items .iter() - .map(|key| { - Ok(corpus - .get(key)? - .with_context(|| format!("no item {key}"))? - .text) - }) + .map(|key| Ok(corpus.get(key)?.context("no such item")?.text)) .collect::>>()?; let inputs: Vec = references .iter() @@ -587,7 +610,7 @@ fn main() -> Result<()> { Command::Split { item_key } => { let item = CorpusStore::open(&cli.corpus)? .get(&item_key)? - .with_context(|| format!("no item {item_key}"))?; + .context("no such item")?; println!("{}", split_text(&item)); } Command::Schema { out } => { diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index 764a7be..bc5e9fa 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -13,7 +13,9 @@ use unicode_segmentation::UnicodeSegmentation; use crate::corpus::{CorpusItem, CorpusItemId, Source, item_key, key_slug}; use crate::entity::{Entity, EntityOrigin, Pronunciation}; -use crate::form::{is_attestation, is_joining_only, is_language_tag, is_refused_format}; +use crate::form::{ + is_attestation, is_calendar_date, is_joining_only, is_language_tag, is_refused_format, +}; use crate::inventory::{Change, InvId, Inventory, InventoryEntry, InventoryRevision, revision_key}; use crate::packet::{EntityId, Packet}; use crate::validate::{LoweringInput, Refusal, admissible}; @@ -81,6 +83,44 @@ pub enum StoreRefusal { Malformed { key: String, why: &'static str }, } +/// What the CLI prints: the rule broken and the numbers and reasons that +/// explain it. Item keys, entity ids and groups are the caller's identifiers +/// and are left to the typed value, never to text. +impl fmt::Display for StoreRefusal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use StoreRefusal::*; + match self { + S1ItemImmutable { .. } => f.write_str("S1ItemImmutable"), + S2RevisionChain { expected, got } => { + write!(f, "S2RevisionChain expected {expected} got {got}") + } + S3Frozen { frozen } => write!(f, "S3Frozen at {frozen}"), + S4IntroducedRev { + introduced, + revision, + .. + } => write!(f, "S4IntroducedRev {introduced} in revision {revision}"), + S5SupersedeGroup { .. } => f.write_str("S5SupersedeGroup"), + S6TextHeld { .. } => f.write_str("S6TextHeld"), + UnknownCorpusItem { .. } => f.write_str("UnknownCorpusItem"), + PacketImmutable { .. } => f.write_str("PacketImmutable"), + UnknownEntity { .. } => f.write_str("UnknownEntity"), + NameSpanRange { start, end, .. } => write!(f, "NameSpanRange {start}..{end}"), + NameSpanSplitsCluster { start, end, .. } => { + write!(f, "NameSpanSplitsCluster {start}..{end}") + } + NotNfc { .. } => f.write_str("NotNfc"), + EntityBounds { bound, .. } => write!(f, "EntityBounds {bound:?}"), + NameSpanOverlap { .. } => f.write_str("NameSpanOverlap"), + NameSpanNotAForm { start, end, .. } => write!(f, "NameSpanNotAForm {start}..{end}"), + EntityNotAppendOnly { .. } => f.write_str("EntityNotAppendOnly"), + EntityNotOnItem { .. } => f.write_str("EntityNotOnItem"), + WithdrawnCorpusItem { .. } => f.write_str("WithdrawnCorpusItem"), + Malformed { why, .. } => write!(f, "Malformed: {why}"), + } + } +} + /// The capacity bound an entity broke. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum EntityBound { @@ -111,7 +151,7 @@ pub enum StoreError { impl fmt::Display for StoreError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - StoreError::Refused(r) => write!(f, "refused: {r:?}"), + StoreError::Refused(r) => write!(f, "refused: {r}"), StoreError::Invalid(rs) => { write!(f, "invalid packet:")?; rs.iter().try_for_each(|r| write!(f, " [{r}]")) @@ -244,6 +284,13 @@ impl CorpusStore { if !text.chars().any(char::is_alphanumeric) { return Err(malformed(&item.id, "text has no alphanumeric character")); } + // Read from the text as given: NFKC maps modifier letters (Ê°) to base letters. + if !item.text.chars().any(is_base) { + return Err(malformed(&item.id, "text has no base letter or digit")); + } + if !is_calendar_date(&item.admitted_on) { + return Err(malformed(&item.id, "admitted_on is not a YYYY-MM-DD date")); + } if let Some(why) = empty_attestation(&item.source) { return Err(malformed(&item.id, why)); } @@ -364,16 +411,140 @@ fn check_entities(item: &CorpusItem, entities: &EntityStore) -> Result<(), Store /// default case fold, NFKC) keeping letters, digits and combining marks /// (General_Category L, N, M) so tone marks and viramas keep words apart. Composed and decomposed, /// fullwidth, ligature and case-fold variants (straße, STRASSE) and spacing and -/// punctuation variants compare equal. Homoglyphs across scripts (Cyrillic a for -/// Latin a) stay apart: no key here reads what a character looks like. +/// punctuation variants compare equal. Three writings of one thing also fold: +/// Turkish dotless i and dotted capital I read as i, katakana reads as the +/// hiragana of the same syllable, and Arabic-Indic and Extended Arabic-Indic +/// digits read as 0-9. Homoglyphs across scripts (Cyrillic a for Latin a) stay +/// apart: no key here reads what a character looks like. fn normalized(text: &str) -> String { - let folded = default_case_fold_str(&text.nfkc().collect::()); - folded + let mapped: String = text.nfkc().map(fold_variant).collect(); + default_case_fold_str(&mapped) .nfkc() .filter(|c| c.is_alphanumeric() || is_combining_mark(*c)) .collect() } +/// The one-to-one folds `normalized` adds to NFKC and case folding, applied +/// before the case fold (U+0130 would otherwise fold to i plus a combining dot, +/// which the key keeps as a mark). +fn fold_variant(c: char) -> char { + match c { + '\u{131}' => 'i', + '\u{130}' => 'I', + // Katakana letters and small letters to hiragana; the iteration marks + // ヽヾ to ã‚ã‚ž. The prolonged sound mark U+30FC stays. + '\u{30a1}'..='\u{30f6}' | '\u{30fd}'..='\u{30fe}' => { + char::from_u32(c as u32 - 0x60).unwrap_or(c) + } + '\u{660}'..='\u{669}' => char::from(b'0' + (c as u32 - 0x660) as u8), + '\u{6f0}'..='\u{6f9}' => char::from(b'0' + (c as u32 - 0x6f0) as u8), + _ => c, + } +} + +/// A character that can carry a word: alphanumeric and neither a combining +/// mark (Mn, Mc, Me) nor a modifier letter (Lm). A text of none is marks and +/// modifiers only, like a text of punctuation. +fn is_base(c: char) -> bool { + c.is_alphanumeric() && !is_combining_mark(c) && !is_modifier_letter(c) +} + +/// General_Category Lm, Unicode 17 (the Rust 1.99 tables), as inclusive ranges. +fn is_modifier_letter(c: char) -> bool { + const LM: [(u32, u32); 79] = [ + (0x2b0, 0x2c1), + (0x2c6, 0x2d1), + (0x2e0, 0x2e4), + (0x2ec, 0x2ec), + (0x2ee, 0x2ee), + (0x374, 0x374), + (0x37a, 0x37a), + (0x559, 0x559), + (0x640, 0x640), + (0x6e5, 0x6e6), + (0x7f4, 0x7f5), + (0x7fa, 0x7fa), + (0x81a, 0x81a), + (0x824, 0x824), + (0x828, 0x828), + (0x8c9, 0x8c9), + (0x971, 0x971), + (0xe46, 0xe46), + (0xec6, 0xec6), + (0x10fc, 0x10fc), + (0x17d7, 0x17d7), + (0x1843, 0x1843), + (0x1aa7, 0x1aa7), + (0x1c78, 0x1c7d), + (0x1d2c, 0x1d6a), + (0x1d78, 0x1d78), + (0x1d9b, 0x1dbf), + (0x2071, 0x2071), + (0x207f, 0x207f), + (0x2090, 0x209c), + (0x2c7c, 0x2c7d), + (0x2d6f, 0x2d6f), + (0x2e2f, 0x2e2f), + (0x3005, 0x3005), + (0x3031, 0x3035), + (0x303b, 0x303b), + (0x309d, 0x309e), + (0x30fc, 0x30fe), + (0xa015, 0xa015), + (0xa4f8, 0xa4fd), + (0xa60c, 0xa60c), + (0xa67f, 0xa67f), + (0xa69c, 0xa69d), + (0xa717, 0xa71f), + (0xa770, 0xa770), + (0xa788, 0xa788), + (0xa7f1, 0xa7f4), + (0xa7f8, 0xa7f9), + (0xa9cf, 0xa9cf), + (0xa9e6, 0xa9e6), + (0xaa70, 0xaa70), + (0xaadd, 0xaadd), + (0xaaf3, 0xaaf4), + (0xab5c, 0xab5f), + (0xab69, 0xab69), + (0xff70, 0xff70), + (0xff9e, 0xff9f), + (0x10780, 0x10785), + (0x10787, 0x107b0), + (0x107b2, 0x107ba), + (0x10d4e, 0x10d4e), + (0x10d6f, 0x10d6f), + (0x10ec5, 0x10ec5), + (0x11dd9, 0x11dd9), + (0x16b40, 0x16b43), + (0x16d40, 0x16d42), + (0x16d6b, 0x16d6c), + (0x16f93, 0x16f9f), + (0x16fe0, 0x16fe1), + (0x16fe3, 0x16fe3), + (0x16ff2, 0x16ff3), + (0x1aff0, 0x1aff3), + (0x1aff5, 0x1affb), + (0x1affd, 0x1affe), + (0x1e030, 0x1e06d), + (0x1e137, 0x1e13d), + (0x1e4eb, 0x1e4eb), + (0x1e6ff, 0x1e6ff), + (0x1e94b, 0x1e94b), + ]; + let c = c as u32; + LM.binary_search_by(|&(lo, hi)| { + if hi < c { + std::cmp::Ordering::Less + } else if lo > c { + std::cmp::Ordering::Greater + } else { + std::cmp::Ordering::Equal + } + }) + .is_ok() +} + /// The field that holds an unattested provenance string, if any: a string with /// no alphanumeric, a control or format character, or over 200 bytes attests /// nothing. @@ -563,18 +734,7 @@ fn check_pronunciation(entity: &Entity, p: &Pronunciation) -> Result<(), StoreEr if !is_attestation(&p.ruled_by) || !is_attestation(&p.source) { return Err(bad("a pronunciation names who ruled it and where")); } - let b = p.date.as_bytes(); - let date_ok = b.len() == 10 - && b.iter().enumerate().all(|(i, c)| { - if i == 4 || i == 7 { - *c == b'-' - } else { - c.is_ascii_digit() - } - }) - && (1..=12).contains(&p.date[5..7].parse::().unwrap_or(0)) - && (1..=31).contains(&p.date[8..].parse::().unwrap_or(0)); - if !date_ok { + if !is_calendar_date(&p.date) { return Err(bad("a pronunciation is dated YYYY-MM-DD")); } if p.rejected.len() > EntityBound::MAX_FORMS { diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 72ad140..fda6901 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -836,7 +836,7 @@ fn proof_packet_lowers_and_the_lowering_scores_against_items() { assert!(text.contains("NIBU will stop"), "{text}"); let missing = weksa(d.path(), &["lower", "./p.cc", "packet:0000000000000000"]); - assert!(!missing.status.success() && out(&missing).contains("no packet")); + assert!(!missing.status.success() && out(&missing).contains("no such packet")); // IPA is the ruled pronunciation or nothing: spelling is not one. let args = ["lower", "./p.cc", "packet:6e6962752d303031", "--ipa"]; @@ -908,5 +908,229 @@ fn proof_packet_lowers_and_the_lowering_scores_against_items() { &url, ], ); - assert!(!unknown.status.success() && out(&unknown).contains("no item item:nibu:9")); + assert!(!unknown.status.success() && out(&unknown).contains("no such item")); + assert!(!out(&unknown).contains("item:nibu:9"), "{}", out(&unknown)); +} + +/// Runs `args` and demands a refusal whose text holds `reason` and none of the +/// identifiers the caller supplied. +fn refuses_without_echo(d: &std::path::Path, args: &[&str], reason: &str) { + let o = weksa(d, args); + let text = out(&o); + assert!(!o.status.success(), "{args:?}: {text}"); + assert!(text.contains(reason), "{args:?} lacks {reason:?}: {text}"); + for a in args.iter().filter(|a| a.to_lowercase().contains("canary")) { + // An argument may carry the canary beside flag syntax (`--x=...`); the + // identifier itself is the part from the canary on. + let id = &a[a.to_lowercase().find("canary").unwrap()..]; + assert!( + !text.to_lowercase().contains(&id.to_lowercase()), + "{args:?} echoes {id:?}: {text}" + ); + } +} + +#[test] +fn cli_errors_never_echo_identifiers() { + let d = tempfile::tempdir().unwrap(); + let p = d.path(); + cast_entities(p, &["nibu"]); + let add = |extra: &[&str]| -> Vec { + ["corpus", "add", "--admitted-on", "2026-10-29"] + .into_iter() + .chain(extra.iter().copied()) + .map(String::from) + .collect() + }; + let run = |args: Vec, reason: &str| { + let refs: Vec<&str> = args.iter().map(String::as_str).collect(); + refuses_without_echo(p, &refs, reason); + }; + // The positional entity id, malformed and well formed but unheld. + refuses_without_echo( + p, + &[ + "entity", + "put", + "canary-id", + "--form", + "en-US=X", + "--origin", + "live", + "--persona", + "k", + ], + "not a EntityId", + ); + // --speaker: a malformed id, and a well formed one no entity holds. + run( + add(&[ + "--slug", + "q", + "--text", + "Hello there.", + "--speaker", + "canary-speaker", + "--group", + "g", + "--source", + "operator-written", + ]), + "--speaker", + ); + run( + add(&[ + "--slug", + "q", + "--text", + "Hello there.", + "--speaker", + "entity:canary", + "--group", + "g", + "--source", + "operator-written", + ]), + "UnknownEntity", + ); + // clap enum values. + run( + add(&[ + "--slug", + "q", + "--text", + "Hello there.", + "--speaker", + "entity:nibu", + "--group", + "g", + "--source", + "canary-source", + ]), + "--source", + ); + refuses_without_echo( + p, + &[ + "entity", + "put", + "entity:kay", + "--form", + "en-US=X", + "--origin", + "canary-origin", + ], + "--origin", + ); + run( + add(&[ + "--slug", + "q", + "--text", + "Hello there.", + "--speaker", + "entity:nibu", + "--group", + "g", + "--source", + "model-written", + "--model-family", + "canary-family", + ]), + "--model-family", + ); + // Keys on withdraw, split, validate, lower and score. + refuses_without_echo(p, &["corpus", "withdraw", "item:canary:7"], "no such item"); + refuses_without_echo(p, &["split", "item:canary:7"], "no such item"); + std::fs::write(p.join("out.txt"), "text").unwrap(); + refuses_without_echo( + p, + &["score", "item:canary:7", "--output", "out.txt"], + "no such item", + ); + let packets = weksa(p, &["inventory", "seed"]); + assert!(packets.status.success(), "{}", out(&packets)); + let made = weksa(p, &["packet", "proof", "p.cc"]); + let _ = made; + refuses_without_echo( + p, + &["validate", "p.cc", "packet:canary:7"], + "no such packet", + ); + refuses_without_echo(p, &["lower", "p.cc", "packet:canary:7"], "no such packet"); + // S6 names no held key; S5 names no group. + let held = weksa( + p, + &add(&[ + "--slug", + "canarya", + "--text", + "Fresh words here.", + "--speaker", + "entity:nibu", + "--group", + "canary-first", + "--source", + "operator-written", + ]) + .iter() + .map(String::as_str) + .collect::>(), + ); + assert!(held.status.success(), "{}", out(&held)); + let copy = add(&[ + "--slug", + "other", + "--text", + "Fresh words here.", + "--speaker", + "entity:nibu", + "--group", + "g", + "--source", + "operator-written", + ]); + let copy = weksa(p, ©.iter().map(String::as_str).collect::>()); + assert!( + !copy.status.success() && out(©).contains("S6TextHeld"), + "{}", + out(©) + ); + assert!(!out(©).contains("canary"), "{}", out(©)); + let moved = add(&[ + "--slug", + "other", + "--text", + "Corrected words here.", + "--speaker", + "entity:nibu", + "--group", + "canary-second", + "--supersedes", + "item:canarya:1", + "--source", + "operator-written", + ]); + let moved = weksa(p, &moved.iter().map(String::as_str).collect::>()); + assert!( + !moved.status.success() && out(&moved).contains("S5SupersedeGroup"), + "{}", + out(&moved) + ); + assert!(!out(&moved).contains("canary"), "{}", out(&moved)); +} + +#[test] +fn admitted_on_must_be_a_real_calendar_date() { + let d = tempfile::tempdir().unwrap(); + cast_entities(d.path(), &["nibu"]); + for bad in ["2026-02-30", "", "junk", "2026-13-01", "2026-1-1"] { + let mut args = ADD.to_vec(); + let at = args.iter().position(|a| *a == "--admitted-on").unwrap(); + args[at + 1] = bad; + let o = weksa(d.path(), &args); + assert!(!o.status.success(), "{bad:?}: {}", out(&o)); + assert!(out(&o).contains("--admitted-on"), "{bad:?}: {}", out(&o)); + } + assert!(!out(&weksa(d.path(), &["corpus", "dump"])).contains("item:")); } diff --git a/crates/weksa/tests/forms.rs b/crates/weksa/tests/forms.rs index 61b62e1..f0b2469 100644 --- a/crates/weksa/tests/forms.rs +++ b/crates/weksa/tests/forms.rs @@ -51,15 +51,12 @@ fn forms_refuse_surface_text() { } #[test] -fn a_refusal_names_the_form_and_the_value_and_a_form_reads_back_as_written() { +fn a_refusal_names_the_form_and_keeps_the_value_out_of_its_text() { let e = NodeId::parse("n0").unwrap_err(); assert_eq!(e.form, "NodeId"); assert_eq!(e.got, "n0"); - assert_eq!(e.to_string(), "\"n0\" is not a NodeId"); - assert_eq!( - Grade::new(1001).unwrap_err().to_string(), - "\"1001\" is not a Grade" - ); + assert_eq!(e.to_string(), "not a NodeId"); + assert_eq!(Grade::new(1001).unwrap_err().to_string(), "not a Grade"); assert_eq!(EncoderId::parse(ENCODER_ID).unwrap().as_str(), ENCODER_ID); assert_eq!(PacketId::parse(PACKET_ID).unwrap().as_str(), PACKET_ID); assert_eq!( diff --git a/crates/weksa/tests/modifier_letters.txt b/crates/weksa/tests/modifier_letters.txt new file mode 100644 index 0000000..991618d --- /dev/null +++ b/crates/weksa/tests/modifier_letters.txt @@ -0,0 +1,411 @@ +# Lm, Node 24 (Unicode 17.0) over 0..10FFFF, one hex code point per line +2B0 +2B1 +2B2 +2B3 +2B4 +2B5 +2B6 +2B7 +2B8 +2B9 +2BA +2BB +2BC +2BD +2BE +2BF +2C0 +2C1 +2C6 +2C7 +2C8 +2C9 +2CA +2CB +2CC +2CD +2CE +2CF +2D0 +2D1 +2E0 +2E1 +2E2 +2E3 +2E4 +2EC +2EE +374 +37A +559 +640 +6E5 +6E6 +7F4 +7F5 +7FA +81A +824 +828 +8C9 +971 +E46 +EC6 +10FC +17D7 +1843 +1AA7 +1C78 +1C79 +1C7A +1C7B +1C7C +1C7D +1D2C +1D2D +1D2E +1D2F +1D30 +1D31 +1D32 +1D33 +1D34 +1D35 +1D36 +1D37 +1D38 +1D39 +1D3A +1D3B +1D3C +1D3D +1D3E +1D3F +1D40 +1D41 +1D42 +1D43 +1D44 +1D45 +1D46 +1D47 +1D48 +1D49 +1D4A +1D4B +1D4C +1D4D +1D4E +1D4F +1D50 +1D51 +1D52 +1D53 +1D54 +1D55 +1D56 +1D57 +1D58 +1D59 +1D5A +1D5B +1D5C +1D5D +1D5E +1D5F +1D60 +1D61 +1D62 +1D63 +1D64 +1D65 +1D66 +1D67 +1D68 +1D69 +1D6A +1D78 +1D9B +1D9C +1D9D +1D9E +1D9F +1DA0 +1DA1 +1DA2 +1DA3 +1DA4 +1DA5 +1DA6 +1DA7 +1DA8 +1DA9 +1DAA +1DAB +1DAC +1DAD +1DAE +1DAF +1DB0 +1DB1 +1DB2 +1DB3 +1DB4 +1DB5 +1DB6 +1DB7 +1DB8 +1DB9 +1DBA +1DBB +1DBC +1DBD +1DBE +1DBF +2071 +207F +2090 +2091 +2092 +2093 +2094 +2095 +2096 +2097 +2098 +2099 +209A +209B +209C +2C7C +2C7D +2D6F +2E2F +3005 +3031 +3032 +3033 +3034 +3035 +303B +309D +309E +30FC +30FD +30FE +A015 +A4F8 +A4F9 +A4FA +A4FB +A4FC +A4FD +A60C +A67F +A69C +A69D +A717 +A718 +A719 +A71A +A71B +A71C +A71D +A71E +A71F +A770 +A788 +A7F1 +A7F2 +A7F3 +A7F4 +A7F8 +A7F9 +A9CF +A9E6 +AA70 +AADD +AAF3 +AAF4 +AB5C +AB5D +AB5E +AB5F +AB69 +FF70 +FF9E +FF9F +10780 +10781 +10782 +10783 +10784 +10785 +10787 +10788 +10789 +1078A +1078B +1078C +1078D +1078E +1078F +10790 +10791 +10792 +10793 +10794 +10795 +10796 +10797 +10798 +10799 +1079A +1079B +1079C +1079D +1079E +1079F +107A0 +107A1 +107A2 +107A3 +107A4 +107A5 +107A6 +107A7 +107A8 +107A9 +107AA +107AB +107AC +107AD +107AE +107AF +107B0 +107B2 +107B3 +107B4 +107B5 +107B6 +107B7 +107B8 +107B9 +107BA +10D4E +10D6F +10EC5 +11DD9 +16B40 +16B41 +16B42 +16B43 +16D40 +16D41 +16D42 +16D6B +16D6C +16F93 +16F94 +16F95 +16F96 +16F97 +16F98 +16F99 +16F9A +16F9B +16F9C +16F9D +16F9E +16F9F +16FE0 +16FE1 +16FE3 +16FF2 +16FF3 +1AFF0 +1AFF1 +1AFF2 +1AFF3 +1AFF5 +1AFF6 +1AFF7 +1AFF8 +1AFF9 +1AFFA +1AFFB +1AFFD +1AFFE +1E030 +1E031 +1E032 +1E033 +1E034 +1E035 +1E036 +1E037 +1E038 +1E039 +1E03A +1E03B +1E03C +1E03D +1E03E +1E03F +1E040 +1E041 +1E042 +1E043 +1E044 +1E045 +1E046 +1E047 +1E048 +1E049 +1E04A +1E04B +1E04C +1E04D +1E04E +1E04F +1E050 +1E051 +1E052 +1E053 +1E054 +1E055 +1E056 +1E057 +1E058 +1E059 +1E05A +1E05B +1E05C +1E05D +1E05E +1E05F +1E060 +1E061 +1E062 +1E063 +1E064 +1E065 +1E066 +1E067 +1E068 +1E069 +1E06A +1E06B +1E06C +1E06D +1E137 +1E138 +1E139 +1E13A +1E13B +1E13C +1E13D +1E4EB +1E6FF +1E94B diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs index cf5ba4a..8eb5f17 100644 --- a/crates/weksa/tests/stores.rs +++ b/crates/weksa/tests/stores.rs @@ -336,6 +336,157 @@ fn s6_reads_text_under_compatibility_case_folding() { .unwrap(); } +#[test] +fn s6_folds_dotless_i_kana_and_arabic_indic_digits() { + let d = dir(); + let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + // Turkish dotless and dotted capital i read as i; hiragana and katakana + // (small, voiced, iteration marks, halfwidth through NFKC) read as one + // syllabary; Arabic-Indic and Extended Arabic-Indic digits read as 0-9. + let pairs = [ + ("isi", "\u{131}s\u{131}"), + ("istanbul", "\u{130}stanbul"), + ( + "\u{3072}\u{3089}\u{304c}\u{306a}", + "\u{30d2}\u{30e9}\u{30ac}\u{30ca}", + ), + ("\u{3041}\u{3083}", "\u{30a1}\u{30e3}"), + ("\u{3042}\u{309d}\u{309e}", "\u{3042}\u{30fd}\u{30fe}"), + ("\u{306d}\u{3053}", "\u{ff88}\u{ff7a}"), + ( + "room 1234567890", + "room \u{661}\u{662}\u{663}\u{664}\u{665}\u{666}\u{667}\u{668}\u{669}\u{660}", + ), + ("gate 45", "gate \u{6f4}\u{6f5}"), + ]; + for (n, (a, b)) in pairs.into_iter().enumerate() { + let (first, second) = ( + format!("item:fold:{}", 2 * n + 1), + format!("item:fold:{}", 2 * n + 2), + ); + store.add(item(&first, a), &ents(&d)).unwrap(); + assert_eq!( + refusal(store.add(item(&second, b), &ents(&d))), + StoreRefusal::S6TextHeld { + key: second.clone(), + held_by: first.clone() + }, + "{a:?} vs {b:?}" + ); + } + // Nothing wider: the prolonged sound mark, Devanagari digits and the + // Cyrillic homoglyph of i stay apart from what they resemble. + store + .add(item("item:fold:90", "\u{30a2}\u{30fc}"), &ents(&d)) + .unwrap(); + store + .add(item("item:fold:91", "\u{30a2}"), &ents(&d)) + .unwrap(); + store + .add(item("item:fold:92", "room 12"), &ents(&d)) + .unwrap(); + store + .add(item("item:fold:93", "room \u{967}\u{968}"), &ents(&d)) + .unwrap(); + store.add(item("item:fold:94", "bit"), &ents(&d)).unwrap(); + store + .add(item("item:fold:95", "b\u{456}t"), &ents(&d)) + .unwrap(); +} + +#[test] +fn a_text_of_only_marks_is_refused() { + let d = dir(); + let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + // Prolonged sound mark, Thai maiyamok, fathatan, tatweel: alphabetic to + // Unicode, but none carries a word. + for (n, text) in [ + "\u{30fc}", + "\u{e46}", + "\u{64b}", + "\u{640}", + "\u{30fc}\u{30fc}", + "\u{640}\u{64b}", + ] + .into_iter() + .enumerate() + { + let key = format!("item:mo:{}", n + 1); + assert_eq!( + refusal(store.add(item(&key, text), &ents(&d))), + malformed(&key, "text has no base letter or digit"), + "{text:?}" + ); + } + // Every modifier letter (Lm) alone is refused: for having no base letter, + // or, where Unicode does not call it alphabetic, no alphanumeric at all, or + // (U+0374, U+037A) for not being NFC. + for line in include_str!("modifier_letters.txt") + .lines() + .filter(|l| !l.starts_with('#')) + { + let c = char::from_u32(u32::from_str_radix(line, 16).unwrap()).unwrap(); + let key = "item:mo:90"; + let got = refusal(store.add(item(key, &c.to_string()), &ents(&d))); + assert!( + got == StoreRefusal::NotNfc { key: key.into() } + || got == malformed(key, "text has no base letter or digit") + || got == malformed(key, "text has no alphanumeric character"), + "{c:?}: {got:?}" + ); + } + // With a base letter or digit the same marks are text. + store + .add(item("item:mo:91", "\u{30a2}\u{30fc}"), &ents(&d)) + .unwrap(); + store + .add(item("item:mo:92", "\u{e01}\u{e46}"), &ents(&d)) + .unwrap(); + store + .add(item("item:mo:93", "\u{640}7"), &ents(&d)) + .unwrap(); +} + +#[test] +fn admitted_on_is_a_real_calendar_date() { + let d = dir(); + let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + for (n, bad) in [ + "", + "junk", + "2026-02-30", + "2026-02-29", + "2026-13-01", + "2026-00-10", + "2026-04-31", + "2026-1-1", + "2026-10-29 ", + "20261029", + "2026-10-29T00:00Z", + "\u{661}\u{662}\u{663}\u{664}-10-29", + ] + .into_iter() + .enumerate() + { + let key = format!("item:dt:{}", n + 1); + let mut it = item(&key, "a date"); + it.admitted_on = bad.into(); + assert_eq!( + refusal(store.add(it, &ents(&d))), + malformed(&key, "admitted_on is not a YYYY-MM-DD date"), + "{bad:?}" + ); + } + for (n, good) in ["2026-10-29", "2024-02-29", "2000-02-29"] + .into_iter() + .enumerate() + { + let mut it = item(&format!("item:dt:{}", 90 + n), &format!("a date {n}")); + it.admitted_on = good.into(); + store.add(it, &ents(&d)).unwrap(); + } +} + #[test] fn s6_keeps_tone_marks_and_viramas_apart() { let d = dir(); From 2ce6325b704f94f98a1dbd09e82ac0c65fdeccd2 Mon Sep 17 00:00:00 2001 From: Weksa Date: Mon, 9 Nov 2026 10:44:53 +0000 Subject: [PATCH 012/175] Test: pin the modifier-letter table to the toolchain's Unicode version Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/stores.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs index 1eb6951..1922c44 100644 --- a/crates/weksa/tests/stores.rs +++ b/crates/weksa/tests/stores.rs @@ -394,6 +394,18 @@ fn s6_folds_dotless_i_kana_and_arabic_indic_digits() { .unwrap(); } +/// `modifier_letters.txt` is the Unicode 17 Lm list, the oracle for the test +/// below. When the toolchain's Unicode tables move, this fails first: regenerate +/// the file from the new tables, then bump the pin. +#[test] +fn the_modifier_letter_table_matches_the_toolchains_unicode_version() { + assert_eq!( + char::UNICODE_VERSION, + (17, 0, 0), + "regenerate crates/weksa/tests/modifier_letters.txt for the new Unicode version" + ); +} + #[test] fn a_text_of_only_marks_is_refused() { let d = dir(); From 3a56d3180fbcde6a96338b1332f80ac7125be137 Mon Sep 17 00:00:00 2001 From: Weksa Date: Mon, 9 Nov 2026 11:03:58 +0000 Subject: [PATCH 013/175] =?UTF-8?q?Tests:=20pin=20the=20100-year=20rule,?= =?UTF-8?q?=20day=200,=20ASCII-only=20digits=20in=20dates,=20and=20the=20?= =?UTF-8?q?=E3=83=B6=20fold=20(Soul=20s19=20survivors)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/forms.rs | 20 ++++++++++++++++++++ crates/weksa/tests/stores.rs | 1 + 2 files changed, 21 insertions(+) diff --git a/crates/weksa/tests/forms.rs b/crates/weksa/tests/forms.rs index f0b2469..dba73c8 100644 --- a/crates/weksa/tests/forms.rs +++ b/crates/weksa/tests/forms.rs @@ -147,3 +147,23 @@ fn grade_bound_is_the_type() { assert_eq!(grade["minimum"], 0, "{grade}"); assert_eq!(schema["$defs"]["NodeId"]["pattern"], "^n[1-9][0-9]{0,3}$"); } + +#[test] +fn calendar_dates_follow_the_gregorian_rule_and_ascii_digits_only() { + use weksa::form::is_calendar_date; + for ok in ["2024-02-29", "2000-02-29", "2026-12-31"] { + assert!(is_calendar_date(ok), "{ok}"); + } + for bad in [ + "1900-02-29", + "2023-02-29", + "2026-04-31", + "2026-10-00", + "2026-00-10", + "+026-10-29", + "2026-10-2\u{669}", + "\u{6f0}026-10-29", + ] { + assert!(!is_calendar_date(bad), "{bad}"); + } +} diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs index 1922c44..a8486e7 100644 --- a/crates/weksa/tests/stores.rs +++ b/crates/weksa/tests/stores.rs @@ -358,6 +358,7 @@ fn s6_folds_dotless_i_kana_and_arabic_indic_digits() { "room \u{661}\u{662}\u{663}\u{664}\u{665}\u{666}\u{667}\u{668}\u{669}\u{660}", ), ("gate 45", "gate \u{6f4}\u{6f5}"), + ("\u{3096}", "\u{30f6}"), ]; for (n, (a, b)) in pairs.into_iter().enumerate() { let (first, second) = ( From a300a1ce9e045f68e62b71e1aca9b9817faabe9e Mon Sep 17 00:00:00 2001 From: Weksa Date: Mon, 9 Nov 2026 11:09:46 +0000 Subject: [PATCH 014/175] Test: the asker is the role glossed agent, not the first role (kills request_gate survivor) Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/lower.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/crates/weksa/tests/lower.rs b/crates/weksa/tests/lower.rs index 9e5cccc..3ec94cf 100644 --- a/crates/weksa/tests/lower.rs +++ b/crates/weksa/tests/lower.rs @@ -655,6 +655,34 @@ fn consent_to_one_of_two_requests_leaves_the_other_unconsented() { ); } +#[test] +fn the_asker_is_the_role_the_inventory_glosses_agent_not_the_first_role() { + // role:1 and role:2 swap glosses: "agent" is now role:2, so the asker of + // `asked()` (role:1 filled by the addressee) is the one role:2 names, Nibu. + // A lookup that took the first role, or the first live entry, would read the + // addressee and let the request through. + let dir = tempfile::tempdir().unwrap(); + let mut store = InventoryStore::open(&dir.path().join("i.cc")).unwrap(); + let (rev1, mut entries) = weksa::inventory::seed(); + for e in &mut entries { + if e.id == iv("role:1") { + e.gloss = "patient".into(); + } else if e.id == iv("role:2") { + e.gloss = "agent".into(); + } + } + store.admit_revision(rev1, entries).unwrap(); + let (rev2, entries) = proof_revision(); + store.admit_revision(rev2, entries).unwrap(); + let inv = store.inventory().unwrap(); + assert_eq!( + lower(&admitted(&asked(), &store), &inv, &committed_entities()), + Err(LowerError::AskerNotAddressee { + predication: nd("n4") + }) + ); +} + #[test] fn a_request_by_someone_other_than_the_addressee_is_refused() { let (inv, _dir) = committed(); From 8c94838c303d28969321f0726ee919d87313567d Mon Sep 17 00:00:00 2001 From: Weksa Date: Mon, 9 Nov 2026 11:10:23 +0000 Subject: [PATCH 015/175] =?UTF-8?q?S6=20key:=20voiced=20katakana=20?= =?UTF-8?q?=E3=83=B7-=E3=83=BA=20fold,=20dot=20above=20after=20i=20drops?= =?UTF-8?q?=20(=C4=B0x=20=3D=20i=CC=87x);=20marks-only=20text=20gets=20the?= =?UTF-8?q?=20base-letter=20reason=20first?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/store.rs | 66 +++++++++++++++++++++++++----------- crates/weksa/tests/stores.rs | 39 +++++++++++++++++++-- 2 files changed, 84 insertions(+), 21 deletions(-) diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index f469589..752900e 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -281,12 +281,23 @@ impl CorpusStore { return Err(StoreRefusal::NotNfc { key: item.id }.into()); } let text = normalized(&item.text); - if !text.chars().any(char::is_alphanumeric) { - return Err(malformed(&item.id, "text has no alphanumeric character")); - } // Read from the text as given: NFKC maps modifier letters (Ê°) to base letters. + // A text of marks, joiners and modifiers says so; any other text with + // no alphanumeric character (punctuation, space) gets the plainer reason. if !item.text.chars().any(is_base) { - return Err(malformed(&item.id, "text has no base letter or digit")); + let marks_only = item.text.chars().any(|c| { + is_combining_mark(c) + || matches!(c, '\u{200c}' | '\u{200d}') + || is_modifier_letter(c) + }); + return Err(malformed( + &item.id, + if marks_only { + "text has no base letter or digit" + } else { + "text has no alphanumeric character" + }, + )); } if !is_calendar_date(&item.admitted_on) { return Err(malformed(&item.id, "admitted_on is not a YYYY-MM-DD date")); @@ -415,31 +426,48 @@ fn check_entities(item: &CorpusItem, entities: &EntityStore) -> Result<(), Store /// Turkish dotless i and dotted capital I read as i, katakana reads as the /// hiragana of the same syllable, and Arabic-Indic and Extended Arabic-Indic /// digits read as 0-9. Homoglyphs across scripts (Cyrillic a for Latin a) stay -/// apart: no key here reads what a character looks like. +/// apart: no key here reads what a character looks like. A combining dot above +/// (U+0307) after i is dropped, so Ä°x and i̇x collide as the old case fold made +/// them; so does plain ix, with both. Over-merging a rare sequence errs toward +/// held, a lost collision toward leak. U+0307 after any other letter stays. fn normalized(text: &str) -> String { - let mapped: String = text.nfkc().map(fold_variant).collect(); - default_case_fold_str(&mapped) - .nfkc() - .filter(|c| c.is_alphanumeric() || is_combining_mark(*c)) - .collect() + let mapped: String = text.nfkc().flat_map(fold_variant).collect(); + let mut key = String::new(); + for c in default_case_fold_str(&mapped).nfkc() { + if c == '\u{307}' && key.ends_with('i') { + continue; + } + if c.is_alphanumeric() || is_combining_mark(c) { + key.push(c); + } + } + key } /// The one-to-one folds `normalized` adds to NFKC and case folding, applied /// before the case fold (U+0130 would otherwise fold to i plus a combining dot, /// which the key keeps as a mark). -fn fold_variant(c: char) -> char { - match c { - '\u{131}' => 'i', - '\u{130}' => 'I', +fn fold_variant(c: char) -> impl Iterator { + let one = |c: char| (c, None); + let (first, second) = match c { + '\u{131}' => one('i'), + '\u{130}' => one('I'), + // Voiced katakana wa, wi, we, wo: the hiragana (U+308F-3092) plus the + // combining voiced mark, which is how NFKC spells the hiragana forms. + '\u{30f7}'..='\u{30fa}' => ( + char::from_u32(c as u32 - 0x68).unwrap_or(c), + Some('\u{3099}'), + ), // Katakana letters and small letters to hiragana; the iteration marks // ヽヾ to ã‚ã‚ž. The prolonged sound mark U+30FC stays. '\u{30a1}'..='\u{30f6}' | '\u{30fd}'..='\u{30fe}' => { - char::from_u32(c as u32 - 0x60).unwrap_or(c) + one(char::from_u32(c as u32 - 0x60).unwrap_or(c)) } - '\u{660}'..='\u{669}' => char::from(b'0' + (c as u32 - 0x660) as u8), - '\u{6f0}'..='\u{6f9}' => char::from(b'0' + (c as u32 - 0x6f0) as u8), - _ => c, - } + '\u{660}'..='\u{669}' => one(char::from(b'0' + (c as u32 - 0x660) as u8)), + '\u{6f0}'..='\u{6f9}' => one(char::from(b'0' + (c as u32 - 0x6f0) as u8)), + _ => one(c), + }; + std::iter::once(first).chain(second) } /// A character that can carry a word: alphanumeric and neither a combining diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs index a8486e7..726f26e 100644 --- a/crates/weksa/tests/stores.rs +++ b/crates/weksa/tests/stores.rs @@ -359,6 +359,15 @@ fn s6_folds_dotless_i_kana_and_arabic_indic_digits() { ), ("gate 45", "gate \u{6f4}\u{6f5}"), ("\u{3096}", "\u{30f6}"), + // Voiced wa, wi, we, wo: katakana U+30F7-30FA read as the hiragana + // plus the combining voiced mark. + ("\u{308f}\u{3099}", "\u{30f7}"), + ("\u{3090}\u{3099}", "\u{30f8}"), + ("\u{3091}\u{3099}", "\u{30f9}"), + ("\u{3092}\u{3099}", "\u{30fa}"), + // Dotted capital I and i plus a combining dot above + // read alike (plain ix collides with both, below). + ("\u{130}x", "i\u{307}x"), ]; for (n, (a, b)) in pairs.into_iter().enumerate() { let (first, second) = ( @@ -375,6 +384,15 @@ fn s6_folds_dotless_i_kana_and_arabic_indic_digits() { "{a:?} vs {b:?}" ); } + // Plain ix collides with the dotted spellings too: a rare sequence + // over-merges toward held. + assert_eq!( + refusal(store.add(item("item:fold:80", "ix"), &ents(&d))), + StoreRefusal::S6TextHeld { + key: "item:fold:80".into(), + held_by: "item:fold:27".into() + } + ); // Nothing wider: the prolonged sound mark, Devanagari digits and the // Cyrillic homoglyph of i stay apart from what they resemble. store @@ -390,6 +408,11 @@ fn s6_folds_dotless_i_kana_and_arabic_indic_digits() { .add(item("item:fold:93", "room \u{967}\u{968}"), &ents(&d)) .unwrap(); store.add(item("item:fold:94", "bit"), &ents(&d)).unwrap(); + // A combining dot above after any other letter is kept. + store.add(item("item:fold:96", "x"), &ents(&d)).unwrap(); + store + .add(item("item:fold:97", "\u{1e8b}"), &ents(&d)) + .unwrap(); store .add(item("item:fold:95", "b\u{456}t"), &ents(&d)) .unwrap(); @@ -520,11 +543,23 @@ fn s6_keeps_tone_marks_and_viramas_apart() { .unwrap_or_else(|e| panic!("{b:?} held by {a:?}: {e}")); } // A mark alone is still no text. - for (n, text) in ["\u{e49}", "\u{94d}"].into_iter().enumerate() { + for (n, text) in [ + "\u{e49}", + "\u{94d}", + "\u{301}", + "\u{200d}", + "\u{200c}", + "\u{fe0f}", + "\u{200d}\u{fe0f}\u{301}", + ] + .into_iter() + .enumerate() + { let key = format!("item:mk:{}", 90 + n); assert_eq!( refusal(store.add(item(&key, text), &ents(&d))), - malformed(&key, "text has no alphanumeric character") + malformed(&key, "text has no base letter or digit"), + "{text:?}" ); } } From 461f4f812695f3ebddef66fb3cc21e394dc2a4ea Mon Sep 17 00:00:00 2001 From: Weksa Date: Mon, 9 Nov 2026 11:11:12 +0000 Subject: [PATCH 016/175] Error text stops echoing identifiers: fixed Display for LowerError and Refusal, clap unknown-argument/subcommand rewrite, pronunciation-language echo pinned Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/lower.rs | 49 ++++++++++-- crates/weksa/src/main.rs | 25 ++++++- crates/weksa/src/validate.rs | 81 ++++++++++++++++---- crates/weksa/tests/cli.rs | 85 ++++++++++++++++++++- crates/weksa/tests/display.rs | 135 ++++++++++++++++++++++++++++++++++ 5 files changed, 347 insertions(+), 28 deletions(-) create mode 100644 crates/weksa/tests/display.rs diff --git a/crates/weksa/src/lower.rs b/crates/weksa/src/lower.rs index 619eb41..9f90239 100644 --- a/crates/weksa/src/lower.rs +++ b/crates/weksa/src/lower.rs @@ -161,7 +161,11 @@ pub enum LowerError { /// Phones were requested for an entity whose ruled IPA holds a symbol the /// language's phone inventory does not list (or the language has no /// inventory). The lowerer does not guess a phone. - UnknownPhone { entity: EntityId, at: usize, text: String }, + UnknownPhone { + entity: EntityId, + at: usize, + text: String, + }, /// Phones were requested for an entity whose ruled IPA has a stress mark /// with no vowel after it. DanglingStress { entity: EntityId, at: usize }, @@ -177,7 +181,24 @@ pub enum LowerError { impl fmt::Display for LowerError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{self:?}") + f.write_str(match self { + LowerError::Unrendered { .. } => "packet carries a slot the lowerer does not render", + LowerError::NoRow { .. } => "no table row for an inventory id the packet uses", + LowerError::GlossMismatch { .. } => "table row does not match the inventory's gloss", + LowerError::UnknownRole { .. } => "frame template names a role the inventory lacks", + LowerError::MissingRole { .. } => "predication fills no role its frame needs", + LowerError::UnknownEntity { .. } => "referent cites an entity the port does not hold", + LowerError::NoForm { .. } => "entity has no form in the target language", + LowerError::NoPronunciation { .. } => "entity has no ruled pronunciation", + LowerError::NoEntityReferent => "packet cites no entity", + LowerError::UnknownPhone { .. } => { + "ruled pronunciation holds a symbol with no listed phone" + } + LowerError::DanglingStress { .. } => "stress mark with no vowel after it", + LowerError::ConsentMissing { .. } => "request lacks the addressee's consent", + LowerError::AskerNotAddressee { .. } => "request agent is not the addressee", + LowerError::EntityRead { .. } => "entity port failed to read", + }) } } @@ -324,7 +345,10 @@ pub fn pronounce( input: &LoweringInput, entities: &dyn EntityForms, ) -> Result, LowerError> { - Ok(ruled(input, entities)?.into_iter().map(|(_, ipa)| ipa).collect()) + Ok(ruled(input, entities)? + .into_iter() + .map(|(_, ipa)| ipa) + .collect()) } /// One entity's ruled pronunciation as the phones line a sound consumer reads: @@ -347,15 +371,24 @@ pub fn phones( ruled(input, entities)? .into_iter() .map(|(entity, source)| { - let unknown = |at, text| LowerError::UnknownPhone { entity: entity.clone(), at, text }; + let unknown = |at, text| LowerError::UnknownPhone { + entity: entity.clone(), + at, + text, + }; let inventory = inventory.ok_or_else(|| unknown(0, source.clone()))?; let phones = inventory.segment(&source).map_err(|e| match e { SegmentError::UnknownPhone { at, text } => unknown(at, text), - SegmentError::DanglingStress { at } => { - LowerError::DanglingStress { entity: entity.clone(), at } - } + SegmentError::DanglingStress { at } => LowerError::DanglingStress { + entity: entity.clone(), + at, + }, })?; - Ok(PhoneLine { language: TARGET_LANGUAGE, source, phones }) + Ok(PhoneLine { + language: TARGET_LANGUAGE, + source, + phones, + }) }) .collect() } diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index e923303..d6432a1 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -325,6 +325,7 @@ fn parse_date(s: &str) -> std::result::Result { /// Parses the command line. A rejected value is reported by the flag that took /// it and the reason, never by the value: values are the caller's identifiers. +/// The same holds for the unknown flags and subcommands clap would quote. fn parse_cli() -> Cli { use std::error::Error; match Cli::try_parse() { @@ -345,10 +346,25 @@ fn parse_cli() -> Cli { eprintln!("error: invalid value for {arg}{reason}"); std::process::exit(2); } - Err(e) => e.exit(), + Err(e) => match e.kind() { + ErrorKind::UnknownArgument => refuse_cli("unknown argument"), + ErrorKind::InvalidSubcommand => refuse_cli("unknown subcommand"), + ErrorKind::TooManyValues | ErrorKind::WrongNumberOfValues => { + refuse_cli("wrong number of values") + } + ErrorKind::NoEquals => refuse_cli("value needs '='"), + _ => e.exit(), + }, } } +/// A clap error that would quote the caller's token, as a fixed message that +/// names the kind only. Usage is `--help`'s to print. +fn refuse_cli(what: &str) -> ! { + eprintln!("error: {what}\n\nFor more information, try '--help'."); + std::process::exit(2); +} + fn need(value: Option, flag: &str) -> Result { value.with_context(|| format!("this source needs --{flag}")) } @@ -615,7 +631,12 @@ fn main() -> Result<()> { .map_err(refused)?; println!("admitted {key}"); } - Command::Lower { store, key, ipa, phones } => { + Command::Lower { + store, + key, + ipa, + phones, + } => { let inventory_store = InventoryStore::open(&cli.inventory)?; let inventory = inventory_store.inventory()?; let mut packets = PacketStore::open(&store)?; diff --git a/crates/weksa/src/validate.rs b/crates/weksa/src/validate.rs index e8aaf2f..6a0075f 100644 --- a/crates/weksa/src/validate.rs +++ b/crates/weksa/src/validate.rs @@ -36,15 +36,24 @@ use crate::packet::{ #[derive(Clone, Debug, PartialEq, Eq)] pub enum Refusal { /// V1: the packet's schema_version is not the one these types define. - V1SchemaVersion { found: u32 }, + V1SchemaVersion { + found: u32, + }, /// V2: no referents, predications or relations. V2PayloadRequired, /// V3: a node id repeats across referents and predications. - V3DuplicateNode { id: NodeId }, + V3DuplicateNode { + id: NodeId, + }, /// V4: a reference names no node. - V4DanglingRef { id: NodeId, at: &'static str }, + V4DanglingRef { + id: NodeId, + at: &'static str, + }, /// V5: an inventory id does not resolve at the packet's revision. - V5Unresolved { id: InvId }, + V5Unresolved { + id: InvId, + }, /// V6: an inventory id is of the wrong family for its slot. V6WrongFamily { id: InvId, @@ -52,21 +61,40 @@ pub enum Refusal { at: &'static str, }, /// V7: a feature or extension value does not fit its value kind. - V7ValueKind { what: String }, + V7ValueKind { + what: String, + }, /// V8: a feature is not declared for the slot it sits on. - V8NotApplicable { feature: InvId, slot: Slot }, + V8NotApplicable { + feature: InvId, + slot: Slot, + }, /// V10: the packet's inventory revision has no record. - V10UnknownRevision { rev: u32 }, + V10UnknownRevision { + rev: u32, + }, /// V11: a feature id repeats in one slot's list (`slot` is `None` for an /// extension id repeated in the packet). - V11DuplicateFeature { feature: InvId, slot: Option }, + V11DuplicateFeature { + feature: InvId, + slot: Option, + }, /// V11: a node id repeats in `participants`, `audience` or `preserve` /// (`at` names the list). - V11DuplicateNodeRef { id: NodeId, at: &'static str }, + V11DuplicateNodeRef { + id: NodeId, + at: &'static str, + }, /// V11: a role id repeats in one predication's roles. - V11DuplicateRole { role: InvId, predication: NodeId }, + V11DuplicateRole { + role: InvId, + predication: NodeId, + }, /// V12: more referents, predications and relations than `Packet::MAX_NODES`. - V12TooManyNodes { count: usize, max: usize }, + V12TooManyNodes { + count: usize, + max: usize, + }, /// V13: scene features were given for a packet with no addressee. They /// state the speaker's relation to the addressee, so with none they /// describe nobody. @@ -75,11 +103,15 @@ pub enum Refusal { /// names a node that is not a predication. It is what the addressee asked /// for or agreed to, an act, so a referent cannot be its object. V14ConsentWithoutAddressee, - V14ConsentNotAPredication { id: NodeId }, + V14ConsentNotAPredication { + id: NodeId, + }, /// V15: an addressee consent set by a source that only read wording. /// A decomposition of text, or imported corpus text, sees "you asked her /// to" as words; only authoring or game telemetry saw the request. - V15ConsentSource { source: ProvenanceSource }, + V15ConsentSource { + source: ProvenanceSource, + }, } impl Refusal { @@ -109,7 +141,28 @@ impl Refusal { impl fmt::Display for Refusal { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}: {self:?}", self.rule()) + let what = match self { + Refusal::V1SchemaVersion { .. } => "unsupported schema version", + Refusal::V2PayloadRequired => "packet has no referents, predications or relations", + Refusal::V3DuplicateNode { .. } => "a node id repeats", + Refusal::V4DanglingRef { .. } => "a reference names no node", + Refusal::V5Unresolved { .. } => { + "an inventory id does not resolve at the packet's revision" + } + Refusal::V6WrongFamily { .. } => "an inventory id is of the wrong family for its slot", + Refusal::V7ValueKind { .. } => "a value does not fit its value kind", + Refusal::V8NotApplicable { .. } => "a feature is not declared for its slot", + Refusal::V10UnknownRevision { .. } => "the inventory revision has no record", + Refusal::V11DuplicateFeature { .. } => "a feature id repeats in one list", + Refusal::V11DuplicateNodeRef { .. } => "a node id repeats in one list", + Refusal::V11DuplicateRole { .. } => "a role id repeats in one predication", + Refusal::V12TooManyNodes { .. } => "too many nodes", + Refusal::V13SceneWithoutAddressee => "scene features given without an addressee", + Refusal::V14ConsentWithoutAddressee => "consent given without an addressee", + Refusal::V14ConsentNotAPredication { .. } => "consent names no predication", + Refusal::V15ConsentSource { .. } => "consent set by a source that only read wording", + }; + write!(f, "{}: {what}", self.rule()) } } diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index e588c5e..18edb84 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -279,7 +279,13 @@ fn validate_reports_each_packet() { .unwrap(); let refused = validate(&short, &[]); assert!( - !refused.status.success() && out(&refused).contains("V10"), + !refused.status.success() + && out(&refused).contains("V10: the inventory revision has no record"), + "{}", + out(&refused) + ); + assert!( + !out(&refused).contains("V10UnknownRevision"), "{}", out(&refused) ); @@ -930,10 +936,13 @@ fn proof_packet_lowers_and_the_lowering_scores_against_items() { let args = ["lower", "./p.cc", "packet:6e6962752d303031", "--ipa"]; let unruled = weksa(d.path(), &args); assert!( - !unruled.status.success() && out(&unruled).contains("NoPronunciation"), + !unruled.status.success() && out(&unruled).contains("entity has no ruled pronunciation"), "{}", out(&unruled) ); + for leak in ["NoPronunciation", "entity:nibu", "entity {"] { + assert!(!out(&unruled).contains(leak), "{leak}: {}", out(&unruled)); + } let ruled = weksa( d.path(), &[ @@ -966,7 +975,10 @@ fn proof_packet_lowers_and_the_lowering_scores_against_items() { assert_eq!(String::from_utf8(ipa.stdout).unwrap(), "ˈnibu\n"); // The phones line is the committed fixture, byte for byte, and is a different // output from --ipa, so the two flags cannot be combined. - let phones = weksa(d.path(), &["lower", "./p.cc", "packet:6e6962752d303031", "--phones"]); + let phones = weksa( + d.path(), + &["lower", "./p.cc", "packet:6e6962752d303031", "--phones"], + ); assert!(phones.status.success(), "{}", out(&phones)); let fixture = std::fs::read_to_string( std::path::Path::new(env!("CARGO_MANIFEST_DIR")) @@ -974,7 +986,16 @@ fn proof_packet_lowers_and_the_lowering_scores_against_items() { ) .unwrap(); assert_eq!(String::from_utf8(phones.stdout).unwrap(), fixture); - let both = weksa(d.path(), &["lower", "./p.cc", "packet:6e6962752d303031", "--phones", "--ipa"]); + let both = weksa( + d.path(), + &[ + "lower", + "./p.cc", + "packet:6e6962752d303031", + "--phones", + "--ipa", + ], + ); assert!(!both.status.success(), "{}", out(&both)); let again = weksa(d.path(), &["lower", "./p.cc", "packet:6e6962752d303031"]); assert_eq!(String::from_utf8(again.stdout).unwrap(), text); @@ -1234,3 +1255,59 @@ fn admitted_on_must_be_a_real_calendar_date() { } assert!(!out(&weksa(d.path(), &["corpus", "dump"])).contains("item:")); } + +#[test] +fn clap_errors_never_echo_the_token() { + let d = tempfile::tempdir().unwrap(); + for args in [ + &["--sentinelflagzz"][..], + &["validate", "--sentinelflagzz"], + &["sentinelsubzz"], + &["corpus", "sentinelsubzz"], + &["--inventory"], + ] { + let o = weksa(d.path(), args); + let text = out(&o); + assert_eq!(o.status.code(), Some(2), "{args:?}: {text}"); + assert!(text.contains("error:"), "{args:?}: {text}"); + assert!( + !text.to_lowercase().contains("sentinel"), + "{args:?} echoes the token: {text}" + ); + } +} + +#[test] +fn pronunciation_language_is_never_echoed() { + let d = tempfile::tempdir().unwrap(); + let o = weksa( + d.path(), + &[ + "entity", + "put", + "entity:nibu", + "--form", + "en-US=NIBU", + "--origin", + "cast", + "--persona", + "nibu", + "--fixture", + "f.yaml", + "--pronunciation", + "sentinel-LANG=ˈnibu", + "--ruled-by", + "N", + "--ruled-on", + "2026-11-07", + "--ruling-source", + "t", + ], + ); + let text = out(&o); + assert!( + !o.status.success() && text.contains("names a language that has no --form"), + "{text}" + ); + assert!(!text.to_lowercase().contains("sentinel"), "{text}"); +} diff --git a/crates/weksa/tests/display.rs b/crates/weksa/tests/display.rs new file mode 100644 index 0000000..3c1ba63 --- /dev/null +++ b/crates/weksa/tests/display.rs @@ -0,0 +1,135 @@ +//! Error text is a fixed message naming the rule or variant. The caller's +//! identifiers (entity, node and inventory ids, IPA text, value strings) stay +//! on the typed value and never reach Display. + +use std::collections::HashSet; +use weksa::packet::{EntityId, NodeId, ProvenanceSource}; +use weksa::{Family, InvId, LowerError, Refusal, Slot}; + +const SENTINEL: &str = "sentinelzz"; + +fn inv() -> InvId { + InvId::parse("concept:4242").unwrap() +} +fn node() -> NodeId { + NodeId::parse("n4242").unwrap() +} +fn entity() -> EntityId { + EntityId::parse(&format!("entity:{SENTINEL}")).unwrap() +} + +fn assert_clean_and_distinct(all: Vec) { + let mut seen = HashSet::new(); + for e in all { + let text = e.to_string(); + assert!(!text.is_empty(), "{e:?}"); + for leak in [SENTINEL, "4242", "{", "}"] { + assert!(!text.contains(leak), "{e:?} leaks {leak:?}: {text}"); + } + assert!(seen.insert(text.clone()), "{e:?} repeats {text:?}"); + } +} + +#[test] +fn lower_error_display_names_the_variant_and_no_identifier() { + assert_clean_and_distinct(vec![ + LowerError::Unrendered { what: SENTINEL }, + LowerError::NoRow { id: inv() }, + LowerError::GlossMismatch { + id: inv(), + row: SENTINEL, + }, + LowerError::UnknownRole { + frame: inv(), + role: SENTINEL.into(), + }, + LowerError::MissingRole { + predication: node(), + role: SENTINEL.into(), + }, + LowerError::UnknownEntity { entity: entity() }, + LowerError::NoForm { entity: entity() }, + LowerError::NoPronunciation { entity: entity() }, + LowerError::NoEntityReferent, + LowerError::UnknownPhone { + entity: entity(), + at: 4242, + text: SENTINEL.into(), + }, + LowerError::DanglingStress { + entity: entity(), + at: 4242, + }, + LowerError::ConsentMissing { + predication: node(), + }, + LowerError::AskerNotAddressee { + predication: node(), + }, + LowerError::EntityRead { + entity: entity(), + why: SENTINEL.into(), + }, + ]); + assert_eq!( + LowerError::NoPronunciation { entity: entity() }.to_string(), + "entity has no ruled pronunciation" + ); +} + +#[test] +fn refusal_display_names_the_rule_and_no_identifier() { + let all = vec![ + Refusal::V1SchemaVersion { found: 4242 }, + Refusal::V2PayloadRequired, + Refusal::V3DuplicateNode { id: node() }, + Refusal::V4DanglingRef { + id: node(), + at: SENTINEL, + }, + Refusal::V5Unresolved { id: inv() }, + Refusal::V6WrongFamily { + id: inv(), + expected: Family::Role, + at: SENTINEL, + }, + Refusal::V7ValueKind { + what: SENTINEL.into(), + }, + Refusal::V8NotApplicable { + feature: inv(), + slot: Slot::Scene, + }, + Refusal::V10UnknownRevision { rev: 4242 }, + Refusal::V11DuplicateFeature { + feature: inv(), + slot: Some(Slot::Scene), + }, + Refusal::V11DuplicateNodeRef { + id: node(), + at: SENTINEL, + }, + Refusal::V11DuplicateRole { + role: inv(), + predication: node(), + }, + Refusal::V12TooManyNodes { + count: 4242, + max: 4242, + }, + Refusal::V13SceneWithoutAddressee, + Refusal::V14ConsentWithoutAddressee, + Refusal::V14ConsentNotAPredication { id: node() }, + Refusal::V15ConsentSource { + source: ProvenanceSource::ImportedCorpus, + }, + ]; + for r in &all { + assert!(r.to_string().starts_with(r.rule()), "{r:?}"); + } + assert_clean_and_distinct(all); + assert_eq!( + Refusal::V14ConsentNotAPredication { id: node() }.to_string(), + "V14: consent names no predication" + ); +} From d933a3942aedeae8e5bddd27efee01d8c8db79bc Mon Sep 17 00:00:00 2001 From: Weksa Date: Mon, 9 Nov 2026 11:14:26 +0000 Subject: [PATCH 017/175] Delete clap rewrite arms no CLI flag can reach (Soul s21 low) Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/main.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index d6432a1..d6832e7 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -349,10 +349,6 @@ fn parse_cli() -> Cli { Err(e) => match e.kind() { ErrorKind::UnknownArgument => refuse_cli("unknown argument"), ErrorKind::InvalidSubcommand => refuse_cli("unknown subcommand"), - ErrorKind::TooManyValues | ErrorKind::WrongNumberOfValues => { - refuse_cli("wrong number of values") - } - ErrorKind::NoEquals => refuse_cli("value needs '='"), _ => e.exit(), }, } From a59d0a5e3c457dd673e4b639304b54145abe8acb Mon Sep 17 00:00:00 2001 From: Weksa Date: Mon, 9 Nov 2026 11:15:46 +0000 Subject: [PATCH 018/175] S6 key: drop the dot above i before the last NFKC so following marks compose; test the guard with j (Soul s22) Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/store.rs | 17 ++++++++++------- crates/weksa/tests/stores.rs | 17 +++++++++++++++++ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index 752900e..6739db6 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -432,16 +432,19 @@ fn check_entities(item: &CorpusItem, entities: &EntityStore) -> Result<(), Store /// held, a lost collision toward leak. U+0307 after any other letter stays. fn normalized(text: &str) -> String { let mapped: String = text.nfkc().flat_map(fold_variant).collect(); - let mut key = String::new(); - for c in default_case_fold_str(&mapped).nfkc() { - if c == '\u{307}' && key.ends_with('i') { + // The dot goes before the last NFKC so the marks left on the i compose with + // it (i, dot, acute reads as the i with an acute). + let mut dotless = String::new(); + for c in default_case_fold_str(&mapped).chars() { + if c == '\u{307}' && dotless.ends_with('i') { continue; } - if c.is_alphanumeric() || is_combining_mark(c) { - key.push(c); - } + dotless.push(c); } - key + dotless + .nfkc() + .filter(|&c| c.is_alphanumeric() || is_combining_mark(c)) + .collect() } /// The one-to-one folds `normalized` adds to NFKC and case folding, applied diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs index 726f26e..22ecb83 100644 --- a/crates/weksa/tests/stores.rs +++ b/crates/weksa/tests/stores.rs @@ -416,6 +416,23 @@ fn s6_folds_dotless_i_kana_and_arabic_indic_digits() { store .add(item("item:fold:95", "b\u{456}t"), &ents(&d)) .unwrap(); + // j has no precomposed dot-above form, so this reaches the guard itself. + store.add(item("item:fold:98", "jx"), &ents(&d)).unwrap(); + store + .add(item("item:fold:99", "j\u{307}x"), &ents(&d)) + .unwrap(); + // Marks after the dotted i compose with it: i, dot, acute is an i with an + // acute, whatever order the writer used. + store + .add(item("item:fold:100", "\u{ed}x"), &ents(&d)) + .unwrap(); + assert_eq!( + refusal(store.add(item("item:fold:101", "i\u{307}\u{301}x"), &ents(&d))), + StoreRefusal::S6TextHeld { + key: "item:fold:101".into(), + held_by: "item:fold:100".into() + } + ); } /// `modifier_letters.txt` is the Unicode 17 Lm list, the oracle for the test From ba52b0106965de6563b85753bec1f1482c56c2d7 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 01:47:39 +0000 Subject: [PATCH 019/175] =?UTF-8?q?Map:=20speaker-scene=20facts=20?= =?UTF-8?q?=E2=80=94=20current=20seed,=20persona-state=20question,=204b/4c?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 5.5 --- docs/targets/interlingua-lowering-map.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 5fa3022..5af5612 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -11,7 +11,7 @@ by Imagination for the Eureka-area Self, 2026-10-29. The interlingua spec has two layers. The **schema** is Rust types: packet families, node ids, role slots, value kinds. It changes rarely and by commit. The **inventory** is data in CultCache: the concepts, frames, roles, relation -types, speech acts and nuance features (hedge, certainty, warmth, irony and +types, speech acts and nuance features (hedge, certainty, irony, stance affects and restraint, and whatever the loop finds missing) that the schema's slots name by id. Every open slot in a packet is an inventory id; a referent names an entity by id from the entity store, and a packet's only free text is the trace. No door judges text for leakage: what each model call may see, and what a score may count, own that @@ -133,8 +133,9 @@ standing.** `gamecult.persona_state.v0` (Body facts) carries open trait maps So a `PersonaSnapshot` projects mood-like and relation-like values, and the stable axes the standard lacks are annotated. Whether the standard gains a typed speech-community reference or Weksa keeps the persona-to-community -binding (`data/cultural-ontology/persona-stacks/`) is decided when -`live-lowering` is mapped (follow-up `persona-state-speech-community`); the +binding (`data/cultural-ontology/persona-stacks/`) is question `persona-speech-community-home` (recommended: a Weksa-owned +binding; follow-up `persona-state-speech-community`), decided when +`live-lowering` is mapped; the corpus path needs neither. Putting Weksa feature ids into the open `voiceStyle` trait map is rejected: it would make an extension field portable authority, which the standard forbids. @@ -259,10 +260,10 @@ operator-attested annotations alone. Who annotates is question with the same text and a different source returns `Unchanged` and writes nothing, so a relabel through `add` silently does nothing. `key_slug` parses `n` with `u32::parse`, which accepts `01` and `+1`. -- Inventory rev 1 (seed): features 6-11 are hedge, certainty, warmth, - formality, intensity (Grade) and irony (Flag), slots Discourse and - Predication. Nothing outside the seed and `docs/interlingua-standard.md:179-180` - names warmth, formality or intensity; no committed packet store exists. +- Seed inventory (2026-11-10, base `5e10d7c`): features 6-8 are hedge, certainty + and irony; 9-18 are the stance affects and restraint (slot Stance). The + committed inventory holds revisions 1-2 only. Nothing calls `with_scene` + except tests, and the proof packet's addressee carries no entity. - Yggdrasil's Ollama (`127.0.0.1:11434`) is the Docker container `ollama-cpu`, Huginn's embedder (runbook `gamecult-ops/runbooks/huginn-yggdrasil.md`, ruling `eureka-body:ruling:huginn-embedder-yggdrasil-primary`). `/api/embed` @@ -398,7 +399,7 @@ One line per cut; specs exist for the cuts about to run (2d and 4, siblings on ` 3b. `lowerer-first-cut` (follow-up `golf-loop-lowerer-cut`): the section above. `lower`, `weksa lower`, `weksa score`, `weksa packet proof`, inventory rev 2 for `nibu-proof-001`. 4. `speaker-state`: packet `stance` (felt affect and restraint, slot `Stance`, packet schema v4); formality, warmth and intensity out of the seed and the standard; only `PacketStore::admit` yields a `LoweringInput`, and it refuses packets for withdrawn items (follow-up `lowering-input-from-packet-store`). (spec r1) 4a. `speaker-relation-input`: slot `Scene` and its first seed features (`relative_status`, `intimacy`, `in_group`; inventory revision 3, `scene_revision`); `LoweringInput::with_scene` takes the speaker's relation to the addressee as typed `Scene` feature values, validated like every feature list, refused without an addressee (V13). The English lowerer does not render it. 4b's `weksa.scene` document is its future supplier; no store, document or CLI yet. (PR for follow-up `chat-s3352-3352`) -4b. `speaker-scene`: slots `Speaker` and `Scene` with their seed features; `weksa.speaker` (`speaker:@`) and `weksa.scene` (`scene:@`, participants and ordered-pair relations) documents with `Annotator` `Operator` and `Model { family, generator }`, validated by the packet's shared feature check, append-only per key; their stores, `weksa speaker` and `weksa scene` commands, published schemas; Model-annotated cast speakers and scenes for the committed items. Waits on 2d and 4; `pd-speaker-annotation` blocks only the public-domain annotator. +4b. `speaker-scene` (spec r1; `speaker-scene-seeds` is its cut 4c: CLI, inventory revision 3, cast speakers and scenes; `Party::Unnamed` covers an addressee with no entity; Speaker and Scene share one `SpeakerStateStore`): slots `Speaker` and `Scene` with their seed features; `weksa.speaker` (`speaker:@`) and `weksa.scene` (`scene:@`, participants and ordered-pair relations) documents with `Annotator` `Operator` and `Model { family, generator }`, validated by the packet's shared feature check, append-only per key; their stores, `weksa speaker` and `weksa scene` commands, published schemas; Model-annotated cast speakers and scenes for the committed items. Waits on 2d and 4; `pd-speaker-annotation` blocks only the public-domain annotator. 5. `language-ontology`: `weksa.community` from `data/cultural-ontology/` (owns follow-up `edda-referent-yaml-parse`), `weksa.ontology_entry` for en-US and ja-JP from `data/target-language-ontology/` (YAML deleted), speaker communities, deterministic `select`, the lowering view (packet without trace, speaker, scene, selected entries). Waits on 4. 6. `pd-importer`: public-domain play and fiction dialogue into the corpus with speaker, scene group, edition provenance, and Speaker and Scene annotations by the ruled annotator. Waits on 4, follow-up `gutenberg-terms` and question `pd-speaker-annotation`. 7. `golf-loop-en`: the loop driver, the Train-only context builder for inventory- and ontology-minting calls (follow-up `loop-context-builder-train-only`), the packet store bound to one inventory store (`packet-store-inventory-binding`), `ModelPort` with the `claude -p` adapter, encoder (packet with stance) and lowerer prompts, round records, Score documents, the composite and accept rule, `weksa score`, the detached unit on Yggdrasil. Waits on 3, 5, 6 and follow-up `claude-on-yggdrasil`. From 45071fe34606e98821e8f88936369b2ccdcf7124 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 01:54:47 +0000 Subject: [PATCH 020/175] Delete Aggregate.test and LeakDiagnostic: the round path reads no Test Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/score/diagnostics.rs | 13 +----- crates/weksa/src/score/mod.rs | 6 +-- crates/weksa/tests/score.rs | 62 +++++++++------------------ 3 files changed, 24 insertions(+), 57 deletions(-) diff --git a/crates/weksa/src/score/diagnostics.rs b/crates/weksa/src/score/diagnostics.rs index 9d91cf2..61eee82 100644 --- a/crates/weksa/src/score/diagnostics.rs +++ b/crates/weksa/src/score/diagnostics.rs @@ -56,12 +56,6 @@ pub struct SplitLeak { pub worst: Option, } -#[derive(Clone, Debug, PartialEq, Default)] -pub struct LeakDiagnostic { - pub validation: SplitLeak, - pub test: SplitLeak, -} - /// The distinct word n-grams of a text for each order 1..=ORDERS, over its /// normalised word segments. fn grams(text: &str) -> Vec> { @@ -153,7 +147,7 @@ fn split_leak( /// single-word glosses saturate n = 1, so the leak signature is pooled n >= 2 /// and the worst item at n >= 2. Train items are not measured; items in no /// split are dropped. -pub fn leak_diagnostic(minted: &[&str], items: &[CorpusItem]) -> LeakDiagnostic { +pub fn leak_diagnostic(minted: &[&str], items: &[CorpusItem]) -> SplitLeak { let of = |want: Split| -> Vec<&CorpusItem> { items .iter() @@ -162,8 +156,5 @@ pub fn leak_diagnostic(minted: &[&str], items: &[CorpusItem]) -> LeakDiagnostic }; let train = union(of(Split::Train).iter().map(|i| i.text.as_str())); let minted_grams = union(minted.iter().copied()); - LeakDiagnostic { - validation: split_leak(minted, &train, &minted_grams, &of(Split::Validation)), - test: split_leak(minted, &train, &minted_grams, &of(Split::Test)), - } + split_leak(minted, &train, &minted_grams, &of(Split::Validation)) } diff --git a/crates/weksa/src/score/mod.rs b/crates/weksa/src/score/mod.rs index 713dede..3f52653 100644 --- a/crates/weksa/src/score/mod.rs +++ b/crates/weksa/src/score/mod.rs @@ -12,9 +12,7 @@ mod words; use anyhow::{Result, ensure}; pub use chrf::{CHRF_VERSION, chrf}; -pub use diagnostics::{ - LeakDiagnostic, OrderCoverage, PairChrf, SplitLeak, WorstItem, leak_diagnostic, -}; +pub use diagnostics::{OrderCoverage, PairChrf, SplitLeak, WorstItem, leak_diagnostic}; pub use embed::{Embedder, OllamaEmbedder, cosine}; use crate::corpus::CorpusItem; @@ -115,7 +113,6 @@ pub struct SplitMean { pub struct Aggregate { pub train: SplitMean, pub validation: SplitMean, - pub test: SplitMean, } /// Groups scores by the item's derived split. Items in no split are counted @@ -139,6 +136,5 @@ pub fn aggregate(scored: &[(&CorpusItem, ItemScore)]) -> Aggregate { Aggregate { train: mean(Split::Train), validation: mean(Split::Validation), - test: mean(Split::Test), } } diff --git a/crates/weksa/tests/score.rs b/crates/weksa/tests/score.rs index e8c363a..bbe8d41 100644 --- a/crates/weksa/tests/score.rs +++ b/crates/weksa/tests/score.rs @@ -353,12 +353,6 @@ fn aggregate_by_derived_split() { assert_eq!(agg.train.chrf, Some(15.0)); assert_eq!(agg.train.meaning, Some(0.5), "None is not averaged as zero"); assert_eq!((agg.validation.count, agg.validation.chrf), (1, Some(40.0))); - assert_eq!((agg.test.count, agg.test.chrf), (1, Some(60.0))); - assert_eq!( - agg.test.meaning, - Some(0.6), - "the model-written test item is in no split" - ); assert_eq!(aggregate(&[]).validation.chrf, None); } @@ -385,7 +379,6 @@ fn train(key: &str, text: &str) -> CorpusItem { /// (novel, covered) per order for the Validation split. fn read(minted: &[&str], items: &[CorpusItem]) -> Vec<(usize, usize)> { leak_diagnostic(minted, items) - .validation .coverage_by_n .iter() .map(|c| (c.novel, c.covered)) @@ -414,7 +407,7 @@ fn leak_diagnostic_catches_chunked_glosses() { test_model, ], ); - let pair = d.validation.max_chrf.as_ref().unwrap(); + let pair = d.max_chrf.as_ref().unwrap(); assert!( pair.chrf < 50.0, "each chunk alone scores low: {}", @@ -426,24 +419,21 @@ fn leak_diagnostic_catches_chunked_glosses() { read(&minted, &[held.clone(), unrelated, train_item.clone()]), [(16, 12), (14, 8), (12, 4)] ); - let worst = d.validation.worst.as_ref().unwrap(); + let worst = d.worst.as_ref().unwrap(); assert_eq!( (worst.item.as_str(), worst.n, worst.covered, worst.novel), ("item:v:1", 2, 8, 11) ); - assert_eq!(d.validation.items, 2); - assert_eq!(d.test.items, 0, "a model-written test item is in no split"); - assert_eq!(d.test.max_chrf, None); - assert_eq!(d.test.worst, None); + assert_eq!(d.items, 2); let only_train = leak_diagnostic(&minted, &[train_item]); - assert_eq!(only_train.validation.items, 0); - assert_eq!(only_train.validation.max_chrf, None); - assert_eq!(only_train.validation.worst, None); + assert_eq!(only_train.items, 0); + assert_eq!(only_train.max_chrf, None); + assert_eq!(only_train.worst, None); let full = leak_diagnostic(&[HELD_OUT], &[held]); - assert_eq!(full.validation.coverage_by_n[2].share, Some(1.0)); - assert!(full.validation.max_chrf.unwrap().chrf > 99.0); + assert_eq!(full.coverage_by_n[2].share, Some(1.0)); + assert!(full.max_chrf.unwrap().chrf > 99.0); } #[test] @@ -485,9 +475,9 @@ fn coverage_reads_joined_labels() { "kick the bucket", ] { let d = leak_diagnostic(&[label], &items); - let by_n = &d.validation.coverage_by_n; + let by_n = &d.coverage_by_n; assert_eq!((by_n[2].novel, by_n[2].covered), (3, 1), "{label}"); - let worst = d.validation.worst.unwrap(); + let worst = d.worst.unwrap(); assert_eq!((worst.n, worst.covered), (2, 2), "{label}"); } } @@ -520,7 +510,7 @@ fn coverage_pooled_per_order() { "left", "group", "found", "place", "storm", "after", "warn", "leave", "harbor", ]; let d = leak_diagnostic(&glosses, &items); - let by_n = &d.validation.coverage_by_n; + let by_n = &d.coverage_by_n; assert_eq!(by_n.iter().map(|c| c.n).collect::>(), [1, 2, 3]); assert_eq!((by_n[0].novel, by_n[0].covered), (6, 5)); assert!(by_n[0].share.unwrap() < 1.0); @@ -528,7 +518,7 @@ fn coverage_pooled_per_order() { assert!(c.novel > 0); assert_eq!((c.covered, c.share), (0, Some(0.0))); } - assert_eq!(d.validation.worst, None); + assert_eq!(d.worst, None); // no novel n-gram reads None, never NaN; an empty split reads the same let items = [ @@ -536,16 +526,13 @@ fn coverage_pooled_per_order() { validation("item:v:1", "harbors guard the quiet"), ]; let d = leak_diagnostic(&["anything at all"], &items); - for split in [&d.validation, &d.test] { - assert_eq!(split.coverage_by_n.len(), 3); - assert!( - split - .coverage_by_n - .iter() - .all(|c| (c.novel, c.share) == (0, None)) - ); - assert_eq!(split.worst, None); - } + assert_eq!(d.coverage_by_n.len(), 3); + assert!( + d.coverage_by_n + .iter() + .all(|c| (c.novel, c.share) == (0, None)) + ); + assert_eq!(d.worst, None); } #[test] @@ -569,7 +556,6 @@ fn worst_item_rules() { validation("item:v:2", "delta epsilon"), ]; let w = leak_diagnostic(&["alpha beta", "delta epsilon"], &items) - .validation .worst .unwrap(); assert_eq!( @@ -582,7 +568,6 @@ fn worst_item_rules() { validation("item:v:2", "alpha beta gamma"), ]; let w = leak_diagnostic(&["alpha beta", "delta epsilon"], &items) - .validation .worst .unwrap(); assert_eq!(w.item, "item:v:1"); @@ -592,7 +577,6 @@ fn worst_item_rules() { validation("item:v:2", "alpha beta gamma delta"), ]; let w = leak_diagnostic(&["alpha beta gamma", "delta epsilon"], &items) - .validation .worst .unwrap(); assert_eq!((w.item.as_str(), w.covered, w.novel), ("item:v:2", 2, 3)); @@ -601,17 +585,13 @@ fn worst_item_rules() { validation("item:v:1", "delta epsilon"), validation("item:v:2", "delta epsilon"), ]; - let w = leak_diagnostic(&["delta epsilon"], &items) - .validation - .worst - .unwrap(); + let w = leak_diagnostic(&["delta epsilon"], &items).worst.unwrap(); assert_eq!(w.item, "item:v:1"); let items = [ validation("item:v:1", "alpha beta gamma"), train("item:t:1", "alpha beta"), ]; let w = leak_diagnostic(&["alpha beta gamma"], &items) - .validation .worst .unwrap(); assert_eq!((w.n, w.covered, w.novel), (2, 1, 1)); @@ -631,7 +611,7 @@ fn leak_diagnostic_reports_the_best_pair_and_the_first_of_ties() { let b = validation("item:v:2", HELD_OUT); // the better minted string comes second; equal items tie, the first is kept let d = leak_diagnostic(&["zzzzzz qqqqqq", HELD_OUT], &[a, b]); - let pair = d.validation.max_chrf.unwrap(); + let pair = d.max_chrf.unwrap(); assert_eq!( (pair.minted.as_str(), pair.item.as_str()), (HELD_OUT, "item:v:1") From ce0208f0d0a2af6a81a974b000e653300e8ba567 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 01:55:55 +0000 Subject: [PATCH 021/175] Read every split by SplitScheme::live; CorpusItem::cited_entities owns the cited id set Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/corpus.rs | 14 +++++++ crates/weksa/src/score/diagnostics.rs | 29 +++++++++----- crates/weksa/src/score/mod.rs | 58 +++++++++++++++++---------- crates/weksa/src/split.rs | 12 +++++- crates/weksa/src/store.rs | 4 +- crates/weksa/tests/score.rs | 53 ++++++++++++++++++++++++ crates/weksa/tests/split_schema.rs | 29 ++++++++++++++ 7 files changed, 163 insertions(+), 36 deletions(-) diff --git a/crates/weksa/src/corpus.rs b/crates/weksa/src/corpus.rs index 0beaa17..2604c6a 100644 --- a/crates/weksa/src/corpus.rs +++ b/crates/weksa/src/corpus.rs @@ -97,6 +97,20 @@ pub struct CorpusItem { pub names: Vec, } +impl CorpusItem { + /// The entities a packet for this item may cite (leak-owned-by-flow-and-scorer-3 + /// (3)): the entities its text names, then its speaker, then its prior-turn + /// speakers (which may repeat). `PacketStore::admit` and `EncoderOffer` read + /// only this. + pub fn cited_entities(&self) -> impl Iterator { + self.names + .iter() + .map(|n| &n.entity) + .chain(std::iter::once(&self.speaker)) + .chain(self.prior_turns.iter().map(|t| &t.speaker)) + } +} + /// A byte range `[start, end)` of an item's text that names an entity. The /// store admits it only when both ends sit on extended grapheme cluster /// boundaries (UAX #29) of the NFC text and the range's bytes equal one of the diff --git a/crates/weksa/src/score/diagnostics.rs b/crates/weksa/src/score/diagnostics.rs index 61eee82..54bb10d 100644 --- a/crates/weksa/src/score/diagnostics.rs +++ b/crates/weksa/src/score/diagnostics.rs @@ -139,22 +139,29 @@ fn split_leak( leak } -/// Reads minted text (inventory glosses, option labels, later ontology text) -/// against the held-out items. Per held-out item and for n in 1..=3, its word -/// n-grams that occur in no Train item are its novel n-grams; coverage at each -/// n is the covered over the novel n-grams summed across the split's items -/// (unigrams over all minted strings, longer grams within one string). Honest -/// single-word glosses saturate n = 1, so the leak signature is pooled n >= 2 -/// and the worst item at n >= 2. Train items are not measured; items in no -/// split are dropped. -pub fn leak_diagnostic(minted: &[&str], items: &[CorpusItem]) -> SplitLeak { +/// The held-out split `want` read against minted text: Train items are the +/// live Train items; the readings are over the live items of `want`. +pub(super) fn held_out(minted: &[&str], items: &[CorpusItem], want: Split) -> SplitLeak { let of = |want: Split| -> Vec<&CorpusItem> { items .iter() - .filter(|i| SplitScheme::V1.split(i) == Some(want)) + .filter(|i| SplitScheme::V1.live(i) == Some(want)) .collect() }; let train = union(of(Split::Train).iter().map(|i| i.text.as_str())); let minted_grams = union(minted.iter().copied()); - split_leak(minted, &train, &minted_grams, &of(Split::Validation)) + split_leak(minted, &train, &minted_grams, &of(want)) +} + +/// Reads minted text (inventory glosses, option labels, later ontology text) +/// against the Validation items. Per Validation item and for n in 1..=3, its +/// word n-grams that occur in no Train item are its novel n-grams; coverage at +/// each n is the covered over the novel n-grams summed across the split's +/// items (unigrams over all minted strings, longer grams within one string). +/// Honest single-word glosses saturate n = 1, so the leak signature is pooled +/// n >= 2 and the worst item at n >= 2. Train items are not measured; withdrawn +/// items and items in no split are dropped. Test is read only through +/// `TestReadings`. +pub fn leak_diagnostic(minted: &[&str], items: &[CorpusItem]) -> SplitLeak { + held_out(minted, items, Split::Validation) } diff --git a/crates/weksa/src/score/mod.rs b/crates/weksa/src/score/mod.rs index 3f52653..abf20f9 100644 --- a/crates/weksa/src/score/mod.rs +++ b/crates/weksa/src/score/mod.rs @@ -2,7 +2,9 @@ //! output against the whole reference. Nothing is deleted, hidden or replaced //! before either is read, and the scorer takes no input from a packet, so no //! text the output carries can raise a score except by resembling the -//! reference. No door judges text; the diagnostics report. +//! reference. No door judges text; the diagnostics report. The round path +//! (`aggregate`, `leak_diagnostic`) reads Train and Validation; Test is read +//! only through `TestReadings::at_checkpoint` (convergence-test). mod chrf; mod diagnostics; @@ -109,32 +111,46 @@ pub struct SplitMean { pub meaning: Option, } +/// The round readings a round record may hold. The scorer decides when Test +/// can be read, by type: `Aggregate` has no Test field. +/// +/// ``` +/// let agg = weksa::score::aggregate(&[]); +/// let _ = agg.validation; +/// ``` +/// +/// ```compile_fail +/// let agg = weksa::score::aggregate(&[]); +/// let _ = agg.test; +/// ``` #[derive(Clone, Debug, PartialEq, Default)] pub struct Aggregate { pub train: SplitMean, pub validation: SplitMean, } -/// Groups scores by the item's derived split. Items in no split are counted -/// nowhere. The scorer only groups: the loop decides when Test is read. -pub fn aggregate(scored: &[(&CorpusItem, ItemScore)]) -> Aggregate { - let mean = |want: Split| { - let of: Vec<&ItemScore> = scored - .iter() - .filter(|(item, _)| SplitScheme::V1.split(item) == Some(want)) - .map(|(_, score)| score) - .collect(); - let average = |values: Vec| { - (!values.is_empty()).then(|| values.iter().sum::() / values.len() as f64) - }; - SplitMean { - count: of.len(), - chrf: average(of.iter().map(|s| s.chrf).collect()), - meaning: average(of.iter().filter_map(|s| s.meaning).collect()), - } +/// The mean of the scores of the items read in split `want`. +fn split_mean(scored: &[(&CorpusItem, ItemScore)], want: Split) -> SplitMean { + let of: Vec<&ItemScore> = scored + .iter() + .filter(|(item, _)| SplitScheme::V1.live(item) == Some(want)) + .map(|(_, score)| score) + .collect(); + let average = |values: Vec| { + (!values.is_empty()).then(|| values.iter().sum::() / values.len() as f64) }; - Aggregate { - train: mean(Split::Train), - validation: mean(Split::Validation), + SplitMean { + count: of.len(), + chrf: average(of.iter().map(|s| s.chrf).collect()), + meaning: average(of.iter().filter_map(|s| s.meaning).collect()), + } +} + +/// Groups scores by the split each item is read in. Items read in no split +/// (withdrawn, or model-written in a Test group) are counted nowhere. +pub fn aggregate(scored: &[(&CorpusItem, ItemScore)]) -> Aggregate { + Aggregate { + train: split_mean(scored, Split::Train), + validation: split_mean(scored, Split::Validation), } } diff --git a/crates/weksa/src/split.rs b/crates/weksa/src/split.rs index d2258a2..d325de5 100644 --- a/crates/weksa/src/split.rs +++ b/crates/weksa/src/split.rs @@ -1,5 +1,6 @@ //! The train/validation/test split is derived from an item's group and nothing -//! else. No stored assignment exists, so no writer can reassign one. +//! else. No stored assignment exists, so no writer can reassign one. An item is +//! read in its live split; Test is read only under a Checkpoint. use crate::corpus::{CorpusItem, Source}; @@ -51,4 +52,13 @@ impl SplitScheme { split => Some(split), } } + + /// The split an item is read in; a withdrawn item is read in none. + pub fn live(self, item: &CorpusItem) -> Option { + if item.withdrawn { + None + } else { + self.split(item) + } + } } diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index 6739db6..980b040 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -958,9 +958,7 @@ impl PacketStore { } .into()); } - let on_item = item.names.iter().any(|n| &n.entity == entity) - || &item.speaker == entity - || item.prior_turns.iter().any(|t| &t.speaker == entity); + let on_item = item.cited_entities().any(|e| e == entity); if !on_item { return Err(StoreRefusal::EntityNotOnItem { entity: entity.clone(), diff --git a/crates/weksa/tests/score.rs b/crates/weksa/tests/score.rs index bbe8d41..24a4274 100644 --- a/crates/weksa/tests/score.rs +++ b/crates/weksa/tests/score.rs @@ -822,3 +822,56 @@ fn an_unspaced_script_ends_a_word_beside_any_other_script() { assert_eq!(read(&["\u{4e00}\u{301}\u{200d}", "ab"], &items)[0], (2, 2)); assert_eq!(read(&["\u{4e00}"], &items)[0], (2, 0)); } + +#[test] +fn round_path_reads_no_test() { + let train = item_in("item:a:1", "t", &group_in_bucket(10), cast()); + let validation = item_in("item:b:1", "v", &group_in_bucket(50), cast()); + let mut withdrawn = item_in("item:b:2", "w", &group_in_bucket(50), cast()); + withdrawn.withdrawn = true; + let test = item_in("item:c:1", "x", &group_in_bucket(90), cast()); + let test_two = item_in("item:c:2", "y", &group_in_bucket(90), cast()); + let reading = |test_score: f64, extra: bool| { + let mut scored = vec![ + (&train, score(10.0, Some(0.5))), + (&validation, score(40.0, Some(0.8))), + (&withdrawn, score(99.0, Some(0.99))), + (&test, score(test_score, Some(0.1))), + ]; + if extra { + scored.push((&test_two, score(100.0, Some(1.0)))); + } + aggregate(&scored) + }; + let base = reading(60.0, false); + assert_eq!( + (base.validation.count, base.validation.chrf), + (1, Some(40.0)) + ); + assert_eq!(base.validation.meaning, Some(0.8)); + assert_eq!((base.train.count, base.train.chrf), (1, Some(10.0))); + assert_eq!(reading(5.0, false), base, "a Test score moves nothing"); + assert_eq!(reading(5.0, true), base, "a Test item adds nothing"); +} + +#[test] +fn leak_diagnostic_reads_validation_only() { + let test = item_in("item:t:9", HELD_OUT, &group_in_bucket(90), cast()); + let val = validation("item:v:1", "gardening is quiet work"); + let mut gone = validation("item:v:2", HELD_OUT); + gone.withdrawn = true; + let train_item = train("item:t:1", UNRELATED); + let items = [test, val.clone(), gone, train_item.clone()]; + + let d = leak_diagnostic(&[HELD_OUT], &items); + assert_eq!(d.items, 1, "Validation only, withdrawn dropped"); + assert_ne!(d.max_chrf.as_ref().unwrap().item, "item:t:9"); + assert_eq!(d.max_chrf.as_ref().unwrap().item, "item:v:1"); + assert!(d.max_chrf.as_ref().unwrap().chrf < 50.0); + assert_eq!(d.worst, None); + let without_test = leak_diagnostic(&[HELD_OUT], &[val.clone(), train_item.clone()]); + assert_eq!(d, without_test, "removing the Test item changes nothing"); + + let on_validation = leak_diagnostic(&["gardening is quiet work"], &items); + assert!(on_validation.max_chrf.unwrap().chrf > 99.0); +} diff --git a/crates/weksa/tests/split_schema.rs b/crates/weksa/tests/split_schema.rs index d1e638a..c2829d4 100644 --- a/crates/weksa/tests/split_schema.rs +++ b/crates/weksa/tests/split_schema.rs @@ -151,3 +151,32 @@ fn published_schemas_match() { assert_eq!(held, json, "{name} drifted; run `weksa schema`"); } } + +#[test] +fn live_is_the_split_of_an_item_that_is_not_withdrawn() { + for (bucket, want) in [ + (10, Split::Train), + (50, Split::Validation), + (90, Split::Test), + ] { + let group = group_in_bucket(bucket); + let live = item(&group, cast()); + assert_eq!(SplitScheme::V1.live(&live), Some(want)); + let mut gone = live.clone(); + gone.withdrawn = true; + assert_eq!( + SplitScheme::V1.live(&gone), + None, + "withdrawn: no live split" + ); + assert_eq!( + SplitScheme::V1.split(&gone), + Some(want), + "split still names the group's split" + ); + } + let mut model_test = item(&group_in_bucket(90), model()); + assert_eq!(SplitScheme::V1.live(&model_test), None); + model_test.withdrawn = true; + assert_eq!(SplitScheme::V1.live(&model_test), None); +} From a6dd52774a614da44c65a9e44d6b1e5c91d555aa Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 01:56:29 +0000 Subject: [PATCH 022/175] Checkpoint and TestReadings: Test is read only at a declared checkpoint, scalars only Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/score/mod.rs | 88 ++++++++++++++++++++++++++++++++++- crates/weksa/src/split.rs | 42 +++++++++++++++++ crates/weksa/tests/score.rs | 75 ++++++++++++++++++++++++++++- 3 files changed, 203 insertions(+), 2 deletions(-) diff --git a/crates/weksa/src/score/mod.rs b/crates/weksa/src/score/mod.rs index abf20f9..a5556de 100644 --- a/crates/weksa/src/score/mod.rs +++ b/crates/weksa/src/score/mod.rs @@ -18,7 +18,7 @@ pub use diagnostics::{OrderCoverage, PairChrf, SplitLeak, WorstItem, leak_diagno pub use embed::{Embedder, OllamaEmbedder, cosine}; use crate::corpus::CorpusItem; -use crate::split::{Split, SplitScheme}; +use crate::split::{Checkpoint, Split, SplitScheme}; /// One reference/output pair; both are scored whole. pub struct ScoreInput<'a> { @@ -154,3 +154,89 @@ pub fn aggregate(scored: &[(&CorpusItem, ItemScore)]) -> Aggregate { validation: split_mean(scored, Split::Validation), } } + +/// The Test split's readings, scalars only: no item key, no minted string, no +/// pair. The checkpoint decision (convergence-test) needs numbers; a key beside +/// a minted string is the handle an edit step would act on. The only producer +/// is `at_checkpoint`, which consumes a `Checkpoint`. +/// +/// ``` +/// use weksa::score::TestReadings; +/// use weksa::split::Checkpoint; +/// fn s(_: &T) {} +/// let t = TestReadings::at_checkpoint(Checkpoint::declare(1), &[], &[], &[]); +/// assert_eq!(t.count(), 0); +/// s(&0u8); +/// ``` +/// +/// ```compile_fail +/// use weksa::score::TestReadings; +/// use weksa::split::Checkpoint; +/// let t = TestReadings::at_checkpoint(Checkpoint::declare(1), &[], &[], &[]); +/// let _u = t.clone(); +/// ``` +/// +/// ```compile_fail +/// use weksa::score::TestReadings; +/// use weksa::split::Checkpoint; +/// let t = TestReadings::at_checkpoint(Checkpoint::declare(1), &[], &[], &[]); +/// let _ = format!("{t:?}"); +/// ``` +/// +/// ```compile_fail +/// use weksa::score::TestReadings; +/// use weksa::split::Checkpoint; +/// fn s(_: &T) {} +/// let t = TestReadings::at_checkpoint(Checkpoint::declare(1), &[], &[], &[]); +/// s(&t); +/// ``` +pub struct TestReadings { + mean: SplitMean, + leak: SplitLeak, +} + +impl TestReadings { + /// Reads the live Test items against the scores and the minted strings, + /// by the same `split_leak` method the Validation reading uses. The + /// checkpoint is spent. + pub fn at_checkpoint( + checkpoint: Checkpoint, + scored: &[(&CorpusItem, ItemScore)], + minted: &[&str], + items: &[CorpusItem], + ) -> TestReadings { + let _spent = checkpoint; + TestReadings { + mean: split_mean(scored, Split::Test), + leak: diagnostics::held_out(minted, items, Split::Test), + } + } + + /// Scored Test items. + pub fn count(&self) -> usize { + self.mean.count + } + + pub fn chrf(&self) -> Option { + self.mean.chrf + } + + pub fn meaning(&self) -> Option { + self.mean.meaning + } + + /// Test items in the leak reading. + pub fn items(&self) -> usize { + self.leak.items + } + + /// The highest chrF of any minted string against any Test item; the score + /// only, never the pair. + pub fn max_chrf(&self) -> Option { + self.leak.max_chrf.as_ref().map(|p| p.chrf) + } + + pub fn coverage_by_n(&self) -> &[OrderCoverage] { + &self.leak.coverage_by_n + } +} diff --git a/crates/weksa/src/split.rs b/crates/weksa/src/split.rs index d325de5..37e4006 100644 --- a/crates/weksa/src/split.rs +++ b/crates/weksa/src/split.rs @@ -62,3 +62,45 @@ impl SplitScheme { } } } + +/// Permission to read Test once. The loop declares one at a convergence +/// checkpoint (convergence-test); `TestReadings::at_checkpoint` consumes it, so +/// one declaration buys one Test reading, and `EncoderOffer::for_checkpoint` +/// borrows it first. +/// +/// ``` +/// use weksa::score::TestReadings; +/// use weksa::split::Checkpoint; +/// let c = Checkpoint::declare(3); +/// assert_eq!(c.round(), 3); +/// let _ = TestReadings::at_checkpoint(c, &[], &[], &[]); +/// ``` +/// +/// One declaration, one reading: +/// +/// ```compile_fail +/// use weksa::score::TestReadings; +/// use weksa::split::Checkpoint; +/// let c = Checkpoint::declare(3); +/// let _ = TestReadings::at_checkpoint(c, &[], &[], &[]); +/// let _ = TestReadings::at_checkpoint(c, &[], &[], &[]); +/// ``` +/// +/// ```compile_fail +/// use weksa::split::Checkpoint; +/// let c = Checkpoint::declare(3); +/// let _again = c.clone(); +/// ``` +pub struct Checkpoint { + round: u32, +} + +impl Checkpoint { + pub fn declare(round: u32) -> Self { + Self { round } + } + + pub fn round(&self) -> u32 { + self.round + } +} diff --git a/crates/weksa/tests/score.rs b/crates/weksa/tests/score.rs index 24a4274..32e9bfa 100644 --- a/crates/weksa/tests/score.rs +++ b/crates/weksa/tests/score.rs @@ -10,9 +10,10 @@ use common::{cast, group_in_bucket, item_in, model}; use serde_json::Value; use weksa::CorpusItem; use weksa::score::{ - Embedder, ItemScore, OllamaEmbedder, ScoreInput, Scorer, aggregate, chrf, cosine, + Embedder, ItemScore, OllamaEmbedder, ScoreInput, Scorer, TestReadings, aggregate, chrf, cosine, leak_diagnostic, }; +use weksa::split::Checkpoint; /// (reference, hypothesis, sacrebleu 2.4.3 sentence_chrf), map 491536c. const VECTORS: [(&str, &str, f64); 6] = [ @@ -875,3 +876,75 @@ fn leak_diagnostic_reads_validation_only() { let on_validation = leak_diagnostic(&["gardening is quiet work"], &items); assert!(on_validation.max_chrf.unwrap().chrf > 99.0); } + +#[test] +fn test_readings_at_checkpoint() { + let train_item = train("item:t:1", UNRELATED); + let test = item_in("item:c:1", HELD_OUT, &group_in_bucket(90), cast()); + let test_model = item_in( + "item:c:2", + "model words here", + &group_in_bucket(90), + model(), + ); + let mut gone = item_in("item:c:3", "withdrawn words", &group_in_bucket(90), cast()); + gone.withdrawn = true; + let val = validation("item:v:1", "gardening is quiet work"); + let items = [ + train_item.clone(), + test.clone(), + test_model.clone(), + gone.clone(), + val.clone(), + ]; + let scored = [ + (&train_item, score(10.0, Some(0.5))), + (&val, score(40.0, Some(0.8))), + (&test, score(60.0, Some(0.6))), + (&test_model, score(100.0, Some(1.0))), + (&gone, score(90.0, Some(0.9))), + ]; + let t = TestReadings::at_checkpoint(Checkpoint::declare(2), &scored, &[HELD_OUT], &items); + assert_eq!( + (t.count(), t.chrf(), t.meaning()), + (1, Some(60.0), Some(0.6)) + ); + assert_eq!( + t.items(), + 1, + "model-written and withdrawn Test items are excluded" + ); + assert!(t.max_chrf().unwrap() > 99.0); + assert_eq!(t.coverage_by_n()[2].share, Some(1.0)); + + // A minted string that is not the Test text reads low; the same method as + // the Validation reading, so the same minted text reads alike in each. + let t = TestReadings::at_checkpoint(Checkpoint::declare(2), &scored, &["zzz"], &items); + assert!(t.max_chrf().unwrap() < 20.0); + assert_eq!(t.coverage_by_n()[2].share, Some(0.0)); + + let empty = TestReadings::at_checkpoint(Checkpoint::declare(2), &[], &[HELD_OUT], &[]); + assert_eq!( + (empty.count(), empty.chrf(), empty.meaning()), + (0, None, None) + ); + assert_eq!((empty.items(), empty.max_chrf()), (0, None)); +} + +#[test] +fn validation_and_test_leak_agree_in_method() { + // The same text and minted strings, once in Validation and once in Test, + // read the same numbers. + let minted = chunks(3, " "); + let minted = refs(&minted); + let train_item = train("item:t:1", UNRELATED); + let v = leak_diagnostic( + &minted, + &[validation("item:v:1", HELD_OUT), train_item.clone()], + ); + let test = item_in("item:c:1", HELD_OUT, &group_in_bucket(90), cast()); + let t = TestReadings::at_checkpoint(Checkpoint::declare(1), &[], &minted, &[test, train_item]); + assert_eq!(t.items(), v.items); + assert_eq!(t.max_chrf(), v.max_chrf.as_ref().map(|p| p.chrf)); + assert_eq!(t.coverage_by_n(), v.coverage_by_n.as_slice()); +} From 3bd4a376459fa711cf300677be266221045fd2c8 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 01:57:40 +0000 Subject: [PATCH 023/175] context module: TrainItems, EncoderOffer and Prompt are the only road from corpus text to a model call Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/context.rs | 208 ++++++++++++++++++++++++ crates/weksa/src/lib.rs | 1 + crates/weksa/tests/context.rs | 298 ++++++++++++++++++++++++++++++++++ 3 files changed, 507 insertions(+) create mode 100644 crates/weksa/src/context.rs create mode 100644 crates/weksa/tests/context.rs diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs new file mode 100644 index 0000000..13fce61 --- /dev/null +++ b/crates/weksa/src/context.rs @@ -0,0 +1,208 @@ +//! The one place prompt text is built from corpus text. Corpus text reaches a +//! model call only as a [`Prompt`], and a `Prompt` is made only by minting from +//! [`TrainItems`] or by encoding one item an [`EncoderOffer`] holds: Train-only +//! minting and per-item encoding (leak-owned-by-flow-and-scorer-3, +//! loop-context-builder-2). Instruction wording is not written here; a caller +//! passes `&'static str`, so the templates are fixed at compile time and hence +//! within a run. + +use crate::corpus::CorpusItem; +use crate::packet::EntityId; +use crate::split::{Checkpoint, Split, SplitScheme}; +use crate::store::CorpusStore; + +/// The live Train items of a corpus, in key order. The only input minting +/// reads; it cannot be built from a slice, and no accessor returns text or an +/// item. +/// +/// ``` +/// let dir = tempfile::tempdir().unwrap(); +/// let store = weksa::CorpusStore::open(&dir.path().join("c.cc")).unwrap(); +/// let t = weksa::context::TrainItems::from_corpus(&store).unwrap(); +/// assert!(t.is_empty()); +/// ``` +/// +/// ```compile_fail +/// let t = weksa::context::TrainItems { items: vec![] }; +/// ``` +/// +/// ```compile_fail +/// let dir = tempfile::tempdir().unwrap(); +/// let store = weksa::CorpusStore::open(&dir.path().join("c.cc")).unwrap(); +/// let t = weksa::context::TrainItems::from_corpus(&store).unwrap(); +/// let _u = t.clone(); +/// ``` +/// +/// ```compile_fail +/// fn s(_: &T) {} +/// let dir = tempfile::tempdir().unwrap(); +/// let store = weksa::CorpusStore::open(&dir.path().join("c.cc")).unwrap(); +/// let t = weksa::context::TrainItems::from_corpus(&store).unwrap(); +/// s(&t); +/// ``` +pub struct TrainItems { + items: Vec, +} + +impl TrainItems { + pub fn from_corpus(corpus: &CorpusStore) -> anyhow::Result { + let items = corpus + .all()? + .into_iter() + .filter(|item| SplitScheme::V1.live(item) == Some(Split::Train)) + .collect(); + Ok(Self { items }) + } + + pub fn len(&self) -> usize { + self.items.len() + } + + pub fn is_empty(&self) -> bool { + self.items.is_empty() + } + + pub fn keys(&self) -> impl Iterator { + self.items.iter().map(|item| item.id.as_str()) + } +} + +/// Why an item is not offered to the encoder. +#[derive(Debug, PartialEq, Eq)] +pub enum OfferRefusal { + Withdrawn, + /// A model-written item of a Test group is in no split. + NoSplit, + /// A Test item is offered only under a [`Checkpoint`]. + TestBeforeCheckpoint, +} + +/// One item the encoder may be shown, with the entities a packet for it may +/// cite. Built per item, so a call sees one item and nothing of its group. +pub struct EncoderOffer { + item: CorpusItem, + entities: Vec, +} + +impl EncoderOffer { + /// A round offers Train and Validation items. + pub fn for_round(item: &CorpusItem) -> Result { + match Self::gate(item)? { + Split::Test => Err(OfferRefusal::TestBeforeCheckpoint), + _ => Ok(Self::of(item)), + } + } + + /// A convergence checkpoint offers any live item, Test included. + pub fn for_checkpoint( + item: &CorpusItem, + _checkpoint: &Checkpoint, + ) -> Result { + Self::gate(item)?; + Ok(Self::of(item)) + } + + fn gate(item: &CorpusItem) -> Result { + if item.withdrawn { + return Err(OfferRefusal::Withdrawn); + } + SplitScheme::V1.live(item).ok_or(OfferRefusal::NoSplit) + } + + fn of(item: &CorpusItem) -> Self { + let mut entities: Vec = item.cited_entities().cloned().collect(); + entities.sort(); + entities.dedup(); + Self { + item: item.clone(), + entities, + } + } + + pub fn key(&self) -> &str { + &self.item.id + } + + /// The item's own cited entities, sorted, without repeats. + pub fn entities(&self) -> &[EntityId] { + &self.entities + } +} + +/// Text for a model call, and the keys of the items it holds. The only +/// constructors are [`Prompt::mint`] and [`Prompt::encode`]. +/// +/// ``` +/// let dir = tempfile::tempdir().unwrap(); +/// let store = weksa::CorpusStore::open(&dir.path().join("c.cc")).unwrap(); +/// let t = weksa::context::TrainItems::from_corpus(&store).unwrap(); +/// let p = weksa::context::Prompt::mint("Mint.", &t); +/// assert_eq!(p.text(), "Mint."); +/// ``` +/// +/// Instructions are compile-time text: +/// +/// ```compile_fail +/// let dir = tempfile::tempdir().unwrap(); +/// let store = weksa::CorpusStore::open(&dir.path().join("c.cc")).unwrap(); +/// let t = weksa::context::TrainItems::from_corpus(&store).unwrap(); +/// let p = weksa::context::Prompt::mint(&String::from("x"), &t); +/// ``` +/// +/// ```compile_fail +/// fn s(_: &T) {} +/// let dir = tempfile::tempdir().unwrap(); +/// let store = weksa::CorpusStore::open(&dir.path().join("c.cc")).unwrap(); +/// let t = weksa::context::TrainItems::from_corpus(&store).unwrap(); +/// let p = weksa::context::Prompt::mint("x", &t); +/// s(&p); +/// ``` +pub struct Prompt { + text: String, + items: Vec, +} + +fn block(item: &CorpusItem) -> String { + let mut lines: Vec = item + .prior_turns + .iter() + .map(|t| format!("{}: {}", t.speaker, t.text)) + .collect(); + lines.push(format!("[{}] {}: {}", item.id, item.speaker, item.text)); + lines.join("\n") +} + +impl Prompt { + /// Instructions, then every Train item in key order. + pub fn mint(instructions: &'static str, train: &TrainItems) -> Prompt { + let mut parts = vec![instructions.to_string()]; + parts.extend(train.items.iter().map(block)); + Prompt { + text: parts.join("\n\n"), + items: train.keys().map(str::to_string).collect(), + } + } + + /// Instructions, the one offered item, and the entities a packet for it + /// may cite. + pub fn encode(instructions: &'static str, offer: &EncoderOffer) -> Prompt { + let ids: Vec<&str> = offer.entities.iter().map(EntityId::as_str).collect(); + Prompt { + text: format!( + "{instructions}\n\n{}\n\nentities: {}", + block(&offer.item), + ids.join(", ") + ), + items: vec![offer.item.id.clone()], + } + } + + pub fn text(&self) -> &str { + &self.text + } + + /// The keys of the items the text holds; what a round record lists. + pub fn item_keys(&self) -> &[String] { + &self.items + } +} diff --git a/crates/weksa/src/lib.rs b/crates/weksa/src/lib.rs index d826c0e..bb42311 100644 --- a/crates/weksa/src/lib.rs +++ b/crates/weksa/src/lib.rs @@ -3,6 +3,7 @@ //! train/validation/test split, the stores that admit them, and the scorer //! (chrF, meaning cosine, leak diagnostics). +pub mod context; pub mod corpus; pub mod daodejing; pub mod entity; diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs new file mode 100644 index 0000000..b499ee1 --- /dev/null +++ b/crates/weksa/tests/context.rs @@ -0,0 +1,298 @@ +mod common; + +use common::{cast, ent, entities_in, group_in_bucket, item_in, model, world}; +use weksa::context::{EncoderOffer, OfferRefusal, Prompt, TrainItems}; +use weksa::split::Checkpoint; +use weksa::{ + CorpusItem, CorpusItemId, CorpusStore, NameSpan, PacketId, PacketStore, StoreError, + StoreRefusal, Turn, +}; + +const MINT: &str = "Mint glosses from the lines below."; +const ENCODE: &str = "Encode the line below."; + +struct Corpus { + _dir: tempfile::TempDir, + store: CorpusStore, +} + +/// One live item per split, a withdrawn Train item and a model-written +/// Test-group item, each with a marker word unique to it. +fn corpus() -> Corpus { + let dir = tempfile::tempdir().unwrap(); + let entities = entities_in(dir.path()); + let mut store = CorpusStore::open(&dir.path().join("c.cc")).unwrap(); + let mut put = |item: CorpusItem| { + let withdrawn = item.withdrawn; + let id = item.id.clone(); + store + .add( + CorpusItem { + withdrawn: false, + ..item + }, + &entities, + ) + .unwrap(); + if withdrawn { + store.withdraw(&id).unwrap(); + } + }; + put(item_in( + "item:a:1", + "alphamarker speaks", + &group_in_bucket(10), + cast(), + )); + put(item_in( + "item:b:1", + "betamarker speaks", + &group_in_bucket(50), + cast(), + )); + put(item_in( + "item:c:1", + "gammamarker speaks", + &group_in_bucket(90), + cast(), + )); + let mut gone = item_in( + "item:a:2", + "deltamarker speaks", + &group_in_bucket(10), + cast(), + ); + gone.withdrawn = true; + put(gone); + put(item_in( + "item:c:2", + "epsilonmarker speaks", + &group_in_bucket(90), + model(), + )); + Corpus { _dir: dir, store } +} + +fn get(c: &Corpus, key: &str) -> CorpusItem { + c.store.get(key).unwrap().unwrap() +} + +#[test] +fn train_items_hold_only_live_train() { + let c = corpus(); + let t = TrainItems::from_corpus(&c.store).unwrap(); + assert_eq!(t.keys().collect::>(), ["item:a:1"]); + assert_eq!((t.len(), t.is_empty()), (1, false)); +} + +#[test] +fn train_items_are_in_key_order() { + let dir = tempfile::tempdir().unwrap(); + let entities = entities_in(dir.path()); + let mut store = CorpusStore::open(&dir.path().join("c.cc")).unwrap(); + for (key, text) in [ + ("item:z:1", "zed line"), + ("item:a:1", "ay line"), + ("item:m:1", "em line"), + ] { + store + .add(item_in(key, text, &group_in_bucket(10), cast()), &entities) + .unwrap(); + } + let t = TrainItems::from_corpus(&store).unwrap(); + assert_eq!( + t.keys().collect::>(), + ["item:a:1", "item:m:1", "item:z:1"] + ); +} + +#[test] +fn mint_prompt_holds_train_text_only() { + let c = corpus(); + let t = TrainItems::from_corpus(&c.store).unwrap(); + let p = Prompt::mint(MINT, &t); + assert!(p.text().starts_with(MINT)); + assert!(p.text().contains("alphamarker")); + for other in ["betamarker", "gammamarker", "deltamarker", "epsilonmarker"] { + assert!(!p.text().contains(other), "{other}"); + } + assert_eq!(p.item_keys(), ["item:a:1"]); +} + +#[test] +fn prompts_render_deterministically() { + let dir = tempfile::tempdir().unwrap(); + let entities = entities_in(dir.path()); + let mut store = CorpusStore::open(&dir.path().join("c.cc")).unwrap(); + let mut one = item_in("item:a:1", "Hello there.", &group_in_bucket(10), cast()); + one.prior_turns = vec![Turn { + speaker: common::eid("brokkr"), + text: "Well?".into(), + }]; + store.add(one, &entities).unwrap(); + store + .add( + item_in("item:a:2", "Second line.", &group_in_bucket(10), cast()), + &entities, + ) + .unwrap(); + let t = TrainItems::from_corpus(&store).unwrap(); + assert_eq!( + Prompt::mint(MINT, &t).text(), + format!( + "{MINT}\n\nentity:brokkr: Well?\n[item:a:1] entity:nibu: Hello there.\n\n[item:a:2] entity:nibu: Second line." + ) + ); + assert_eq!(Prompt::mint(MINT, &t).item_keys(), ["item:a:1", "item:a:2"]); + + let offer = EncoderOffer::for_round(&store.get("item:a:2").unwrap().unwrap()).unwrap(); + assert_eq!( + Prompt::encode(ENCODE, &offer).text(), + format!("{ENCODE}\n\n[item:a:2] entity:nibu: Second line.\n\nentities: entity:nibu") + ); + assert_eq!(Prompt::encode(ENCODE, &offer).item_keys(), ["item:a:2"]); +} + +#[test] +fn encoder_offer_split_gate() { + let c = corpus(); + let cp = Checkpoint::declare(1); + let train = get(&c, "item:a:1"); + let validation = get(&c, "item:b:1"); + let test = get(&c, "item:c:1"); + let no_split = get(&c, "item:c:2"); + let withdrawn = get(&c, "item:a:2"); + assert!(withdrawn.withdrawn); + let round = |i: &CorpusItem| EncoderOffer::for_round(i).err(); + let check = |i: &CorpusItem| EncoderOffer::for_checkpoint(i, &cp).err(); + + assert_eq!(round(&train), None); + assert_eq!(round(&validation), None); + assert_eq!(round(&test), Some(OfferRefusal::TestBeforeCheckpoint)); + assert_eq!(round(&no_split), Some(OfferRefusal::NoSplit)); + assert_eq!(round(&withdrawn), Some(OfferRefusal::Withdrawn)); + + assert_eq!(check(&train), None); + assert_eq!(check(&validation), None); + assert_eq!(check(&test), None); + assert_eq!(check(&no_split), Some(OfferRefusal::NoSplit)); + assert_eq!(check(&withdrawn), Some(OfferRefusal::Withdrawn)); + + // Withdrawn wins over NoSplit. + let mut both = no_split.clone(); + both.withdrawn = true; + assert_eq!(round(&both), Some(OfferRefusal::Withdrawn)); + assert_eq!(check(&both), Some(OfferRefusal::Withdrawn)); + // A withdrawn Test item is Withdrawn, not TestBeforeCheckpoint. + let mut test_gone = test.clone(); + test_gone.withdrawn = true; + assert_eq!(round(&test_gone), Some(OfferRefusal::Withdrawn)); +} + +fn span(slug: &str, start: u32, end: u32) -> NameSpan { + NameSpan { + entity: common::eid(slug), + start, + end, + } +} + +/// An item naming kay (e1), spoken by hero (e2) after a turn by brokkr (e3), +/// and a sibling of the same group naming someone (e4). +fn cited_world() -> (Corpus, CorpusItem, CorpusItem) { + let dir = tempfile::tempdir().unwrap(); + let mut entities = entities_in(dir.path()); + entities.put(ent("kay", "en-US", &["Kay"])).unwrap(); + let mut store = CorpusStore::open(&dir.path().join("c.cc")).unwrap(); + let group = group_in_bucket(10); + let mut mine = item_in("item:a:1", "Kay waits here.", &group, cast()); + mine.speaker = common::eid("hero"); + mine.names = vec![span("kay", 0, 3)]; + mine.prior_turns = vec![ + Turn { + speaker: common::eid("brokkr"), + text: "Where?".into(), + }, + Turn { + speaker: common::eid("brokkr"), + text: "Again?".into(), + }, + ]; + let mut sibling = item_in("item:a:2", "Someone waits there.", &group, cast()); + sibling.names = vec![span("someone", 0, 7)]; + store.add(mine.clone(), &entities).unwrap(); + store.add(sibling.clone(), &entities).unwrap(); + (Corpus { _dir: dir, store }, mine, sibling) +} + +#[test] +fn encoder_offer_entities_are_the_items_own() { + let (_c, mine, sibling) = cited_world(); + assert_eq!( + mine.cited_entities() + .map(|e| e.as_str()) + .collect::>(), + [ + "entity:kay", + "entity:hero", + "entity:brokkr", + "entity:brokkr" + ], + "names, speaker, prior-turn speakers, repeats kept" + ); + let offer = EncoderOffer::for_round(&mine).unwrap(); + let ids: Vec<&str> = offer.entities().iter().map(|e| e.as_str()).collect(); + assert_eq!(ids, ["entity:brokkr", "entity:hero", "entity:kay"]); + assert!( + !ids.contains(&"entity:someone"), + "a sibling's entity is not the item's" + ); + assert_eq!(offer.key(), "item:a:1"); + let p = Prompt::encode(ENCODE, &offer); + assert!( + p.text() + .ends_with("entities: entity:brokkr, entity:hero, entity:kay") + ); + assert!(p.text().contains("[item:a:1]")); + assert!(!p.text().contains("item:a:2") && !p.text().contains("Someone waits")); + assert_eq!(p.item_keys(), ["item:a:1"]); + let sib = EncoderOffer::for_round(&sibling).unwrap(); + assert_eq!(sib.entities().len(), 2); +} + +#[test] +fn offer_and_store_agree() { + let mut w = world(); + w.entities.put(ent("kay", "en-US", &["Kay"])).unwrap(); + let mut item = item_in( + "item:nibu:2", + "Kay waits here.", + &group_in_bucket(10), + cast(), + ); + item.speaker = common::eid("hero"); + item.names = vec![span("kay", 0, 3)]; + item.prior_turns = vec![Turn { + speaker: common::eid("brokkr"), + text: "Where?".into(), + }]; + w.corpus.add(item.clone(), &w.entities).unwrap(); + let offer = EncoderOffer::for_round(&item).unwrap(); + + let mut packets = PacketStore::open(&w.dir.path().join("p.cc")).unwrap(); + let all = ["nibu", "hero", "kay", "brokkr", "player", "s", "someone"]; + for (n, slug) in all.iter().enumerate() { + let mut p = common::good(); + p.id = PacketId::parse(&format!("packet:{n:016x}")).unwrap(); + p.provenance.corpus_item = CorpusItemId::parse("item:nibu:2").unwrap(); + p.referents[0].entity = Some(common::eid(slug)); + let offered = offer.entities().contains(&common::eid(slug)); + match packets.admit(p, &w.inventory, &w.corpus, &w.entities) { + Ok(_) => assert!(offered, "{slug} admitted but not offered"), + Err(StoreError::Refused(StoreRefusal::EntityNotOnItem { .. })) => { + assert!(!offered, "{slug} offered but refused") + } + Err(other) => panic!("{slug}: {other:?}"), + } + } +} From b0b5728f0712a8d42f829053fb61982c73c05595 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 01:57:47 +0000 Subject: [PATCH 024/175] Map: cut 7a loop-context-builder; item 3 names the checkpoint-only Test path; item 7 drops the context builder Co-Authored-By: Claude Sonnet 5.5 --- docs/targets/interlingua-lowering-map.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 5fa3022..031103d 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -393,7 +393,7 @@ One line per cut; specs exist for the cuts about to run (2d and 4, siblings on ` 2. `packet-store`: Rust crate with packet, inventory and corpus types, the validator, the derived split, the stores and the `weksa` CLI. Landed at `28284cc`. 2b. Landed. `packet-store-corpus`: corpus labels tell the truth (`corpus-labels-tell-truth`): `ModelFamily` and the generator string on `ModelWritten`; `item:nibu:1` ModelWritten (Unrecorded); `item:brokkr:1` and its fixture carry `codex-local-source-aware-draft`, family OpenAi; intake refuses aliased keys, a whole item changed under its key, and a held item's text under a new key; corpus.cc reseeded. Speaker state is not in 2b: it lives in Speaker and Scene documents and the packet's stance, not on the item. Waits on 2. (spec) 2c. Landed. `entity-store`: `ProperName` and `Referent.name` deleted; `EntityId`, `Referent.entity`, the `weksa.entity` document and its store; corpus name spans and entity-id speakers; the packet store's entity checks; seed entities and spans for the cast items; schema v2. Waits on 2b. (spec r1) -3. Landed at `869c82b`. `scorer`: chrF, meaning cosine behind an `Embedder` port (Ollama adapter with `num_thread`), leak diagnostics, aggregation by derived split; library only, `weksa score` comes with the loop. h2 landed the r2 fix pass on its branch; r3 deletes the name mask (the scorer reads output and reference only) and pools novel coverage per order over word keys that split joined labels. Waits on 2c for merge order. (spec r3) +3. Landed at `869c82b`. `scorer`: chrF, meaning cosine behind an `Embedder` port (Ollama adapter with `num_thread`), leak diagnostics, aggregation by derived split (Train and Validation only on the round path; Test only through `TestReadings::at_checkpoint`, which consumes a `Checkpoint`); library only, `weksa score` comes with the loop. h2 landed the r2 fix pass on its branch; r3 deletes the name mask (the scorer reads output and reference only) and pools novel coverage per order over word keys that split joined labels. Waits on 2c for merge order. (spec r3) 2d. `entity-door-lows`: invisible format characters refused in entity forms (ZWJ, ZWNJ, CGJ and variation selectors kept); the form length bound per question `form-word-bound` (recommended: bytes only); attested origins (`Cast { persona, fixture }`, `Live { persona }`) and `CastAuthored.author`; empty provenance strings refused at the store; S6 under NFKC plus case fold and no-alphanumeric texts refused; `next_key` refuses at `u32::MAX`; CLI flags that do not apply refuse and no error echoes a value. Owns follow-ups `entity-door-lows` and `s6-unicode-normal-form`. (spec r1) 3b. `lowerer-first-cut` (follow-up `golf-loop-lowerer-cut`): the section above. `lower`, `weksa lower`, `weksa score`, `weksa packet proof`, inventory rev 2 for `nibu-proof-001`. 4. `speaker-state`: packet `stance` (felt affect and restraint, slot `Stance`, packet schema v4); formality, warmth and intensity out of the seed and the standard; only `PacketStore::admit` yields a `LoweringInput`, and it refuses packets for withdrawn items (follow-up `lowering-input-from-packet-store`). (spec r1) @@ -401,7 +401,8 @@ One line per cut; specs exist for the cuts about to run (2d and 4, siblings on ` 4b. `speaker-scene`: slots `Speaker` and `Scene` with their seed features; `weksa.speaker` (`speaker:@`) and `weksa.scene` (`scene:@`, participants and ordered-pair relations) documents with `Annotator` `Operator` and `Model { family, generator }`, validated by the packet's shared feature check, append-only per key; their stores, `weksa speaker` and `weksa scene` commands, published schemas; Model-annotated cast speakers and scenes for the committed items. Waits on 2d and 4; `pd-speaker-annotation` blocks only the public-domain annotator. 5. `language-ontology`: `weksa.community` from `data/cultural-ontology/` (owns follow-up `edda-referent-yaml-parse`), `weksa.ontology_entry` for en-US and ja-JP from `data/target-language-ontology/` (YAML deleted), speaker communities, deterministic `select`, the lowering view (packet without trace, speaker, scene, selected entries). Waits on 4. 6. `pd-importer`: public-domain play and fiction dialogue into the corpus with speaker, scene group, edition provenance, and Speaker and Scene annotations by the ruled annotator. Waits on 4, follow-up `gutenberg-terms` and question `pd-speaker-annotation`. -7. `golf-loop-en`: the loop driver, the Train-only context builder for inventory- and ontology-minting calls (follow-up `loop-context-builder-train-only`), the packet store bound to one inventory store (`packet-store-inventory-binding`), `ModelPort` with the `claude -p` adapter, encoder (packet with stance) and lowerer prompts, round records, Score documents, the composite and accept rule, `weksa score`, the detached unit on Yggdrasil. Waits on 3, 5, 6 and follow-up `claude-on-yggdrasil`. +7a. `loop-context-builder`: `context.rs` (`TrainItems`, `EncoderOffer`, `Prompt`: corpus text reaches a model call only through `Prompt::mint` over live Train items or `Prompt::encode` over one offered item; instructions are `&'static str`), `SplitScheme::live`, `CorpusItem::cited_entities` (one owner of the id set `PacketStore::admit` and the offer share), and checkpoint-only Test readings (`Checkpoint`, `TestReadings`; `Aggregate` and `leak_diagnostic` carry no Test). Library only. Merges follow-ups `loop-context-builder-2` and `scorer-test-readings-type`. (spec r1) +7. `golf-loop-en`: the loop driver, the packet store bound to one inventory store (`packet-store-inventory-binding`), `ModelPort` with the `claude -p` adapter (it takes only `&Prompt`), encoder (packet with stance) and lowerer prompts, round records, Score documents, the composite and accept rule, `weksa score`, the detached unit on Yggdrasil. Waits on 3, 5, 6 and follow-up `claude-on-yggdrasil`. 8. `style-axis`: a style-embedding score behind the scorer port. Waits on 3. 9. `ja-panel`: ja-JP into the loop's panel. Waits on 7. 10. `conlang-lexicon`: the first conlang's lexicon placed at concept embeddings, its morphology data, its ontology entries and the gloss reader. Waits on 3, 5 and question `first-conlang`. From c1d530ccea09eafc08970579eebb731aa25735e8 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 02:08:45 +0000 Subject: [PATCH 025/175] context tests: pin TrainItems len and is_empty and the empty mint Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/context.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs index b499ee1..5f5d3de 100644 --- a/crates/weksa/tests/context.rs +++ b/crates/weksa/tests/context.rs @@ -104,6 +104,13 @@ fn train_items_are_in_key_order() { t.keys().collect::>(), ["item:a:1", "item:m:1", "item:z:1"] ); + assert_eq!((t.len(), t.is_empty()), (3, false)); + let dir = tempfile::tempdir().unwrap(); + let none = CorpusStore::open(&dir.path().join("c.cc")).unwrap(); + let none = TrainItems::from_corpus(&none).unwrap(); + assert_eq!((none.len(), none.is_empty()), (0, true)); + assert_eq!(Prompt::mint(MINT, &none).text(), MINT); + assert!(Prompt::mint(MINT, &none).item_keys().is_empty()); } #[test] From 35b32b96390c12af566300a43fd2739ec8f1a76c Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 02:23:29 +0000 Subject: [PATCH 026/175] Prior turns print only text a live Train item of the group holds; pin the Train union against withdrawn items Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/context.rs | 46 +++++++++---- crates/weksa/src/store.rs | 2 +- crates/weksa/tests/context.rs | 117 ++++++++++++++++++++++++++++++++-- crates/weksa/tests/score.rs | 21 ++++++ 4 files changed, 169 insertions(+), 17 deletions(-) diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index 13fce61..de94479 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -9,7 +9,7 @@ use crate::corpus::CorpusItem; use crate::packet::EntityId; use crate::split::{Checkpoint, Split, SplitScheme}; -use crate::store::CorpusStore; +use crate::store::{CorpusStore, normalized}; /// The live Train items of a corpus, in key order. The only input minting /// reads; it cannot be built from a slice, and no accessor returns text or an @@ -62,6 +62,15 @@ impl TrainItems { self.items.is_empty() } + /// Whether a live Train item of `group` holds `text`, compared by the + /// store's own text key. + fn holds(&self, group: &str, text: &str) -> bool { + let key = normalized(text); + self.items + .iter() + .any(|item| item.group == group && normalized(&item.text) == key) + } + pub fn keys(&self) -> impl Iterator { self.items.iter().map(|item| item.id.as_str()) } @@ -162,35 +171,50 @@ pub struct Prompt { items: Vec, } -fn block(item: &CorpusItem) -> String { - let mut lines: Vec = item - .prior_turns - .iter() - .map(|t| format!("{}: {}", t.speaker, t.text)) +/// What a prompt prints of an item: its line, and its prior turns only as the +/// one owner allows. A prior turn is a copy of text, not a reference, so it can +/// outlive a withdrawal or quote a held-out line; its text is printed only when +/// a live Train item of the item's group holds it, and the speaker alone is +/// printed otherwise. Mint prints no prior turns (`prior: None`). +fn block(item: &CorpusItem, prior: Option<&TrainItems>) -> String { + let mut lines: Vec = prior + .into_iter() + .flat_map(|train| { + item.prior_turns.iter().map(move |t| { + if train.holds(&item.group, &t.text) { + format!("{}: {}", t.speaker, t.text) + } else { + format!("{}: {ELIDED}", t.speaker) + } + }) + }) .collect(); lines.push(format!("[{}] {}: {}", item.id, item.speaker, item.text)); lines.join("\n") } +/// Printed in place of a prior turn whose text no live Train item holds. +const ELIDED: &str = "[elided]"; + impl Prompt { /// Instructions, then every Train item in key order. pub fn mint(instructions: &'static str, train: &TrainItems) -> Prompt { let mut parts = vec![instructions.to_string()]; - parts.extend(train.items.iter().map(block)); + parts.extend(train.items.iter().map(|item| block(item, None))); Prompt { text: parts.join("\n\n"), items: train.keys().map(str::to_string).collect(), } } - /// Instructions, the one offered item, and the entities a packet for it - /// may cite. - pub fn encode(instructions: &'static str, offer: &EncoderOffer) -> Prompt { + /// Instructions, the one offered item with its prior turns resolved + /// against `train`, and the entities a packet for it may cite. + pub fn encode(instructions: &'static str, offer: &EncoderOffer, train: &TrainItems) -> Prompt { let ids: Vec<&str> = offer.entities.iter().map(EntityId::as_str).collect(); Prompt { text: format!( "{instructions}\n\n{}\n\nentities: {}", - block(&offer.item), + block(&offer.item, Some(train)), ids.join(", ") ), items: vec![offer.item.id.clone()], diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index 980b040..25909c0 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -430,7 +430,7 @@ fn check_entities(item: &CorpusItem, entities: &EntityStore) -> Result<(), Store /// (U+0307) after i is dropped, so Ä°x and i̇x collide as the old case fold made /// them; so does plain ix, with both. Over-merging a rare sequence errs toward /// held, a lost collision toward leak. U+0307 after any other letter stays. -fn normalized(text: &str) -> String { +pub(crate) fn normalized(text: &str) -> String { let mapped: String = text.nfkc().flat_map(fold_variant).collect(); // The dot goes before the last NFKC so the marks left on the i compose with // it (i, dot, acute reads as the i with an acute). diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs index 5f5d3de..a27b31d 100644 --- a/crates/weksa/tests/context.rs +++ b/crates/weksa/tests/context.rs @@ -147,17 +147,17 @@ fn prompts_render_deterministically() { assert_eq!( Prompt::mint(MINT, &t).text(), format!( - "{MINT}\n\nentity:brokkr: Well?\n[item:a:1] entity:nibu: Hello there.\n\n[item:a:2] entity:nibu: Second line." + "{MINT}\n\n[item:a:1] entity:nibu: Hello there.\n\n[item:a:2] entity:nibu: Second line." ) ); assert_eq!(Prompt::mint(MINT, &t).item_keys(), ["item:a:1", "item:a:2"]); let offer = EncoderOffer::for_round(&store.get("item:a:2").unwrap().unwrap()).unwrap(); assert_eq!( - Prompt::encode(ENCODE, &offer).text(), + Prompt::encode(ENCODE, &offer, &t).text(), format!("{ENCODE}\n\n[item:a:2] entity:nibu: Second line.\n\nentities: entity:nibu") ); - assert_eq!(Prompt::encode(ENCODE, &offer).item_keys(), ["item:a:2"]); + assert_eq!(Prompt::encode(ENCODE, &offer, &t).item_keys(), ["item:a:2"]); } #[test] @@ -234,7 +234,8 @@ fn cited_world() -> (Corpus, CorpusItem, CorpusItem) { #[test] fn encoder_offer_entities_are_the_items_own() { - let (_c, mine, sibling) = cited_world(); + let (c, mine, sibling) = cited_world(); + let t = TrainItems::from_corpus(&c.store).unwrap(); assert_eq!( mine.cited_entities() .map(|e| e.as_str()) @@ -255,7 +256,7 @@ fn encoder_offer_entities_are_the_items_own() { "a sibling's entity is not the item's" ); assert_eq!(offer.key(), "item:a:1"); - let p = Prompt::encode(ENCODE, &offer); + let p = Prompt::encode(ENCODE, &offer, &t); assert!( p.text() .ends_with("entities: entity:brokkr, entity:hero, entity:kay") @@ -303,3 +304,109 @@ fn offer_and_store_agree() { } } } + +/// Items of one Train group and of a Test group, with the lines each prior +/// turn quotes. +#[test] +fn prior_turns_print_only_text_a_live_train_item_holds() { + let dir = tempfile::tempdir().unwrap(); + let entities = entities_in(dir.path()); + let mut store = CorpusStore::open(&dir.path().join("c.cc")).unwrap(); + let (train_g, other_train_g) = (group_in_bucket(10), group_in_bucket(11)); + let turn = |text: &str| Turn { + speaker: common::eid("brokkr"), + text: text.into(), + }; + let mut add = |key: &str, text: &str, group: &str, turns: Vec| { + let mut item = item_in(key, text, group, cast()); + item.prior_turns = turns; + store.add(item, &entities).unwrap(); + }; + add("item:a:1", "withdrawnsecret line", &train_g, vec![]); + add("item:a:4", "livecontext line", &train_g, vec![]); + add( + "item:a:2", + "reply line", + &train_g, + vec![turn("withdrawnsecret line")], + ); + add( + "item:a:3", + "second reply", + &train_g, + vec![turn("testsecret line")], + ); + add( + "item:a:5", + "third reply", + &train_g, + vec![turn("livecontext line")], + ); + add( + "item:a:6", + "other reply", + &train_g, + vec![turn("elsewhere line")], + ); + add("item:o:1", "elsewhere line", &other_train_g, vec![]); + add( + "item:b:1", + "validation reply", + &group_in_bucket(50), + vec![turn("testsecret line")], + ); + add("item:c:1", "testsecret line", &group_in_bucket(90), vec![]); + store.withdraw("item:a:1").unwrap(); + + let t = TrainItems::from_corpus(&store).unwrap(); + let mint = Prompt::mint(MINT, &t); + for secret in ["withdrawnsecret", "testsecret", "validation reply"] { + assert!(!mint.text().contains(secret), "mint printed {secret}"); + } + assert!( + !mint.text().contains("brokkr"), + "mint prints no prior turns" + ); + + let encode = |key: &str| { + let item = store.get(key).unwrap().unwrap(); + Prompt::encode(ENCODE, &EncoderOffer::for_round(&item).unwrap(), &t) + .text() + .to_string() + }; + let withdrawn = encode("item:a:2"); + assert!(!withdrawn.contains("withdrawnsecret")); + assert!(withdrawn.contains("entity:brokkr: [elided]\n[item:a:2]")); + let test_quote = encode("item:a:3"); + assert!(!test_quote.contains("testsecret")); + assert!(test_quote.contains("entity:brokkr: [elided]")); + let validation = encode("item:b:1"); + assert!(!validation.contains("testsecret")); + assert!(validation.contains("entity:brokkr: [elided]")); + assert!(encode("item:a:5").contains("entity:brokkr: livecontext line\n[item:a:5]")); + let cross_group = encode("item:a:6"); + assert!( + !cross_group.contains("elsewhere"), + "a held line of another group is not this group's" + ); +} + +#[test] +fn prior_turn_match_uses_the_stores_text_key() { + let dir = tempfile::tempdir().unwrap(); + let entities = entities_in(dir.path()); + let mut store = CorpusStore::open(&dir.path().join("c.cc")).unwrap(); + let g = group_in_bucket(10); + store + .add(item_in("item:a:1", "Context Line", &g, cast()), &entities) + .unwrap(); + let mut reply = item_in("item:a:2", "reply", &g, cast()); + reply.prior_turns = vec![Turn { + speaker: common::eid("brokkr"), + text: "context line".into(), + }]; + store.add(reply.clone(), &entities).unwrap(); + let t = TrainItems::from_corpus(&store).unwrap(); + let p = Prompt::encode(ENCODE, &EncoderOffer::for_round(&reply).unwrap(), &t); + assert!(p.text().contains("entity:brokkr: context line")); +} diff --git a/crates/weksa/tests/score.rs b/crates/weksa/tests/score.rs index 32e9bfa..cdf2586 100644 --- a/crates/weksa/tests/score.rs +++ b/crates/weksa/tests/score.rs @@ -948,3 +948,24 @@ fn validation_and_test_leak_agree_in_method() { assert_eq!(t.max_chrf(), v.max_chrf.as_ref().map(|p| p.chrf)); assert_eq!(t.coverage_by_n(), v.coverage_by_n.as_slice()); } + +#[test] +fn held_out_train_union_excludes_withdrawn_train_items() { + let mut gone = train("item:t:1", HELD_OUT); + gone.withdrawn = true; + let held = validation("item:v:1", HELD_OUT); + let d = leak_diagnostic( + &[HELD_OUT], + &[gone, held.clone(), train("item:t:2", UNRELATED)], + ); + assert_eq!( + d.coverage_by_n + .iter() + .map(|c| (c.novel, c.covered)) + .collect::>(), + [(12, 12), (11, 11), (10, 10)], + "a withdrawn Train text makes no n-gram known" + ); + let live = leak_diagnostic(&[HELD_OUT], &[train("item:t:1", HELD_OUT), held]); + assert_eq!(live.coverage_by_n[0].novel, 0, "a live Train text does"); +} From 0b62d2f3b422ca2d3e77df0e9c4d91f1c6340e34 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 02:29:51 +0000 Subject: [PATCH 027/175] Retire V13 and with_scene's raw door; Stance and Scene features sit in one slot with_scene is removed until it takes a Scene document (next commit); V13 goes with it. Slot exclusivity becomes a rule of InventoryEntry::well_formed, so FEAT_ALL leaves Stance and its stance duplicate case goes (ANGER pins V11 in Stance). tests/scene.rs is deleted and rewritten on Scene documents later. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/inventory.rs | 15 ++- crates/weksa/src/validate.rs | 47 +--------- crates/weksa/tests/common/mod.rs | 4 +- crates/weksa/tests/display.rs | 1 - crates/weksa/tests/scene.rs | 156 ------------------------------- crates/weksa/tests/stores.rs | 40 ++++++++ crates/weksa/tests/validate.rs | 3 - 7 files changed, 58 insertions(+), 208 deletions(-) delete mode 100644 crates/weksa/tests/scene.rs diff --git a/crates/weksa/src/inventory.rs b/crates/weksa/src/inventory.rs index c9d69a2..d5f2e81 100644 --- a/crates/weksa/src/inventory.rs +++ b/crates/weksa/src/inventory.rs @@ -228,10 +228,21 @@ impl InventoryEntry { } /// Whether the entry's shape matches its family: features carry a value kind - /// and slots, extensions a value kind and no slots, nothing else either. + /// and slots, extensions a value kind and no slots, nothing else either. A + /// feature of slot `Stance` or `Scene` sits in that slot alone: each + /// speaker-state layer has one writer and one lifetime, so its features sit + /// nowhere else. pub fn well_formed(&self) -> bool { match self.family() { - Family::Feature => self.value_kind.is_some() && !self.applies_to.is_empty(), + Family::Feature => { + self.value_kind.is_some() + && !self.applies_to.is_empty() + && (self.applies_to.len() == 1 + || !self + .applies_to + .iter() + .any(|s| matches!(s, Slot::Stance | Slot::Scene))) + } Family::Extension => self.value_kind.is_some() && self.applies_to.is_empty(), _ => self.value_kind.is_none() && self.applies_to.is_empty(), } diff --git a/crates/weksa/src/validate.rs b/crates/weksa/src/validate.rs index 6a0075f..e5a917e 100644 --- a/crates/weksa/src/validate.rs +++ b/crates/weksa/src/validate.rs @@ -3,10 +3,9 @@ //! returns `Ok(())` or the refusals. [`LoweringInput`], the only thing a //! lowerer sees, comes from `PacketStore::admit` alone, which builds it through //! the crate-private `admissible`. The stance is checked like every feature -//! list, in slot `Stance`. The rules are V1..V8 and V10..V15: the shape of an id -//! or a grade is its type's parse, not a rule, so V9 is retired and not -//! renumbered. V13 guards the one other way features reach a lowering view: -//! [`LoweringInput::with_scene`]. +//! list, in slot `Stance`. The rules are V1..V8, V10..V12 and V14..V15: the +//! shape of an id or a grade is its type's parse, not a rule, so V9 is retired +//! and not renumbered; V13 is retired too. //! //! `validate` owns form and capacity and makes no leak decision. A packet's one //! free text is its trace (`notes` and `rejected_readings`): `validate` does not @@ -95,10 +94,6 @@ pub enum Refusal { count: usize, max: usize, }, - /// V13: scene features were given for a packet with no addressee. They - /// state the speaker's relation to the addressee, so with none they - /// describe nobody. - V13SceneWithoutAddressee, /// V14: an addressee consent that no addressee can have given, or that /// names a node that is not a predication. It is what the addressee asked /// for or agreed to, an act, so a referent cannot be its object. @@ -130,7 +125,6 @@ impl Refusal { | Refusal::V11DuplicateNodeRef { .. } | Refusal::V11DuplicateRole { .. } => "V11", Refusal::V12TooManyNodes { .. } => "V12", - Refusal::V13SceneWithoutAddressee => "V13", Refusal::V14ConsentWithoutAddressee | Refusal::V14ConsentNotAPredication { .. } => { "V14" } @@ -157,7 +151,6 @@ impl fmt::Display for Refusal { Refusal::V11DuplicateNodeRef { .. } => "a node id repeats in one list", Refusal::V11DuplicateRole { .. } => "a role id repeats in one predication", Refusal::V12TooManyNodes { .. } => "too many nodes", - Refusal::V13SceneWithoutAddressee => "scene features given without an addressee", Refusal::V14ConsentWithoutAddressee => "consent given without an addressee", Refusal::V14ConsentNotAPredication { .. } => "consent names no predication", Refusal::V15ConsentSource { .. } => "consent set by a source that only read wording", @@ -594,40 +587,6 @@ impl LoweringInput { } } - /// Adds the speaker's relation to the addressee: feature values of slot - /// `Scene`, checked like every feature list (V5 to V8, V11) at this view's - /// inventory revision. They state a relation, so the packet must name an - /// addressee (V13). Typed ids and grades only, never wording, so a scene - /// carries no text to the lowerer. The scene is not part of the packet: a - /// scene document supplies it, and replacing it replaces the earlier one. - pub fn with_scene( - mut self, - inventory: &Inventory, - scene: Vec, - ) -> Result> { - let mut out = Vec::new(); - if !inventory.has_revision(self.inventory_rev) { - out.push(Refusal::V10UnknownRevision { - rev: self.inventory_rev, - }); - } else { - Inventoried { - inv: inventory, - rev: self.inventory_rev, - out: &mut out, - } - .features(&scene, Slot::Scene); - } - if !scene.is_empty() && self.discourse.addressee.is_none() { - out.push(Refusal::V13SceneWithoutAddressee); - } - if !out.is_empty() { - return Err(out); - } - self.scene = scene; - Ok(self) - } - pub fn inventory_rev(&self) -> u32 { self.inventory_rev } diff --git a/crates/weksa/tests/common/mod.rs b/crates/weksa/tests/common/mod.rs index 60ba544..9368227 100644 --- a/crates/weksa/tests/common/mod.rs +++ b/crates/weksa/tests/common/mod.rs @@ -34,7 +34,8 @@ pub const CONCEPT_LATE: &str = "concept:4"; pub const FRAME: &str = "frame:1"; /// An extension entry of value kind Grade, introduced at rev 2. pub const EXT: &str = "extension:1"; -/// A Grade feature introduced at rev 2 that sits in every slot. +/// A Grade feature introduced at rev 2 in every content slot (Discourse, +/// Referent, Predication, Relation). pub const FEAT_ALL: &str = "feature:50"; pub const PACKET_ID: &str = "packet:0123456789abcdef"; @@ -218,7 +219,6 @@ pub fn world() -> World { Slot::Referent, Slot::Predication, Slot::Relation, - Slot::Stance, ], ) }, diff --git a/crates/weksa/tests/display.rs b/crates/weksa/tests/display.rs index 3c1ba63..44063fa 100644 --- a/crates/weksa/tests/display.rs +++ b/crates/weksa/tests/display.rs @@ -117,7 +117,6 @@ fn refusal_display_names_the_rule_and_no_identifier() { count: 4242, max: 4242, }, - Refusal::V13SceneWithoutAddressee, Refusal::V14ConsentWithoutAddressee, Refusal::V14ConsentNotAPredication { id: node() }, Refusal::V15ConsentSource { diff --git a/crates/weksa/tests/scene.rs b/crates/weksa/tests/scene.rs deleted file mode 100644 index 0fc4168..0000000 --- a/crates/weksa/tests/scene.rs +++ /dev/null @@ -1,156 +0,0 @@ -//! The speaker's relation to the addressee as a lowering input: typed feature -//! values of slot `Scene`, checked against the inventory at the view's revision, -//! never wording, and only for a packet that names an addressee. - -mod common; -use std::path::Path; - -use common::{grade, iv, lab}; -use weksa::inventory::{proof_revision, scene_revision, seed}; -use weksa::packet::{FeatValue, FeatureValue, Packet, proof_packet}; -use weksa::{ - CorpusStore, EntityStore, Inventory, InventoryStore, LoweringInput, PacketStore, Refusal, lower, -}; - -fn store() -> (InventoryStore, Inventory, tempfile::TempDir) { - let dir = tempfile::tempdir().unwrap(); - let mut store = InventoryStore::open(&dir.path().join("i.cc")).unwrap(); - for (rev, entries) in [seed(), proof_revision(), scene_revision()] { - store.admit_revision(rev, entries).unwrap(); - } - let inv = store.inventory().unwrap(); - (store, inv, dir) -} - -fn admitted(packet: &Packet, inventory: &InventoryStore) -> LoweringInput { - let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); - let corpus = CorpusStore::open(&root.join("data/corpus/corpus.cc")).unwrap(); - let entities = EntityStore::open(&root.join("data/entity/entity.cc")).unwrap(); - let dir = tempfile::tempdir().unwrap(); - PacketStore::open(&dir.path().join("p.cc")) - .unwrap() - .admit(packet.clone(), inventory, &corpus, &entities) - .unwrap() -} - -fn at_rev(rev: u32) -> Packet { - let mut p = proof_packet(); - p.inventory_rev = rev; - p -} - -fn feat(id: &str, value: FeatValue) -> FeatureValue { - FeatureValue { - feature: iv(id), - value, - } -} - -fn status(inv: &Inventory) -> String { - inv.entries() - .find(|e| e.gloss == "relative_status") - .unwrap() - .id - .to_string() -} - -fn id_of(inv: &Inventory, gloss: &str) -> String { - inv.entries() - .find(|e| e.gloss == gloss) - .unwrap() - .id - .to_string() -} - -fn rules(r: Result>) -> Vec<&'static str> { - r.unwrap_err().iter().map(Refusal::rule).collect() -} - -#[test] -fn a_scene_relation_reaches_the_lowering_view_and_changes_no_english() { - let (store, inv, _dir) = store(); - let plain = admitted(&at_rev(3), &store); - assert!(plain.scene().is_empty()); - let scene = vec![ - feat(&status(&inv), FeatValue::Option(lab("above"))), - feat(&id_of(&inv, "intimacy"), FeatValue::Grade(grade(200))), - feat(&id_of(&inv, "in_group"), FeatValue::Flag(false)), - ]; - let with = plain.clone().with_scene(&inv, scene.clone()).unwrap(); - assert_eq!(with.scene(), scene.as_slice()); - let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); - let entities = EntityStore::open(&root.join("data/entity/entity.cc")).unwrap(); - assert_eq!( - lower(&with, &inv, &entities).unwrap(), - lower(&plain, &inv, &entities).unwrap() - ); -} - -#[test] -fn a_relation_needs_an_addressee() { - let (store, inv, _dir) = store(); - let mut p = at_rev(3); - p.discourse.addressee = None; - let view = admitted(&p, &store); - let scene = vec![feat(&id_of(&inv, "in_group"), FeatValue::Flag(true))]; - assert_eq!(rules(view.clone().with_scene(&inv, scene)), ["V13"]); - assert!(view.with_scene(&inv, vec![]).unwrap().scene().is_empty()); -} - -#[test] -fn a_relation_value_is_checked_like_any_feature() { - let (store, inv, _dir) = store(); - let view = admitted(&at_rev(3), &store); - let status = status(&inv); - let intimacy = id_of(&inv, "intimacy"); - let take = |scene| view.clone().with_scene(&inv, scene); - // A label the feature does not declare. - assert_eq!( - rules(take(vec![feat(&status, FeatValue::Option(lab("lordly")))])), - ["V7"] - ); - // A value of the wrong kind. - assert_eq!( - rules(take(vec![feat(&intimacy, FeatValue::Flag(true))])), - ["V7"] - ); - // A feature declared for another slot: seed feature 1 sits in Predication. - assert_eq!( - rules(take(vec![feat( - "feature:1", - FeatValue::Option(lab("negative")) - )])), - ["V8"] - ); - // An id that is not a feature at all. - assert_eq!( - rules(take(vec![feat("concept:3", FeatValue::Flag(true))])), - ["V6"] - ); - // Twice in one list. - let twice = vec![ - feat(&intimacy, FeatValue::Grade(grade(1))), - feat(&intimacy, FeatValue::Grade(grade(2))), - ]; - assert_eq!(rules(take(twice)), ["V11"]); -} - -#[test] -fn a_view_of_a_revision_before_the_scene_vocabulary_cannot_carry_one() { - let (store, inv, _dir) = store(); - // Revision 2 predates the Scene features, so their ids do not resolve there. - let view = admitted(&at_rev(2), &store); - let scene = vec![feat(&id_of(&inv, "intimacy"), FeatValue::Grade(grade(1)))]; - assert_eq!(rules(view.with_scene(&inv, scene)), ["V5"]); -} - -#[test] -fn a_stance_feature_is_not_a_scene_feature() { - let (store, inv, _dir) = store(); - let view = admitted(&at_rev(3), &store); - let anger = id_of(&inv, "anger"); - assert_eq!( - rules(view.with_scene(&inv, vec![feat(&anger, FeatValue::Grade(grade(1)))])), - ["V8"] - ); -} diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs index 22ecb83..fbcce9a 100644 --- a/crates/weksa/tests/stores.rs +++ b/crates/weksa/tests/stores.rs @@ -1093,6 +1093,46 @@ fn revision_change_rules() { assert!(inv.entry(&iv("role:1")).unwrap().retired_rev.is_none()); } +#[test] +fn speaker_state_slots_are_exclusive() { + let shape = "entry shape does not match its family"; + let d = dir(); + let mut store = seeded(&d.path().join("i.cc")); + for slots in [ + &[Slot::Stance, Slot::Discourse][..], + &[Slot::Scene, Slot::Relation], + &[Slot::Discourse, Slot::Scene], + &[Slot::Scene, Slot::Scene], + &[Slot::Stance, Slot::Stance], + ] { + let rev = rev_with(vec![Change::Add(iv("feature:50"))], 2, Some(1)); + let entries = vec![InventoryEntry { + introduced_rev: 2, + ..feature("feature:50", ValueKind::Flag, slots) + }]; + assert_eq!( + refusal(store.admit_revision(rev, entries)), + malformed("feature:50", shape), + "{slots:?}" + ); + } + assert_eq!(store.inventory().unwrap().latest().unwrap().n, 1); + // One slot alone, and several content slots, are admitted. + for (n, slots) in [ + (2, &[Slot::Scene][..]), + (3, &[Slot::Stance]), + (4, &[Slot::Discourse, Slot::Predication]), + ] { + let id = format!("feature:{}", 50 + n); + let rev = rev_with(vec![Change::Add(iv(&id))], n, Some(n - 1)); + let entries = vec![InventoryEntry { + introduced_rev: n, + ..feature(&id, ValueKind::Flag, slots) + }]; + store.admit_revision(rev, entries).unwrap(); + } +} + #[test] fn a_well_formed_extension_entry_is_admitted() { let d = dir(); diff --git a/crates/weksa/tests/validate.rs b/crates/weksa/tests/validate.rs index aaa87bd..c5bc172 100644 --- a/crates/weksa/tests/validate.rs +++ b/crates/weksa/tests/validate.rs @@ -520,9 +520,6 @@ fn v11_duplicate_feature_each_slot() { refusals(&relation), vec![v11(Some(Slot::Relation), FEAT_ALL)] ); - let mut stance = good(); - stance.stance.features = vec![grade_feature(1), grade_feature(2)]; - assert_eq!(refusals(&stance), vec![v11(Some(Slot::Stance), FEAT_ALL)]); let mut felt = good(); felt.stance .features From 6ef1b7e0347600a2430ff778ad4d40bbe9a0b871 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 02:34:10 +0000 Subject: [PATCH 028/175] Speaker and Scene documents, SpeakerStateStore, with_scene on a Scene document Slot Speaker, weksa.speaker and weksa.scene append-only revisions in one store, V16, feature_refusals as the one feature check, speaker_scene_revision (Speaker features 22-24), schemas, docs and map lines. Part 1 of map cut 4b. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/inventory.rs | 65 ++- crates/weksa/src/lib.rs | 8 +- crates/weksa/src/schema.rs | 3 + crates/weksa/src/speaker.rs | 115 +++++ crates/weksa/src/store.rs | 229 ++++++++- crates/weksa/src/validate.rs | 93 +++- crates/weksa/tests/cli.rs | 4 +- crates/weksa/tests/scene.rs | 244 +++++++++ crates/weksa/tests/speaker_state.rs | 478 ++++++++++++++++++ crates/weksa/tests/stores.rs | 7 +- docs/interlingua-standard.md | 19 + docs/targets/dao-de-jing-test-bed.md | 2 +- docs/targets/interlingua-lowering-map.md | 7 +- .../cultnet/weksa.inventory_entry.schema.json | 7 +- schemas/cultnet/weksa.scene.schema.json | 224 ++++++++ schemas/cultnet/weksa.speaker.schema.json | 171 +++++++ 16 files changed, 1638 insertions(+), 38 deletions(-) create mode 100644 crates/weksa/src/speaker.rs create mode 100644 crates/weksa/tests/scene.rs create mode 100644 crates/weksa/tests/speaker_state.rs create mode 100644 schemas/cultnet/weksa.scene.schema.json create mode 100644 schemas/cultnet/weksa.speaker.schema.json diff --git a/crates/weksa/src/inventory.rs b/crates/weksa/src/inventory.rs index d5f2e81..6c8b984 100644 --- a/crates/weksa/src/inventory.rs +++ b/crates/weksa/src/inventory.rs @@ -189,9 +189,11 @@ pub enum Slot { Predication, Relation, Stance, - /// The relation of the speaker to the addressee in the scene; see - /// `LoweringInput::with_scene`. + /// The speaker's relation to one party of the scene, held by `weksa.scene` + /// documents; see `LoweringInput::with_scene`. Scene, + /// The speaker's stable axes, held by `weksa.speaker` documents. + Speaker, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] @@ -229,7 +231,7 @@ impl InventoryEntry { /// Whether the entry's shape matches its family: features carry a value kind /// and slots, extensions a value kind and no slots, nothing else either. A - /// feature of slot `Stance` or `Scene` sits in that slot alone: each + /// feature of slot `Stance`, `Speaker` or `Scene` sits in that slot alone: each /// speaker-state layer has one writer and one lifetime, so its features sit /// nowhere else. pub fn well_formed(&self) -> bool { @@ -241,7 +243,7 @@ impl InventoryEntry { || !self .applies_to .iter() - .any(|s| matches!(s, Slot::Stance | Slot::Scene))) + .any(|s| matches!(s, Slot::Stance | Slot::Speaker | Slot::Scene))) } Family::Extension => self.value_kind.is_some() && self.applies_to.is_empty(), _ => self.value_kind.is_none() && self.applies_to.is_empty(), @@ -534,28 +536,49 @@ pub fn proof_revision() -> (InventoryRevision, Vec) { ) } -/// Revision 3: the speaker's relation to the addressee, features of slot -/// `Scene`. Hand-seeded like [`proof_revision`] until the loop's minting round -/// and cut `speaker-scene` own the Scene vocabulary. `relative_status` is the -/// speaker's standing against the addressee's, not a register word: -/// what it selects is each language's ontology's business. It mints the three -/// Scene features after the seed's features. -pub fn scene_revision() -> (InventoryRevision, Vec) { +/// Revision 3: the speaker-state vocabulary. First the Scene features, the +/// speaker's relation to one party of a scene (slot `Scene`): `relative_status` +/// is the speaker's standing against the addressee's, not a register word, what +/// it selects is each language's ontology's business. Then the Speaker features +/// (slot `Speaker`): `socioeconomic_standing` is standing in the speaker's own +/// society, 0 lowest; `education` is formal schooling against that society; +/// `age_cohort` is a life stage. Origin and community are not features; they +/// arrive as community ids with cut `language-ontology`. Hand-seeded like +/// [`proof_revision`] until the loop's minting round owns the vocabulary; the +/// features take the ids after the seed's. +pub fn speaker_scene_revision() -> (InventoryRevision, Vec) { let taken = seed() .1 .iter() .filter(|e| e.family() == Family::Feature) .count() as u32; - let labels = ["below", "level", "above"] - .map(|l| OptionLabel::parse(l).expect("seed labels are well formed")); - let relative_status = ValueKind::Options( - OptionSet::new(labels.to_vec()).expect("seed option sets are well formed"), - ); + let options = |labels: &[&str]| { + ValueKind::Options( + OptionSet::new( + labels + .iter() + .map(|l| OptionLabel::parse(l).expect("seed labels are well formed")) + .collect(), + ) + .expect("seed option sets are well formed"), + ) + }; let mut entries = Vec::new(); - for (i, (gloss, kind)) in [ - ("relative_status", relative_status), - ("intimacy", ValueKind::Grade), - ("in_group", ValueKind::Flag), + for (i, (gloss, kind, slot)) in [ + ( + "relative_status", + options(&["below", "level", "above"]), + Slot::Scene, + ), + ("intimacy", ValueKind::Grade, Slot::Scene), + ("in_group", ValueKind::Flag, Slot::Scene), + ("socioeconomic_standing", ValueKind::Grade, Slot::Speaker), + ("education", ValueKind::Grade, Slot::Speaker), + ( + "age_cohort", + options(&["child", "adolescent", "young_adult", "adult", "elder"]), + Slot::Speaker, + ), ] .into_iter() .enumerate() @@ -564,7 +587,7 @@ pub fn scene_revision() -> (InventoryRevision, Vec) { id: Family::Feature.id(NonZeroU32::MIN.saturating_add(taken + i as u32)), gloss: gloss.to_string(), value_kind: Some(kind), - applies_to: vec![Slot::Scene], + applies_to: vec![slot], introduced_rev: 3, retired_rev: None, }); diff --git a/crates/weksa/src/lib.rs b/crates/weksa/src/lib.rs index bb42311..e147524 100644 --- a/crates/weksa/src/lib.rs +++ b/crates/weksa/src/lib.rs @@ -14,6 +14,7 @@ pub mod packet; pub mod phones; pub mod schema; pub mod score; +pub mod speaker; pub mod split; pub mod store; pub mod validate; @@ -26,14 +27,15 @@ pub use inventory::{ Slot, ValueKind, }; pub use lower::{EntityForms, LowerError, PhoneLine, TARGET_LANGUAGE, lower, phones, pronounce}; -pub use phones::{Phone, PhoneInventory, SegmentError, Stress}; pub use packet::{ EncoderId, EntityId, Grade, NodeId, Packet, PacketId, PacketKind, SCHEMA_VERSION, Stance, }; +pub use phones::{Phone, PhoneInventory, SegmentError, Stress}; pub use score::{Embedder, ItemScore, OllamaEmbedder, Scorer, chrf}; +pub use speaker::{Annotator, PairRelation, Party, Scene, Speaker}; pub use split::{Split, SplitScheme}; pub use store::{ - AddOutcome, CorpusStore, EntityBound, EntityStore, InventoryStore, PacketStore, StoreError, - StoreRefusal, + AddOutcome, CorpusStore, EntityBound, EntityStore, InventoryStore, PacketStore, + SpeakerStateStore, StoreError, StoreRefusal, }; pub use validate::{LoweringInput, Refusal, validate}; diff --git a/crates/weksa/src/schema.rs b/crates/weksa/src/schema.rs index 72c27e4..883f44a 100644 --- a/crates/weksa/src/schema.rs +++ b/crates/weksa/src/schema.rs @@ -5,6 +5,7 @@ use crate::corpus::CorpusItem; use crate::entity::Entity; use crate::inventory::{InventoryEntry, InventoryRevision}; use crate::packet::Packet; +use crate::speaker::{Scene, Speaker}; use cultcache_rs::DatabaseEntry; use schemars::{JsonSchema, schema_for}; @@ -21,5 +22,7 @@ pub fn published() -> Vec<(String, String)> { published_for::(), published_for::(), published_for::(), + published_for::(), + published_for::(), ] } diff --git a/crates/weksa/src/speaker.rs b/crates/weksa/src/speaker.rs new file mode 100644 index 0000000..51367e7 --- /dev/null +++ b/crates/weksa/src/speaker.rs @@ -0,0 +1,115 @@ +//! The speaker-state documents: a speaker's stable axes and a scene's relations. +//! Both are append-only revisions under one key per revision and hold typed +//! feature values only, never text (ruling `leak-owned-by-flow-and-scorer-3`). +//! `SpeakerStateStore` is their only writer; `LoweringInput::with_scene` is the +//! only reader that reaches a lowering view. + +use cultcache_rs::DatabaseEntry; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::corpus::ModelFamily; +use crate::packet::{EntityId, FeatureValue}; + +/// The most participants one scene holds. +pub const MAX_PARTICIPANTS: usize = 64; + +/// Who annotated a document. A model's generator is an attestation string, held +/// to the same form as every provenance string. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub enum Annotator { + Operator, + Model { + family: ModelFamily, + generator: String, + }, +} + +/// The speaker's stable axes. Key `speaker:@`; a revision +/// appends and never edits. +#[derive(Clone, Debug, PartialEq, Eq, JsonSchema, DatabaseEntry)] +#[cultcache(type = "weksa.speaker")] +pub struct Speaker { + #[cultcache(key = 0)] + pub entity: EntityId, + /// The revision, from 1. + #[cultcache(key = 1)] + pub n: u32, + #[cultcache(key = 2)] + pub inventory_rev: u32, + /// Feature values of slot `Speaker`. + #[cultcache(key = 3)] + pub features: Vec, + #[cultcache(key = 4)] + pub annotator: Annotator, +} + +/// The other party of a relation. `Unnamed` is an addressee the scene does not +/// name, such as the player or a crowd. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, JsonSchema)] +pub enum Party { + Named(EntityId), + Unnamed, +} + +/// The speaker's relation to one party: feature values of slot `Scene`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct PairRelation { + pub from: EntityId, + pub to: Party, + pub features: Vec, +} + +/// A scene: its participants and the relations between them. Key +/// `scene:@`; a revision appends and never edits. +#[derive(Clone, Debug, PartialEq, Eq, JsonSchema, DatabaseEntry)] +#[cultcache(type = "weksa.scene")] +pub struct Scene { + /// The corpus split group of the scene, a slug. + #[cultcache(key = 0)] + pub group: String, + /// The revision, from 1. + #[cultcache(key = 1)] + pub n: u32, + #[cultcache(key = 2)] + pub inventory_rev: u32, + #[cultcache(key = 3)] + pub participants: Vec, + #[cultcache(key = 4)] + pub relations: Vec, + #[cultcache(key = 5)] + pub annotator: Annotator, +} + +impl Scene { + /// The features of the relation from `speaker` to the named `addressee`, or + /// to `Unnamed` when there is none; empty when the scene holds no such + /// relation. A named addressee the scene lacks never falls back to `Unnamed`. + pub fn relation(&self, speaker: &EntityId, addressee: Option<&EntityId>) -> &[FeatureValue] { + self.relations + .iter() + .find(|r| { + &r.from == speaker + && match (&r.to, addressee) { + (Party::Named(a), Some(b)) => a == b, + (Party::Unnamed, None) => true, + _ => false, + } + }) + .map_or(&[], |r| r.features.as_slice()) + } +} + +/// `speaker:@`. +pub fn speaker_key(entity: &EntityId, n: u32) -> String { + let slug = entity + .as_str() + .strip_prefix("entity:") + .unwrap_or(entity.as_str()); + format!("speaker:{slug}@{n}") +} + +/// `scene:@`. +pub fn scene_key(group: &str, n: u32) -> String { + format!("scene:{group}@{n}") +} diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index 25909c0..0263712 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -14,11 +14,14 @@ use unicode_segmentation::UnicodeSegmentation; use crate::corpus::{CorpusItem, CorpusItemId, Source, item_key, key_slug}; use crate::entity::{Entity, EntityOrigin, Pronunciation}; use crate::form::{ - is_attestation, is_calendar_date, is_joining_only, is_language_tag, is_refused_format, + is_attestation, is_calendar_date, is_joining_only, is_language_tag, is_refused_format, is_slug, +}; +use crate::inventory::{ + Change, InvId, Inventory, InventoryEntry, InventoryRevision, Slot, revision_key, }; -use crate::inventory::{Change, InvId, Inventory, InventoryEntry, InventoryRevision, revision_key}; use crate::packet::{EntityId, Packet}; -use crate::validate::{LoweringInput, Refusal, admissible}; +use crate::speaker::{Annotator, MAX_PARTICIPANTS, Party, Scene, Speaker, scene_key, speaker_key}; +use crate::validate::{LoweringInput, Refusal, admissible, feature_refusals}; /// A broken store rule, named. #[derive(Clone, Debug, PartialEq, Eq)] @@ -79,6 +82,15 @@ pub enum StoreRefusal { /// A new packet names a corpus item already withdrawn. Packets admitted /// before the withdrawal stay stored and readable. WithdrawnCorpusItem { key: String }, + /// A speaker or scene revision that is not the entity's or group's latest + /// plus one (1 when none). + RevisionNotNext { + key: String, + expected: u32, + got: u32, + }, + /// A scene that omits the speaker of a live item of its group. + SceneSpeakerMissing { group: String, entity: EntityId }, /// Any other malformed document or change. Malformed { key: String, why: &'static str }, } @@ -116,6 +128,10 @@ impl fmt::Display for StoreRefusal { EntityNotAppendOnly { .. } => f.write_str("EntityNotAppendOnly"), EntityNotOnItem { .. } => f.write_str("EntityNotOnItem"), WithdrawnCorpusItem { .. } => f.write_str("WithdrawnCorpusItem"), + RevisionNotNext { expected, got, .. } => { + write!(f, "RevisionNotNext expected {expected} got {got}") + } + SceneSpeakerMissing { .. } => f.write_str("SceneSpeakerMissing"), Malformed { why, .. } => write!(f, "Malformed: {why}"), } } @@ -978,3 +994,210 @@ impl PacketStore { Ok(view) } } + +/// The speaker and scene documents: append-only revisions, one key each. The +/// only writer of `weksa.speaker` and `weksa.scene`. +pub struct SpeakerStateStore { + cache: CultCache, +} + +impl SpeakerStateStore { + pub fn open(path: &Path) -> anyhow::Result { + Ok(Self { + cache: open_cache(path, |c| { + c.register_entry_type::()?; + c.register_entry_type::() + })?, + }) + } + + /// Admits the next revision of a speaker, or an identical replay. The key + /// derives from the document. Revision `n` is the entity's latest plus one + /// (1 when none). The entity is in the entity store, the features pass + /// `feature_refusals` in slot `Speaker` at the document's inventory + /// revision, and a model annotator's generator is an attestation. + pub fn put_speaker( + &mut self, + s: Speaker, + inventory: &InventoryStore, + entities: &EntityStore, + ) -> Result { + let key = speaker_key(&s.entity, s.n); + if self.get_speaker(&key)?.is_some_and(|old| old == s) { + return Ok(AddOutcome::Unchanged); + } + check_annotator(&key, &s.annotator)?; + known_entity(entities, &s.entity)?; + let out = feature_refusals( + &inventory.inventory()?, + s.inventory_rev, + &s.features, + Slot::Speaker, + ); + if !out.is_empty() { + return Err(StoreError::Invalid(out)); + } + let latest = self.speaker(&s.entity)?.map_or(0, |l| l.n); + check_next(&key, latest, s.n)?; + self.cache.put(key, &s)?; + Ok(AddOutcome::Added) + } + + /// Admits the next revision of a scene, or an identical replay. The group + /// is a slug some corpus item (live or withdrawn) holds; there are 1 to + /// `MAX_PARTICIPANTS` participants, each named once and in the entity + /// store; a relation's `from` is a participant, a named `to` is another + /// participant, and an ordered pair has at most one relation; every speaker + /// of a live item of the group is a participant. Features pass + /// `feature_refusals` in slot `Scene` for every relation. + pub fn put_scene( + &mut self, + s: Scene, + inventory: &InventoryStore, + corpus: &CorpusStore, + entities: &EntityStore, + ) -> Result { + let key = scene_key(&s.group, s.n); + if self.get_scene(&key)?.is_some_and(|old| old == s) { + return Ok(AddOutcome::Unchanged); + } + let bad = |why| malformed(&key, why); + if !is_slug(&s.group) { + return Err(bad("a scene group is a slug")); + } + check_annotator(&key, &s.annotator)?; + let items: Vec = corpus + .all()? + .into_iter() + .filter(|i| i.group == s.group) + .collect(); + if items.is_empty() { + return Err(bad("no corpus item holds the scene's group")); + } + if s.participants.is_empty() || s.participants.len() > MAX_PARTICIPANTS { + return Err(bad("a scene has 1 to 64 participants")); + } + let mut seen = BTreeSet::new(); + for p in &s.participants { + if !seen.insert(p) { + return Err(bad("a participant is named once")); + } + known_entity(entities, p)?; + } + let mut pairs = BTreeSet::new(); + for r in &s.relations { + if !seen.contains(&r.from) { + return Err(bad("a relation's from is a participant")); + } + if let Party::Named(to) = &r.to + && (!seen.contains(to) || to == &r.from) + { + return Err(bad("a relation's named to is another participant")); + } + if !pairs.insert((&r.from, &r.to)) { + return Err(bad("an ordered pair has one relation")); + } + } + if let Some(i) = items + .iter() + .find(|i| !i.withdrawn && !seen.contains(&i.speaker)) + { + return Err(StoreRefusal::SceneSpeakerMissing { + group: s.group, + entity: i.speaker.clone(), + } + .into()); + } + let inv = inventory.inventory()?; + let mut out = Vec::new(); + if s.relations.is_empty() { + out = feature_refusals(&inv, s.inventory_rev, &[], Slot::Scene); + } + for r in &s.relations { + out.extend(feature_refusals( + &inv, + s.inventory_rev, + &r.features, + Slot::Scene, + )); + } + if !out.is_empty() { + return Err(StoreError::Invalid(out)); + } + let latest = self.scene(&s.group)?.map_or(0, |l| l.n); + check_next(&key, latest, s.n)?; + self.cache.put(key, &s)?; + Ok(AddOutcome::Added) + } + + /// The latest revision (highest `n`) of the entity's speaker document. + pub fn speaker(&self, entity: &EntityId) -> anyhow::Result> { + Ok(self + .all_speakers()? + .into_iter() + .filter(|s| &s.entity == entity) + .max_by_key(|s| s.n)) + } + + /// The latest revision (highest `n`) of the group's scene document. + pub fn scene(&self, group: &str) -> anyhow::Result> { + Ok(self + .all_scenes()? + .into_iter() + .filter(|s| s.group == group) + .max_by_key(|s| s.n)) + } + + pub fn get_speaker(&self, key: &str) -> anyhow::Result> { + self.cache.get::(key) + } + + pub fn get_scene(&self, key: &str) -> anyhow::Result> { + self.cache.get::(key) + } + + /// Every speaker revision, by entity then revision number. + pub fn all_speakers(&self) -> anyhow::Result> { + let mut all = self.cache.get_all::()?; + all.sort_by(|a, b| (&a.entity, a.n).cmp(&(&b.entity, b.n))); + Ok(all) + } + + /// Every scene revision, by group then revision number. + pub fn all_scenes(&self) -> anyhow::Result> { + let mut all = self.cache.get_all::()?; + all.sort_by(|a, b| (&a.group, a.n).cmp(&(&b.group, b.n))); + Ok(all) + } +} + +fn known_entity(entities: &EntityStore, id: &EntityId) -> Result<(), StoreError> { + match entities.get(id)? { + Some(_) => Ok(()), + None => Err(StoreRefusal::UnknownEntity { entity: id.clone() }.into()), + } +} + +fn check_annotator(key: &str, a: &Annotator) -> Result<(), StoreError> { + match a { + Annotator::Model { generator, .. } if !is_attestation(generator) => { + Err(malformed(key, "a model annotator attests its generator")) + } + _ => Ok(()), + } +} + +fn check_next(key: &str, latest: u32, got: u32) -> Result<(), StoreError> { + let expected = latest + .checked_add(1) + .ok_or_else(|| malformed(key, "no revision follows the latest"))?; + if got != expected { + return Err(StoreRefusal::RevisionNotNext { + key: key.to_string(), + expected, + got, + } + .into()); + } + Ok(()) +} diff --git a/crates/weksa/src/validate.rs b/crates/weksa/src/validate.rs index e5a917e..9641062 100644 --- a/crates/weksa/src/validate.rs +++ b/crates/weksa/src/validate.rs @@ -3,9 +3,11 @@ //! returns `Ok(())` or the refusals. [`LoweringInput`], the only thing a //! lowerer sees, comes from `PacketStore::admit` alone, which builds it through //! the crate-private `admissible`. The stance is checked like every feature -//! list, in slot `Stance`. The rules are V1..V8, V10..V12 and V14..V15: the +//! list, in slot `Stance`. The rules are V1..V8, V10..V12 and V14..V16: the //! shape of an id or a grade is its type's parse, not a rule, so V9 is retired -//! and not renumbered; V13 is retired too. +//! and not renumbered; V13 is retired too. [`LoweringInput::with_scene`] takes +//! a [`Scene`](crate::speaker::Scene) document, and `feature_refusals` is the +//! one feature check it and the speaker-state store share. //! //! `validate` owns form and capacity and makes no leak decision. A packet's one //! free text is its trace (`notes` and `rejected_readings`): `validate` does not @@ -27,9 +29,10 @@ use std::fmt; use crate::inventory::{Family, InvId, Inventory, InventoryEntry, Slot, ValueKind}; use crate::packet::{ - Constraints, Context, Discourse, Extension, FeatValue, FeatureValue, NodeId, Packet, + Constraints, Context, Discourse, EntityId, Extension, FeatValue, FeatureValue, NodeId, Packet, PacketKind, Predication, ProvenanceSource, Referent, Relation, SCHEMA_VERSION, Stance, }; +use crate::speaker::Scene; /// A broken rule, named. #[derive(Clone, Debug, PartialEq, Eq)] @@ -107,6 +110,8 @@ pub enum Refusal { V15ConsentSource { source: ProvenanceSource, }, + /// V16: `with_scene` was given a speaker the scene does not list. + V16SpeakerNotInScene, } impl Refusal { @@ -129,6 +134,7 @@ impl Refusal { "V14" } Refusal::V15ConsentSource { .. } => "V15", + Refusal::V16SpeakerNotInScene => "V16", } } } @@ -154,6 +160,7 @@ impl fmt::Display for Refusal { Refusal::V14ConsentWithoutAddressee => "consent given without an addressee", Refusal::V14ConsentNotAPredication { .. } => "consent names no predication", Refusal::V15ConsentSource { .. } => "consent set by a source that only read wording", + Refusal::V16SpeakerNotInScene => "the speaker is not a participant of the scene", }; write!(f, "{}: {what}", self.rule()) } @@ -465,6 +472,29 @@ impl<'a> Inventoried<'a> { } } +/// The one feature check of the speaker-state documents and `with_scene`: V10 +/// when `rev` has no record, else the rules the packet's own check applies to a +/// feature list in `slot` (V5 to V8, V11). +pub(crate) fn feature_refusals( + inventory: &Inventory, + rev: u32, + features: &[FeatureValue], + slot: Slot, +) -> Vec { + let mut out = Vec::new(); + if !inventory.has_revision(rev) { + out.push(Refusal::V10UnknownRevision { rev }); + } else { + Inventoried { + inv: inventory, + rev, + out: &mut out, + } + .features(features, slot); + } + out +} + fn fits(kind: Option<&ValueKind>, value: &FeatValue) -> bool { match (kind, value) { (Some(ValueKind::Grade), FeatValue::Grade(_)) => true, @@ -530,6 +560,27 @@ fn fits(kind: Option<&ValueKind>, value: &FeatValue) -> bool { /// fn read(l: weksa::LoweringInput) { let _ = l.scene; } /// ``` /// +/// Scene values come only from a `weksa.scene` document: the raw-values door +/// is gone. The first block is the same call with a document; the second hands +/// it feature values and fails for that alone. +/// +/// ``` +/// fn pick( +/// l: weksa::LoweringInput, +/// i: &weksa::Inventory, +/// s: &weksa::speaker::Scene, +/// e: &weksa::EntityId, +/// ) { +/// let _ = l.with_scene(i, s, e); +/// } +/// ``` +/// +/// ```compile_fail +/// fn pick(l: weksa::LoweringInput, i: &weksa::Inventory) { +/// let _ = l.with_scene(i, vec![]); +/// } +/// ``` +/// /// The constructor is private: a call from outside the crate compiles only if /// `LoweringInput::of` is `pub`. The first block is the same code without the /// call, so the second fails for that call and nothing else. @@ -587,6 +638,39 @@ impl LoweringInput { } } + /// Adds the speaker's relation to the addressee, the only path for scene + /// values into a view; they come only from a `weksa.scene` document. The + /// speaker must be a participant of `scene` (V16). With no addressee, or + /// one that names no referent, nothing is selected; otherwise the relation + /// from the speaker to the addressee referent's entity (to `Unnamed` when + /// the referent has none) is checked like every feature list, in slot + /// `Scene` at this view's inventory revision, and replaces any earlier + /// scene. The English lowerer ignores it. + pub fn with_scene( + mut self, + inventory: &Inventory, + scene: &Scene, + speaker: &EntityId, + ) -> Result> { + if !scene.participants.contains(speaker) { + return Err(vec![Refusal::V16SpeakerNotInScene]); + } + let addressee = self + .discourse + .addressee + .as_ref() + .and_then(|id| self.referents.iter().find(|r| &r.id == id)); + let selected = addressee + .map(|r| scene.relation(speaker, r.entity.as_ref())) + .unwrap_or_default(); + let out = feature_refusals(inventory, self.inventory_rev, selected, Slot::Scene); + if !out.is_empty() { + return Err(out); + } + self.scene = selected.to_vec(); + Ok(self) + } + pub fn inventory_rev(&self) -> u32 { self.inventory_rev } @@ -617,7 +701,8 @@ impl LoweringInput { pub fn stance(&self) -> &Stance { &self.stance } - /// The speaker's relation to the addressee; empty until `with_scene`. + /// The features of the scene relation from the speaker to the addressee; + /// empty until `with_scene`, or when the scene holds none. pub fn scene(&self) -> &[FeatureValue] { &self.scene } diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 18edb84..4b439b0 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -308,7 +308,9 @@ fn schema_writes_the_published_files() { "weksa.entity.schema.json", "weksa.interlingua_packet.schema.json", "weksa.inventory_entry.schema.json", - "weksa.inventory_revision.schema.json" + "weksa.inventory_revision.schema.json", + "weksa.scene.schema.json", + "weksa.speaker.schema.json" ] ); } diff --git a/crates/weksa/tests/scene.rs b/crates/weksa/tests/scene.rs new file mode 100644 index 0000000..bc64acc --- /dev/null +++ b/crates/weksa/tests/scene.rs @@ -0,0 +1,244 @@ +//! The speaker's relation to the addressee as a lowering input: the relation of +//! a `weksa.scene` document from the speaker to the addressee referent's +//! entity, typed feature values of slot `Scene` checked against the inventory +//! at the view's revision, never wording. + +mod common; +use std::path::Path; + +use common::{eid, grade, iv, lab, nd}; +use weksa::inventory::{proof_revision, seed, speaker_scene_revision}; +use weksa::packet::{FeatValue, FeatureValue, Packet, proof_packet}; +use weksa::speaker::{Annotator, PairRelation, Party, Scene}; +use weksa::{ + CorpusStore, EntityId, EntityStore, Inventory, InventoryStore, LoweringInput, PacketStore, + Refusal, lower, +}; + +fn store() -> (InventoryStore, Inventory, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let mut store = InventoryStore::open(&dir.path().join("i.cc")).unwrap(); + for (rev, entries) in [seed(), proof_revision(), speaker_scene_revision()] { + store.admit_revision(rev, entries).unwrap(); + } + let inv = store.inventory().unwrap(); + (store, inv, dir) +} + +fn admitted(packet: &Packet, inventory: &InventoryStore) -> LoweringInput { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let corpus = CorpusStore::open(&root.join("data/corpus/corpus.cc")).unwrap(); + let entities = EntityStore::open(&root.join("data/entity/entity.cc")).unwrap(); + let dir = tempfile::tempdir().unwrap(); + PacketStore::open(&dir.path().join("p.cc")) + .unwrap() + .admit(packet.clone(), inventory, &corpus, &entities) + .unwrap() +} + +/// The proof packet at `rev`. Its referent `n1` has no entity and `n3` is +/// `entity:nibu`; `addressee` names which one is spoken to, or none. +fn packet(rev: u32, addressee: Option<&str>) -> Packet { + let mut p = proof_packet(); + p.inventory_rev = rev; + p.discourse.addressee = addressee.map(nd); + p +} + +fn feat(inv: &Inventory, gloss: &str, value: FeatValue) -> FeatureValue { + let id = inv + .entries() + .find(|e| e.gloss == gloss) + .unwrap_or_else(|| panic!("no feature {gloss}")) + .id; + FeatureValue { feature: id, value } +} + +fn rules(r: Result>) -> Vec<&'static str> { + r.unwrap_err().iter().map(Refusal::rule).collect() +} + +fn speaker() -> EntityId { + eid("brokkr") +} + +/// Brokkr speaks; the scene holds a relation to Nibu and one to an unnamed +/// party, with different values. +fn scene(inv: &Inventory) -> Scene { + scene_with(vec![ + PairRelation { + from: speaker(), + to: Party::Named(eid("nibu")), + features: vec![ + feat(inv, "relative_status", FeatValue::Option(lab("above"))), + feat(inv, "intimacy", FeatValue::Grade(grade(200))), + feat(inv, "in_group", FeatValue::Flag(false)), + ], + }, + PairRelation { + from: speaker(), + to: Party::Unnamed, + features: vec![feat(inv, "intimacy", FeatValue::Grade(grade(7)))], + }, + ]) +} + +fn scene_with(relations: Vec) -> Scene { + Scene { + group: "g".into(), + n: 1, + inventory_rev: 3, + participants: vec![speaker(), eid("nibu")], + relations, + annotator: Annotator::Operator, + } +} + +fn selected(view: LoweringInput, inv: &Inventory, s: &Scene) -> Vec { + view.with_scene(inv, s, &speaker()) + .unwrap() + .scene() + .to_vec() +} + +#[test] +fn a_scene_relation_reaches_the_lowering_view_and_changes_no_english() { + let (store, inv, _dir) = store(); + let plain = admitted(&packet(3, Some("n3")), &store); + assert!(plain.scene().is_empty()); + let s = scene(&inv); + let with = plain.clone().with_scene(&inv, &s, &speaker()).unwrap(); + assert_eq!(with.scene(), s.relations[0].features.as_slice()); + assert!(!with.scene().is_empty()); + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let entities = EntityStore::open(&root.join("data/entity/entity.cc")).unwrap(); + assert_eq!( + lower(&with, &inv, &entities).unwrap(), + lower(&plain, &inv, &entities).unwrap() + ); +} + +#[test] +fn with_scene_selects_by_addressee() { + let (store, inv, _dir) = store(); + let s = scene(&inv); + // A referent with an entity selects the relation to that entity. + let named = selected(admitted(&packet(3, Some("n3")), &store), &inv, &s); + assert_eq!(named, s.relations[0].features); + // A referent without one selects the relation to an unnamed party. + let unnamed = selected(admitted(&packet(3, Some("n1")), &store), &inv, &s); + assert_eq!(unnamed, s.relations[1].features); + // No addressee selects nothing. + assert!(selected(admitted(&packet(3, None), &store), &inv, &s).is_empty()); + // A named addressee the scene lacks selects nothing, though an unnamed + // relation exists. + let only_unnamed = scene_with(vec![s.relations[1].clone()]); + assert!( + selected( + admitted(&packet(3, Some("n3")), &store), + &inv, + &only_unnamed + ) + .is_empty() + ); + // Another speaker's relations are not selected. + let view = admitted(&packet(3, Some("n3")), &store); + let theirs = Scene { + participants: vec![speaker(), eid("nibu"), eid("hero")], + ..s.clone() + }; + assert!( + view.with_scene(&inv, &theirs, &eid("hero")) + .unwrap() + .scene() + .is_empty() + ); + // A replacement replaces: an empty selection clears the earlier one. + let first = admitted(&packet(3, Some("n3")), &store) + .with_scene(&inv, &s, &speaker()) + .unwrap(); + assert!( + first + .with_scene(&inv, &only_unnamed, &speaker()) + .unwrap() + .scene() + .is_empty() + ); +} + +#[test] +fn with_scene_refusals() { + let (store, inv, _dir) = store(); + let view = admitted(&packet(3, Some("n3")), &store); + let to_nibu = |features| { + scene_with(vec![PairRelation { + from: speaker(), + to: Party::Named(eid("nibu")), + features, + }]) + }; + let take = |s: &Scene| view.clone().with_scene(&inv, s, &speaker()); + // A speaker the scene does not list. + assert_eq!( + view.clone() + .with_scene(&inv, &scene(&inv), &eid("hero")) + .map(|_| ()) + .unwrap_err(), + [Refusal::V16SpeakerNotInScene] + ); + // A label the feature does not declare. + let lordly = feat(&inv, "relative_status", FeatValue::Option(lab("lordly"))); + assert_eq!(rules(take(&to_nibu(vec![lordly]))), ["V7"]); + // A value of the wrong kind. + let flag = feat(&inv, "intimacy", FeatValue::Flag(true)); + assert_eq!(rules(take(&to_nibu(vec![flag]))), ["V7"]); + // A stance feature in a relation. + let anger = feat(&inv, "anger", FeatValue::Grade(grade(1))); + assert_eq!(rules(take(&to_nibu(vec![anger]))), ["V8"]); + // A speaker feature in a relation. + let age = feat(&inv, "education", FeatValue::Grade(grade(1))); + assert_eq!(rules(take(&to_nibu(vec![age]))), ["V8"]); + // Twice in one list. + let twice = vec![ + feat(&inv, "intimacy", FeatValue::Grade(grade(1))), + feat(&inv, "intimacy", FeatValue::Grade(grade(2))), + ]; + assert_eq!(rules(take(&to_nibu(twice))), ["V11"]); + // Revision 2 predates the Scene features, so their ids do not resolve there. + let old = admitted(&packet(2, Some("n3")), &store); + assert_eq!( + rules(old.with_scene(&inv, &scene(&inv), &speaker())), + ["V5", "V5", "V5"] + ); + // An unselected relation is not checked: only the selected one is. + let bad_elsewhere = scene_with(vec![ + PairRelation { + from: speaker(), + to: Party::Unnamed, + features: vec![feat(&inv, "anger", FeatValue::Grade(grade(1)))], + }, + PairRelation { + from: speaker(), + to: Party::Named(eid("nibu")), + features: vec![], + }, + ]); + assert!(take(&bad_elsewhere).unwrap().scene().is_empty()); +} + +#[test] +fn feature_ids_are_stable() { + let (_store, inv, _dir) = store(); + let ids: Vec<_> = [ + "relative_status", + "intimacy", + "in_group", + "socioeconomic_standing", + "education", + "age_cohort", + ] + .map(|g| feat(&inv, g, FeatValue::Flag(true)).feature) + .into(); + let want: Vec<_> = (19..=24).map(|n| iv(&format!("feature:{n}"))).collect(); + assert_eq!(ids, want); +} diff --git a/crates/weksa/tests/speaker_state.rs b/crates/weksa/tests/speaker_state.rs new file mode 100644 index 0000000..864d65a --- /dev/null +++ b/crates/weksa/tests/speaker_state.rs @@ -0,0 +1,478 @@ +//! The speaker-state store: speaker and scene documents are append-only +//! revisions, checked against the inventory, the entity store and the corpus. + +mod common; +use common::{cast, eid, entities_in, grade, item_in, lab}; +use weksa::inventory::{proof_revision, seed, speaker_scene_revision}; +use weksa::packet::{FeatValue, FeatureValue}; +use weksa::speaker::{ + Annotator, MAX_PARTICIPANTS, PairRelation, Party, Scene, Speaker, scene_key, speaker_key, +}; +use weksa::{ + AddOutcome, CorpusStore, EntityStore, InventoryStore, Refusal, SpeakerStateStore, StoreError, + StoreRefusal, +}; + +struct W { + dir: tempfile::TempDir, + inventory: InventoryStore, + corpus: CorpusStore, + entities: EntityStore, + state: SpeakerStateStore, +} + +/// Group `g` holds two live items (speakers nibu and brokkr) and one withdrawn +/// item (speaker hero). +fn world() -> W { + let dir = tempfile::tempdir().unwrap(); + let mut inventory = InventoryStore::open(&dir.path().join("i.cc")).unwrap(); + for (rev, entries) in [seed(), proof_revision(), speaker_scene_revision()] { + inventory.admit_revision(rev, entries).unwrap(); + } + let entities = entities_in(dir.path()); + let mut corpus = CorpusStore::open(&dir.path().join("c.cc")).unwrap(); + for (key, text, speaker) in [ + ("item:nibu:1", "Come here.", "nibu"), + ("item:nibu:2", "Go there.", "brokkr"), + ("item:nibu:3", "Stay away.", "hero"), + ] { + let mut item = item_in(key, text, "g", cast()); + item.speaker = eid(speaker); + corpus.add(item, &entities).unwrap(); + } + corpus.withdraw("item:nibu:3").unwrap(); + let state = SpeakerStateStore::open(&dir.path().join("s.cc")).unwrap(); + W { + dir, + inventory, + corpus, + entities, + state, + } +} + +fn feat(gloss: &str, value: FeatValue) -> FeatureValue { + let feature = seed() + .1 + .into_iter() + .chain(speaker_scene_revision().1) + .find(|e| e.gloss == gloss) + .unwrap_or_else(|| panic!("no feature {gloss}")) + .id; + FeatureValue { feature, value } +} + +fn speaker(n: u32, age: &str) -> Speaker { + Speaker { + entity: eid("nibu"), + n, + inventory_rev: 3, + features: vec![feat("age_cohort", FeatValue::Option(lab(age)))], + annotator: Annotator::Operator, + } +} + +fn scene(n: u32, intimacy: u16) -> Scene { + Scene { + group: "g".into(), + n, + inventory_rev: 3, + participants: vec![eid("nibu"), eid("brokkr")], + relations: vec![PairRelation { + from: eid("nibu"), + to: Party::Named(eid("brokkr")), + features: vec![feat("intimacy", FeatValue::Grade(grade(intimacy)))], + }], + annotator: Annotator::Operator, + } +} + +fn put_speaker(w: &mut W, s: Speaker) -> Result { + w.state.put_speaker(s, &w.inventory, &w.entities) +} + +fn put_scene(w: &mut W, s: Scene) -> Result { + w.state.put_scene(s, &w.inventory, &w.corpus, &w.entities) +} + +fn refused(r: Result) -> StoreRefusal { + match r.unwrap_err() { + StoreError::Refused(r) => r, + other => panic!("not a store refusal: {other}"), + } +} + +fn malformed(r: Result, key: &str, why: &str) { + match refused(r) { + StoreRefusal::Malformed { key: k, why: w } => assert_eq!((k.as_str(), w), (key, why)), + other => panic!("not Malformed: {other:?}"), + } +} + +fn rules(r: Result) -> Vec<&'static str> { + match r.unwrap_err() { + StoreError::Invalid(rs) => rs.iter().map(Refusal::rule).collect(), + other => panic!("not a rule refusal: {other}"), + } +} + +fn not_next(key: &str, expected: u32, got: u32) -> StoreRefusal { + StoreRefusal::RevisionNotNext { + key: key.into(), + expected, + got, + } +} + +#[test] +fn keys_derive_from_the_document() { + assert_eq!(speaker_key(&eid("nibu"), 2), "speaker:nibu@2"); + assert_eq!(scene_key("g", 3), "scene:g@3"); +} + +#[test] +fn speaker_revisions_append() { + let mut w = world(); + let key = |n| speaker_key(&eid("nibu"), n); + assert_eq!( + refused(put_speaker(&mut w, speaker(2, "adult"))), + not_next(&key(2), 1, 2) + ); + assert!(w.state.speaker(&eid("nibu")).unwrap().is_none()); + assert_eq!( + put_speaker(&mut w, speaker(1, "adult")).unwrap(), + AddOutcome::Added + ); + assert_eq!( + put_speaker(&mut w, speaker(1, "adult")).unwrap(), + AddOutcome::Unchanged + ); + assert_eq!( + refused(put_speaker(&mut w, speaker(1, "elder"))), + not_next(&key(1), 2, 1) + ); + assert_eq!( + refused(put_speaker(&mut w, speaker(3, "elder"))), + not_next(&key(3), 2, 3) + ); + assert_eq!( + put_speaker(&mut w, speaker(2, "elder")).unwrap(), + AddOutcome::Added + ); + assert_eq!( + w.state.speaker(&eid("nibu")).unwrap(), + Some(speaker(2, "elder")) + ); + assert_eq!( + w.state.get_speaker(&key(1)).unwrap(), + Some(speaker(1, "adult")) + ); + // Another entity's revisions count from 1 and stay apart. + let mut other = speaker(1, "child"); + other.entity = eid("brokkr"); + assert_eq!(put_speaker(&mut w, other).unwrap(), AddOutcome::Added); + assert_eq!(w.state.speaker(&eid("nibu")).unwrap().unwrap().n, 2); + // Revision 10 sorts after 2 in the listing. + for n in 3..=10 { + put_speaker(&mut w, speaker(n, "adult")).unwrap(); + } + let listed: Vec<_> = w + .state + .all_speakers() + .unwrap() + .iter() + .map(|s| (s.entity.to_string(), s.n)) + .collect(); + assert_eq!(listed.len(), 11); + assert_eq!(listed[0], ("entity:brokkr".into(), 1)); + assert_eq!(listed[10], ("entity:nibu".into(), 10)); + assert_eq!(w.state.speaker(&eid("nibu")).unwrap().unwrap().n, 10); + // The documents survive a reopen. + let again = SpeakerStateStore::open(&w.dir.path().join("s.cc")).unwrap(); + assert_eq!(again.all_speakers().unwrap().len(), 11); +} + +#[test] +fn scene_revisions_append() { + let mut w = world(); + let key = |n| scene_key("g", n); + assert_eq!( + refused(put_scene(&mut w, scene(2, 5))), + not_next(&key(2), 1, 2) + ); + assert!(w.state.scene("g").unwrap().is_none()); + assert_eq!(put_scene(&mut w, scene(1, 5)).unwrap(), AddOutcome::Added); + assert_eq!( + put_scene(&mut w, scene(1, 5)).unwrap(), + AddOutcome::Unchanged + ); + assert_eq!( + refused(put_scene(&mut w, scene(1, 6))), + not_next(&key(1), 2, 1) + ); + assert_eq!(put_scene(&mut w, scene(2, 6)).unwrap(), AddOutcome::Added); + assert_eq!(w.state.scene("g").unwrap(), Some(scene(2, 6))); + assert_eq!(w.state.get_scene(&key(1)).unwrap(), Some(scene(1, 5))); + for n in 3..=10 { + put_scene(&mut w, scene(n, 6)).unwrap(); + } + assert_eq!(w.state.scene("g").unwrap().unwrap().n, 10); + let listed: Vec = w.state.all_scenes().unwrap().iter().map(|s| s.n).collect(); + assert_eq!(listed, (1..=10).collect::>()); +} + +#[test] +fn speaker_needs_a_known_entity() { + let mut w = world(); + let mut s = speaker(1, "adult"); + s.entity = eid("nobody"); + assert_eq!( + refused(put_speaker(&mut w, s)), + StoreRefusal::UnknownEntity { + entity: eid("nobody") + } + ); + assert!(w.state.all_speakers().unwrap().is_empty()); +} + +#[test] +fn annotation_features_are_checked_like_packet_features() { + let mut w = world(); + let age = || feat("age_cohort", FeatValue::Option(lab("adult"))); + let with = |features: Vec, rev| Speaker { + features, + inventory_rev: rev, + ..speaker(1, "adult") + }; + // A Scene feature and a Stance feature do not apply to a speaker. + let f = vec![feat("intimacy", FeatValue::Grade(grade(1)))]; + assert_eq!(rules(put_speaker(&mut w, with(f, 3))), ["V8"]); + let f = vec![feat("anger", FeatValue::Grade(grade(1)))]; + assert_eq!(rules(put_speaker(&mut w, with(f, 3))), ["V8"]); + // A wrong kind and an undeclared label. + let f = vec![feat("education", FeatValue::Flag(true))]; + assert_eq!(rules(put_speaker(&mut w, with(f, 3))), ["V7"]); + let f = vec![feat("age_cohort", FeatValue::Option(lab("ancient")))]; + assert_eq!(rules(put_speaker(&mut w, with(f, 3))), ["V7"]); + // Feature 22 does not exist at revision 2; revision 9 has no record. + assert_eq!(rules(put_speaker(&mut w, with(vec![age()], 2))), ["V5"]); + assert_eq!(rules(put_speaker(&mut w, with(vec![age()], 9))), ["V10"]); + // A repeat. + assert_eq!( + rules(put_speaker(&mut w, with(vec![age(), age()], 3))), + ["V11"] + ); + // A Speaker feature in a scene relation. + let mut s = scene(1, 1); + s.relations[0].features = vec![age()]; + assert_eq!(rules(put_scene(&mut w, s)), ["V8"]); + // Every relation is checked, and a scene of an unknown revision is V10. + let mut s = scene(1, 1); + s.relations.push(PairRelation { + from: eid("brokkr"), + to: Party::Unnamed, + features: vec![age()], + }); + assert_eq!(rules(put_scene(&mut w, s)), ["V8"]); + let mut s = scene(1, 1); + s.inventory_rev = 9; + assert_eq!(rules(put_scene(&mut w, s)), ["V10"]); + let mut s = scene(1, 1); + s.inventory_rev = 9; + s.relations.clear(); + assert_eq!(rules(put_scene(&mut w, s)), ["V10"]); + assert!(w.state.all_speakers().unwrap().is_empty()); + assert!(w.state.all_scenes().unwrap().is_empty()); +} + +#[test] +fn scene_shape_rules() { + let mut w = world(); + let key = scene_key("g", 1); + let relation = |from: &str, to: Party| PairRelation { + from: eid(from), + to, + features: vec![], + }; + let named = |to: &str| Party::Named(eid(to)); + let base = || scene(1, 1); + + let mut s = base(); + s.group = "Not A Slug".into(); + malformed( + put_scene(&mut w, s), + &scene_key("Not A Slug", 1), + "a scene group is a slug", + ); + let mut s = base(); + s.group = "elsewhere".into(); + malformed( + put_scene(&mut w, s), + &scene_key("elsewhere", 1), + "no corpus item holds the scene's group", + ); + let mut s = base(); + s.participants.clear(); + s.relations.clear(); + malformed( + put_scene(&mut w, s), + &key, + "a scene has 1 to 64 participants", + ); + // 64 participants are fine as far as the count goes (the rest fail later); + // 65 are refused for the count. + let many = |count: usize| -> Vec<_> { (0..count).map(|i| eid(&format!("p{i}"))).collect() }; + let mut s = base(); + s.participants = many(MAX_PARTICIPANTS + 1); + s.relations.clear(); + malformed( + put_scene(&mut w, s), + &key, + "a scene has 1 to 64 participants", + ); + let mut s = base(); + s.participants = many(MAX_PARTICIPANTS); + s.relations.clear(); + assert!(matches!( + refused(put_scene(&mut w, s)), + StoreRefusal::UnknownEntity { .. } + )); + let mut s = base(); + s.participants.push(eid("nibu")); + malformed(put_scene(&mut w, s), &key, "a participant is named once"); + let mut s = base(); + s.participants.push(eid("nobody")); + assert_eq!( + refused(put_scene(&mut w, s)), + StoreRefusal::UnknownEntity { + entity: eid("nobody") + } + ); + let mut s = base(); + s.relations = vec![relation("hero", named("nibu"))]; + malformed( + put_scene(&mut w, s), + &key, + "a relation's from is a participant", + ); + let mut s = base(); + s.relations = vec![relation("nibu", named("hero"))]; + malformed( + put_scene(&mut w, s), + &key, + "a relation's named to is another participant", + ); + let mut s = base(); + s.relations = vec![relation("nibu", named("nibu"))]; + malformed( + put_scene(&mut w, s), + &key, + "a relation's named to is another participant", + ); + let mut s = base(); + s.relations = vec![ + relation("nibu", named("brokkr")), + relation("nibu", named("brokkr")), + ]; + malformed( + put_scene(&mut w, s), + &key, + "an ordered pair has one relation", + ); + let mut s = base(); + s.relations = vec![ + relation("nibu", Party::Unnamed), + relation("nibu", Party::Unnamed), + ]; + malformed( + put_scene(&mut w, s), + &key, + "an ordered pair has one relation", + ); + // The reverse pair, and an unnamed party beside a named one, are distinct. + let mut s = base(); + s.relations = vec![ + relation("nibu", named("brokkr")), + relation("brokkr", named("nibu")), + relation("nibu", Party::Unnamed), + ]; + assert_eq!(put_scene(&mut w, s).unwrap(), AddOutcome::Added); +} + +#[test] +fn a_scene_holds_every_live_speaker_of_its_group() { + let mut w = world(); + // Brokkr speaks a live item of the group. + let mut s = scene(1, 1); + s.participants = vec![eid("nibu")]; + s.relations.clear(); + assert_eq!( + refused(put_scene(&mut w, s)), + StoreRefusal::SceneSpeakerMissing { + group: "g".into(), + entity: eid("brokkr") + } + ); + // Hero speaks only a withdrawn item, so the scene may omit them. + assert_eq!(put_scene(&mut w, scene(1, 1)).unwrap(), AddOutcome::Added); + // The group is held by a withdrawn item alone: still a group. + w.corpus.withdraw("item:nibu:1").unwrap(); + w.corpus.withdraw("item:nibu:2").unwrap(); + let mut s = scene(2, 1); + s.participants = vec![eid("player")]; + s.relations.clear(); + assert_eq!(put_scene(&mut w, s).unwrap(), AddOutcome::Added); +} + +#[test] +fn model_annotator_attests_its_generator() { + let mut w = world(); + let model = |generator: &str| Annotator::Model { + family: weksa::corpus::ModelFamily::Anthropic, + generator: generator.into(), + }; + let key = speaker_key(&eid("nibu"), 1); + for bad in ["", " ", "gen\u{7}erator", "gen\nerator", "\u{200d}"] { + let mut s = speaker(1, "adult"); + s.annotator = model(bad); + malformed( + put_speaker(&mut w, s), + &key, + "a model annotator attests its generator", + ); + let mut s = scene(1, 1); + s.annotator = model(bad); + malformed( + put_scene(&mut w, s), + &scene_key("g", 1), + "a model annotator attests its generator", + ); + } + let mut s = speaker(1, "adult"); + s.annotator = model(&"x".repeat(201)); + malformed( + put_speaker(&mut w, s), + &key, + "a model annotator attests its generator", + ); + let mut s = speaker(1, "adult"); + s.annotator = model("claude-sonnet 5.5"); + assert_eq!(put_speaker(&mut w, s).unwrap(), AddOutcome::Added); + let mut s = scene(1, 1); + s.annotator = model("claude-sonnet 5.5"); + assert_eq!(put_scene(&mut w, s).unwrap(), AddOutcome::Added); + let mut s = speaker(2, "adult"); + s.annotator = Annotator::Operator; + assert_eq!(put_speaker(&mut w, s).unwrap(), AddOutcome::Added); +} + +#[test] +fn refusal_displays_print_numbers_and_no_identifiers() { + let r = not_next("speaker:secretname@2", 1, 2); + assert_eq!(r.to_string(), "RevisionNotNext expected 1 got 2"); + let r = StoreRefusal::SceneSpeakerMissing { + group: "secretgroup".into(), + entity: eid("secretname"), + }; + assert_eq!(r.to_string(), "SceneSpeakerMissing"); +} diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs index fbcce9a..fa51e47 100644 --- a/crates/weksa/tests/stores.rs +++ b/crates/weksa/tests/stores.rs @@ -1103,6 +1103,10 @@ fn speaker_state_slots_are_exclusive() { &[Slot::Scene, Slot::Relation], &[Slot::Discourse, Slot::Scene], &[Slot::Scene, Slot::Scene], + &[Slot::Speaker, Slot::Scene], + &[Slot::Speaker, Slot::Stance], + &[Slot::Speaker, Slot::Relation], + &[Slot::Speaker, Slot::Speaker], &[Slot::Stance, Slot::Stance], ] { let rev = rev_with(vec![Change::Add(iv("feature:50"))], 2, Some(1)); @@ -1121,7 +1125,8 @@ fn speaker_state_slots_are_exclusive() { for (n, slots) in [ (2, &[Slot::Scene][..]), (3, &[Slot::Stance]), - (4, &[Slot::Discourse, Slot::Predication]), + (4, &[Slot::Speaker]), + (5, &[Slot::Discourse, Slot::Predication]), ] { let id = format!("feature:{}", 50 + n); let rev = rev_with(vec![Change::Add(iv(&id))], n, Some(n - 1)); diff --git a/docs/interlingua-standard.md b/docs/interlingua-standard.md index e349a90..830a2f7 100644 --- a/docs/interlingua-standard.md +++ b/docs/interlingua-standard.md @@ -213,6 +213,25 @@ affect and restraint to its own register. A held-back feeling is not a claim about the world, so it is never a predication. An empty stance is valid. The stance reaches the lowerer in the lowering view. +### Speaker and scene documents + +Two documents outside the packet hold the speaker's standing, in the store that +owns them (`SpeakerStateStore`). `weksa.speaker` (key `speaker:@`) +holds one entity's stable axes as feature values of slot `Speaker`. +`weksa.scene` (key `scene:@`) holds a corpus group's participants and, +per ordered pair, the speaker's relation to a party (slot `Scene`); the party is +a named participant or `Party::Unnamed`, an addressee the scene does not name, +such as the player or a crowd. A revision appends and never edits: `n` is the +latest plus one, and the latest revision is the highest `n`. Each carries an +`Annotator`, `Operator` or `Model` with an attestation string for its generator. +Both hold typed feature values only, never text. A feature of slot `Stance`, +`Speaker` or `Scene` sits in that slot alone (each layer has one writer and one +lifetime), which `admit_revision` enforces. `LoweringInput::with_scene` takes a +scene document and the speaker; the speaker must be a participant (V16), and +the relation selected is the speaker's to the entity of the addressee referent +(to `Unnamed` when that referent has no entity, to nothing when the packet has +no addressee). V13 is retired. + ### `referents` Entities, groups, places, abstractions and the rest. `kind` is a referent-kind diff --git a/docs/targets/dao-de-jing-test-bed.md b/docs/targets/dao-de-jing-test-bed.md index 4f4de2e..516e2e3 100644 --- a/docs/targets/dao-de-jing-test-bed.md +++ b/docs/targets/dao-de-jing-test-bed.md @@ -15,7 +15,7 @@ plus a separately recorded chosen gloss per lowering). - `data/daodejing/`: its own stores (inventory revisions 1 to 3, one entity, five corpus items, five packets), apart from the production stores in `data/`. Revision 3 is `weksa::daodejing::revision()`; the packets are - `weksa::daodejing::packets()`. The scene revision (`scene_revision()`, also + `weksa::daodejing::packets()`. The scene revision (`speaker_scene_revision()`, also numbered 3) is not applied in these stores and will need renumbering when it is applied anywhere both meet. - Rebuild: `weksa --corpus C --inventory I --entities E` with `inventory seed`, diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index d2917de..2b1959c 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -263,7 +263,8 @@ operator-attested annotations alone. Who annotates is question - Seed inventory (2026-11-10, base `5e10d7c`): features 6-8 are hedge, certainty and irony; 9-18 are the stance affects and restraint (slot Stance). The committed inventory holds revisions 1-2 only. Nothing calls `with_scene` - except tests, and the proof packet's addressee carries no entity. + except tests, and the proof packet's addressee carries no entity (its + referent `n3` does: `entity:nibu`). - Yggdrasil's Ollama (`127.0.0.1:11434`) is the Docker container `ollama-cpu`, Huginn's embedder (runbook `gamecult-ops/runbooks/huginn-yggdrasil.md`, ruling `eureka-body:ruling:huginn-embedder-yggdrasil-primary`). `/api/embed` @@ -398,8 +399,8 @@ One line per cut; specs exist for the cuts about to run (2d and 4, siblings on ` 2d. `entity-door-lows`: invisible format characters refused in entity forms (ZWJ, ZWNJ, CGJ and variation selectors kept); the form length bound per question `form-word-bound` (recommended: bytes only); attested origins (`Cast { persona, fixture }`, `Live { persona }`) and `CastAuthored.author`; empty provenance strings refused at the store; S6 under NFKC plus case fold and no-alphanumeric texts refused; `next_key` refuses at `u32::MAX`; CLI flags that do not apply refuse and no error echoes a value. Owns follow-ups `entity-door-lows` and `s6-unicode-normal-form`. (spec r1) 3b. `lowerer-first-cut` (follow-up `golf-loop-lowerer-cut`): the section above. `lower`, `weksa lower`, `weksa score`, `weksa packet proof`, inventory rev 2 for `nibu-proof-001`. 4. `speaker-state`: packet `stance` (felt affect and restraint, slot `Stance`, packet schema v4); formality, warmth and intensity out of the seed and the standard; only `PacketStore::admit` yields a `LoweringInput`, and it refuses packets for withdrawn items (follow-up `lowering-input-from-packet-store`). (spec r1) -4a. `speaker-relation-input`: slot `Scene` and its first seed features (`relative_status`, `intimacy`, `in_group`; inventory revision 3, `scene_revision`); `LoweringInput::with_scene` takes the speaker's relation to the addressee as typed `Scene` feature values, validated like every feature list, refused without an addressee (V13). The English lowerer does not render it. 4b's `weksa.scene` document is its future supplier; no store, document or CLI yet. (PR for follow-up `chat-s3352-3352`) -4b. `speaker-scene` (spec r1; `speaker-scene-seeds` is its cut 4c: CLI, inventory revision 3, cast speakers and scenes; `Party::Unnamed` covers an addressee with no entity; Speaker and Scene share one `SpeakerStateStore`): slots `Speaker` and `Scene` with their seed features; `weksa.speaker` (`speaker:@`) and `weksa.scene` (`scene:@`, participants and ordered-pair relations) documents with `Annotator` `Operator` and `Model { family, generator }`, validated by the packet's shared feature check, append-only per key; their stores, `weksa speaker` and `weksa scene` commands, published schemas; Model-annotated cast speakers and scenes for the committed items. Waits on 2d and 4; `pd-speaker-annotation` blocks only the public-domain annotator. +4a. `speaker-relation-input`: slot `Scene` and its first seed features (`relative_status`, `intimacy`, `in_group`; inventory revision 3, now `speaker_scene_revision`); `LoweringInput::with_scene` carries the speaker's relation to the addressee as typed `Scene` feature values. Since 4b it takes a `weksa.scene` document and selects the relation by the addressee referent's entity (V16 when the speaker is not a participant; V13 is retired). The English lowerer does not render it. (PR for follow-up `chat-s3352-3352`) +4b. `speaker-scene` (spec r1; `speaker-scene-seeds` is its cut 4c: CLI, inventory revision 3, cast speakers and scenes; `Party::Unnamed` covers an addressee with no entity; Speaker and Scene share one `SpeakerStateStore`): slots `Speaker` and `Scene` with their seed features; `weksa.speaker` (`speaker:@`) and `weksa.scene` (`scene:@`, participants and ordered-pair relations) documents with `Annotator` `Operator` and `Model { family, generator }`, validated by the packet's shared feature check, append-only per key; their stores, `weksa speaker` and `weksa scene` commands, published schemas; Model-annotated cast speakers and scenes for the committed items. Waits on 2d and 4; `pd-speaker-annotation` blocks only the public-domain annotator. Part 1 landed (branch `weksa-lowering/speaker-scene`): `speaker.rs`, `SpeakerStateStore`, slots `Speaker` and `Scene` exclusive to their own feature lists, `speaker_scene_revision()` (Speaker features 22-24), `with_scene` on a Scene document, schemas. The CLI, the committed inventory revision 3 and the seeds stay with 4c. 5. `language-ontology`: `weksa.community` from `data/cultural-ontology/` (owns follow-up `edda-referent-yaml-parse`), `weksa.ontology_entry` for en-US and ja-JP from `data/target-language-ontology/` (YAML deleted), speaker communities, deterministic `select`, the lowering view (packet without trace, speaker, scene, selected entries). Waits on 4. 6. `pd-importer`: public-domain play and fiction dialogue into the corpus with speaker, scene group, edition provenance, and Speaker and Scene annotations by the ruled annotator. Waits on 4, follow-up `gutenberg-terms` and question `pd-speaker-annotation`. 7a. `loop-context-builder`: `context.rs` (`TrainItems`, `EncoderOffer`, `Prompt`: corpus text reaches a model call only through `Prompt::mint` over live Train items or `Prompt::encode` over one offered item; instructions are `&'static str`), `SplitScheme::live`, `CorpusItem::cited_entities` (one owner of the id set `PacketStore::admit` and the offer share), and checkpoint-only Test readings (`Checkpoint`, `TestReadings`; `Aggregate` and `leak_diagnostic` carry no Test). Library only. Merges follow-ups `loop-context-builder-2` and `scorer-test-readings-type`. (spec r1) diff --git a/schemas/cultnet/weksa.inventory_entry.schema.json b/schemas/cultnet/weksa.inventory_entry.schema.json index ba3fe5d..e23ecc0 100644 --- a/schemas/cultnet/weksa.inventory_entry.schema.json +++ b/schemas/cultnet/weksa.inventory_entry.schema.json @@ -81,9 +81,14 @@ ] }, { - "description": "The relation of the speaker to the addressee in the scene; see\n`LoweringInput::with_scene`.", + "description": "The speaker's relation to one party of the scene, held by `weksa.scene`\ndocuments; see `LoweringInput::with_scene`.", "type": "string", "const": "Scene" + }, + { + "description": "The speaker's stable axes, held by `weksa.speaker` documents.", + "type": "string", + "const": "Speaker" } ] }, diff --git a/schemas/cultnet/weksa.scene.schema.json b/schemas/cultnet/weksa.scene.schema.json new file mode 100644 index 0000000..90e240b --- /dev/null +++ b/schemas/cultnet/weksa.scene.schema.json @@ -0,0 +1,224 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Scene", + "description": "A scene: its participants and the relations between them. Key\n`scene:@`; a revision appends and never edits.", + "type": "object", + "properties": { + "annotator": { + "$ref": "#/$defs/Annotator" + }, + "group": { + "description": "The corpus split group of the scene, a slug.", + "type": "string" + }, + "inventory_rev": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "n": { + "description": "The revision, from 1.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "participants": { + "type": "array", + "items": { + "$ref": "#/$defs/EntityId" + } + }, + "relations": { + "type": "array", + "items": { + "$ref": "#/$defs/PairRelation" + } + } + }, + "required": [ + "group", + "n", + "inventory_rev", + "participants", + "relations", + "annotator" + ], + "$defs": { + "Annotator": { + "description": "Who annotated a document. A model's generator is an attestation string, held\nto the same form as every provenance string.", + "oneOf": [ + { + "type": "string", + "enum": [ + "Operator" + ] + }, + { + "type": "object", + "properties": { + "Model": { + "type": "object", + "properties": { + "family": { + "$ref": "#/$defs/ModelFamily" + }, + "generator": { + "type": "string" + } + }, + "required": [ + "family", + "generator" + ] + } + }, + "additionalProperties": false, + "required": [ + "Model" + ] + } + ] + }, + "EntityId": { + "type": "string", + "pattern": "^entity:[a-z0-9]([a-z0-9-]{0,46}[a-z0-9])?$" + }, + "FeatValue": { + "oneOf": [ + { + "type": "object", + "properties": { + "Grade": { + "$ref": "#/$defs/Grade" + } + }, + "additionalProperties": false, + "required": [ + "Grade" + ] + }, + { + "description": "A label the feature's inventory entry declares.", + "type": "object", + "properties": { + "Option": { + "$ref": "#/$defs/OptionLabel" + } + }, + "additionalProperties": false, + "required": [ + "Option" + ] + }, + { + "type": "object", + "properties": { + "Flag": { + "type": "boolean" + } + }, + "additionalProperties": false, + "required": [ + "Flag" + ] + } + ] + }, + "FeatureValue": { + "type": "object", + "properties": { + "feature": { + "description": "Feature id.", + "$ref": "#/$defs/InvId" + }, + "value": { + "$ref": "#/$defs/FeatValue" + } + }, + "required": [ + "feature", + "value" + ] + }, + "Grade": { + "type": "integer", + "maximum": 1000, + "minimum": 0 + }, + "InvId": { + "type": "string", + "pattern": "^(concept|frame|role|relation|speech_act|feature|referent_kind|extension):[1-9][0-9]*$" + }, + "ModelFamily": { + "description": "The family of the model that wrote a line.", + "oneOf": [ + { + "type": "string", + "enum": [ + "Anthropic", + "OpenAi", + "Google", + "Alibaba", + "Meta", + "Mistral" + ] + }, + { + "description": "Nothing records which family wrote the line.", + "type": "string", + "const": "Unrecorded" + } + ] + }, + "OptionLabel": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]{0,23}$" + }, + "PairRelation": { + "description": "The speaker's relation to one party: feature values of slot `Scene`.", + "type": "object", + "properties": { + "features": { + "type": "array", + "items": { + "$ref": "#/$defs/FeatureValue" + } + }, + "from": { + "$ref": "#/$defs/EntityId" + }, + "to": { + "$ref": "#/$defs/Party" + } + }, + "required": [ + "from", + "to", + "features" + ] + }, + "Party": { + "description": "The other party of a relation. `Unnamed` is an addressee the scene does not\nname, such as the player or a crowd.", + "oneOf": [ + { + "type": "string", + "enum": [ + "Unnamed" + ] + }, + { + "type": "object", + "properties": { + "Named": { + "$ref": "#/$defs/EntityId" + } + }, + "additionalProperties": false, + "required": [ + "Named" + ] + } + ] + } + } +} diff --git a/schemas/cultnet/weksa.speaker.schema.json b/schemas/cultnet/weksa.speaker.schema.json new file mode 100644 index 0000000..4113537 --- /dev/null +++ b/schemas/cultnet/weksa.speaker.schema.json @@ -0,0 +1,171 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Speaker", + "description": "The speaker's stable axes. Key `speaker:@`; a revision\nappends and never edits.", + "type": "object", + "properties": { + "annotator": { + "$ref": "#/$defs/Annotator" + }, + "entity": { + "$ref": "#/$defs/EntityId" + }, + "features": { + "description": "Feature values of slot `Speaker`.", + "type": "array", + "items": { + "$ref": "#/$defs/FeatureValue" + } + }, + "inventory_rev": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "n": { + "description": "The revision, from 1.", + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "entity", + "n", + "inventory_rev", + "features", + "annotator" + ], + "$defs": { + "Annotator": { + "description": "Who annotated a document. A model's generator is an attestation string, held\nto the same form as every provenance string.", + "oneOf": [ + { + "type": "string", + "enum": [ + "Operator" + ] + }, + { + "type": "object", + "properties": { + "Model": { + "type": "object", + "properties": { + "family": { + "$ref": "#/$defs/ModelFamily" + }, + "generator": { + "type": "string" + } + }, + "required": [ + "family", + "generator" + ] + } + }, + "additionalProperties": false, + "required": [ + "Model" + ] + } + ] + }, + "EntityId": { + "type": "string", + "pattern": "^entity:[a-z0-9]([a-z0-9-]{0,46}[a-z0-9])?$" + }, + "FeatValue": { + "oneOf": [ + { + "type": "object", + "properties": { + "Grade": { + "$ref": "#/$defs/Grade" + } + }, + "additionalProperties": false, + "required": [ + "Grade" + ] + }, + { + "description": "A label the feature's inventory entry declares.", + "type": "object", + "properties": { + "Option": { + "$ref": "#/$defs/OptionLabel" + } + }, + "additionalProperties": false, + "required": [ + "Option" + ] + }, + { + "type": "object", + "properties": { + "Flag": { + "type": "boolean" + } + }, + "additionalProperties": false, + "required": [ + "Flag" + ] + } + ] + }, + "FeatureValue": { + "type": "object", + "properties": { + "feature": { + "description": "Feature id.", + "$ref": "#/$defs/InvId" + }, + "value": { + "$ref": "#/$defs/FeatValue" + } + }, + "required": [ + "feature", + "value" + ] + }, + "Grade": { + "type": "integer", + "maximum": 1000, + "minimum": 0 + }, + "InvId": { + "type": "string", + "pattern": "^(concept|frame|role|relation|speech_act|feature|referent_kind|extension):[1-9][0-9]*$" + }, + "ModelFamily": { + "description": "The family of the model that wrote a line.", + "oneOf": [ + { + "type": "string", + "enum": [ + "Anthropic", + "OpenAi", + "Google", + "Alibaba", + "Meta", + "Mistral" + ] + }, + { + "description": "Nothing records which family wrote the line.", + "type": "string", + "const": "Unrecorded" + } + ] + }, + "OptionLabel": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]{0,23}$" + } + } +} From 469e46cbc57b20fefabea1715cc7f369b14213fe Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 02:42:02 +0000 Subject: [PATCH 029/175] Cut: Checkpoint has no public constructor; instruction text is consts Checkpoint::declare becomes cfg(test) for_test; Checkpoint-using tests move into src unit tests over split::fixture; Prompt::mint and encode lose the instructions parameter (MINT_INSTRUCTIONS, ENCODE_INSTRUCTIONS in context.rs). Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/context.rs | 62 +++++++++++---- crates/weksa/src/score/mod.rs | 137 +++++++++++++++++++++++++++------- crates/weksa/src/split.rs | 92 ++++++++++++++++++----- crates/weksa/tests/context.rs | 40 ++++------ crates/weksa/tests/score.rs | 75 +------------------ 5 files changed, 249 insertions(+), 157 deletions(-) diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index de94479..11b9a51 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -2,15 +2,22 @@ //! model call only as a [`Prompt`], and a `Prompt` is made only by minting from //! [`TrainItems`] or by encoding one item an [`EncoderOffer`] holds: Train-only //! minting and per-item encoding (leak-owned-by-flow-and-scorer-3, -//! loop-context-builder-2). Instruction wording is not written here; a caller -//! passes `&'static str`, so the templates are fixed at compile time and hence -//! within a run. +//! loop-context-builder-2). Instruction text is [`MINT_INSTRUCTIONS`] and +//! [`ENCODE_INSTRUCTIONS`], consts in this module, and nothing else: `mint` and +//! `encode` take none, so no run-time text can become instructions. use crate::corpus::CorpusItem; use crate::packet::EntityId; use crate::split::{Checkpoint, Split, SplitScheme}; use crate::store::{CorpusStore, normalized}; +/// The instructions a mint prompt starts with. golf-loop-en edits the wording +/// here. +pub const MINT_INSTRUCTIONS: &str = "Mint glosses from the lines below."; + +/// The instructions an encode prompt starts with. +pub const ENCODE_INSTRUCTIONS: &str = "Encode the line below."; + /// The live Train items of a corpus, in key order. The only input minting /// reads; it cannot be built from a slice, and no accessor returns text or an /// item. @@ -145,17 +152,17 @@ impl EncoderOffer { /// let dir = tempfile::tempdir().unwrap(); /// let store = weksa::CorpusStore::open(&dir.path().join("c.cc")).unwrap(); /// let t = weksa::context::TrainItems::from_corpus(&store).unwrap(); -/// let p = weksa::context::Prompt::mint("Mint.", &t); -/// assert_eq!(p.text(), "Mint."); +/// let p = weksa::context::Prompt::mint(&t); +/// assert_eq!(p.text(), weksa::context::MINT_INSTRUCTIONS); /// ``` /// -/// Instructions are compile-time text: +/// Instructions are not a parameter: /// /// ```compile_fail /// let dir = tempfile::tempdir().unwrap(); /// let store = weksa::CorpusStore::open(&dir.path().join("c.cc")).unwrap(); /// let t = weksa::context::TrainItems::from_corpus(&store).unwrap(); -/// let p = weksa::context::Prompt::mint(&String::from("x"), &t); +/// let p = weksa::context::Prompt::mint("x", &t); /// ``` /// /// ```compile_fail @@ -163,7 +170,7 @@ impl EncoderOffer { /// let dir = tempfile::tempdir().unwrap(); /// let store = weksa::CorpusStore::open(&dir.path().join("c.cc")).unwrap(); /// let t = weksa::context::TrainItems::from_corpus(&store).unwrap(); -/// let p = weksa::context::Prompt::mint("x", &t); +/// let p = weksa::context::Prompt::mint(&t); /// s(&p); /// ``` pub struct Prompt { @@ -197,9 +204,9 @@ fn block(item: &CorpusItem, prior: Option<&TrainItems>) -> String { const ELIDED: &str = "[elided]"; impl Prompt { - /// Instructions, then every Train item in key order. - pub fn mint(instructions: &'static str, train: &TrainItems) -> Prompt { - let mut parts = vec![instructions.to_string()]; + /// [`MINT_INSTRUCTIONS`], then every Train item in key order. + pub fn mint(train: &TrainItems) -> Prompt { + let mut parts = vec![MINT_INSTRUCTIONS.to_string()]; parts.extend(train.items.iter().map(|item| block(item, None))); Prompt { text: parts.join("\n\n"), @@ -207,13 +214,13 @@ impl Prompt { } } - /// Instructions, the one offered item with its prior turns resolved + /// [`ENCODE_INSTRUCTIONS`], the one offered item with its prior turns resolved /// against `train`, and the entities a packet for it may cite. - pub fn encode(instructions: &'static str, offer: &EncoderOffer, train: &TrainItems) -> Prompt { + pub fn encode(offer: &EncoderOffer, train: &TrainItems) -> Prompt { let ids: Vec<&str> = offer.entities.iter().map(EntityId::as_str).collect(); Prompt { text: format!( - "{instructions}\n\n{}\n\nentities: {}", + "{ENCODE_INSTRUCTIONS}\n\n{}\n\nentities: {}", block(&offer.item, Some(train)), ids.join(", ") ), @@ -230,3 +237,30 @@ impl Prompt { &self.items } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::split::fixture::{group_in, item}; + + #[test] + fn checkpoint_offer_gate() { + let cp = Checkpoint::for_test(1); + let check = |i: &CorpusItem| EncoderOffer::for_checkpoint(i, &cp).err(); + let train = item("item:a:1", "t", &group_in(Split::Train), false); + let validation = item("item:b:1", "v", &group_in(Split::Validation), false); + let test = item("item:c:1", "x", &group_in(Split::Test), false); + let no_split = item("item:c:2", "m", &group_in(Split::Test), true); + let mut withdrawn = item("item:a:2", "w", &group_in(Split::Train), false); + withdrawn.withdrawn = true; + assert_eq!(check(&train), None); + assert_eq!(check(&validation), None); + assert_eq!(check(&test), None); + assert_eq!(check(&no_split), Some(OfferRefusal::NoSplit)); + assert_eq!(check(&withdrawn), Some(OfferRefusal::Withdrawn)); + // Withdrawn wins over NoSplit. + let mut both = no_split.clone(); + both.withdrawn = true; + assert_eq!(check(&both), Some(OfferRefusal::Withdrawn)); + } +} diff --git a/crates/weksa/src/score/mod.rs b/crates/weksa/src/score/mod.rs index a5556de..c531e89 100644 --- a/crates/weksa/src/score/mod.rs +++ b/crates/weksa/src/score/mod.rs @@ -158,37 +158,27 @@ pub fn aggregate(scored: &[(&CorpusItem, ItemScore)]) -> Aggregate { /// The Test split's readings, scalars only: no item key, no minted string, no /// pair. The checkpoint decision (convergence-test) needs numbers; a key beside /// a minted string is the handle an edit step would act on. The only producer -/// is `at_checkpoint`, which consumes a `Checkpoint`. +/// is `at_checkpoint`, which consumes a `Checkpoint`, which only the lib can make. /// /// ``` /// use weksa::score::TestReadings; -/// use weksa::split::Checkpoint; +/// fn f(t: TestReadings) -> usize { t.count() } +/// ``` +/// +/// ```compile_fail +/// use weksa::score::TestReadings; +/// fn f(t: TestReadings) { let _u = t.clone(); } +/// ``` +/// +/// ```compile_fail +/// use weksa::score::TestReadings; +/// fn f(t: TestReadings) { let _ = format!("{t:?}"); } +/// ``` +/// +/// ```compile_fail +/// use weksa::score::TestReadings; /// fn s(_: &T) {} -/// let t = TestReadings::at_checkpoint(Checkpoint::declare(1), &[], &[], &[]); -/// assert_eq!(t.count(), 0); -/// s(&0u8); -/// ``` -/// -/// ```compile_fail -/// use weksa::score::TestReadings; -/// use weksa::split::Checkpoint; -/// let t = TestReadings::at_checkpoint(Checkpoint::declare(1), &[], &[], &[]); -/// let _u = t.clone(); -/// ``` -/// -/// ```compile_fail -/// use weksa::score::TestReadings; -/// use weksa::split::Checkpoint; -/// let t = TestReadings::at_checkpoint(Checkpoint::declare(1), &[], &[], &[]); -/// let _ = format!("{t:?}"); -/// ``` -/// -/// ```compile_fail -/// use weksa::score::TestReadings; -/// use weksa::split::Checkpoint; -/// fn s(_: &T) {} -/// let t = TestReadings::at_checkpoint(Checkpoint::declare(1), &[], &[], &[]); -/// s(&t); +/// fn f(t: TestReadings) { s(&t); } /// ``` pub struct TestReadings { mean: SplitMean, @@ -240,3 +230,96 @@ impl TestReadings { &self.leak.coverage_by_n } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::split::fixture::{group_in, item}; + + const HELD_OUT: &str = "extraordinary lighthouse keepers whispered unforgettable stories throughout midnight thunderstorms before surrendering peacefully"; + const UNRELATED: &str = "harbor ledgers record ordinary tides"; + + fn score(chrf: f64, meaning: Option) -> ItemScore { + ItemScore { chrf, meaning } + } + + fn in_split(key: &str, text: &str, split: Split, model: bool) -> CorpusItem { + item(key, text, &group_in(split), model) + } + + #[test] + fn test_readings_at_checkpoint() { + let train_item = in_split("item:t:1", UNRELATED, Split::Train, false); + let test = in_split("item:c:1", HELD_OUT, Split::Test, false); + let test_model = in_split("item:c:2", "model words here", Split::Test, true); + let mut gone = in_split("item:c:3", "withdrawn words", Split::Test, false); + gone.withdrawn = true; + let val = in_split( + "item:v:1", + "gardening is quiet work", + Split::Validation, + false, + ); + let items = [ + train_item.clone(), + test.clone(), + test_model.clone(), + gone.clone(), + val.clone(), + ]; + let scored = [ + (&train_item, score(10.0, Some(0.5))), + (&val, score(40.0, Some(0.8))), + (&test, score(60.0, Some(0.6))), + (&test_model, score(100.0, Some(1.0))), + (&gone, score(90.0, Some(0.9))), + ]; + let t = TestReadings::at_checkpoint(Checkpoint::for_test(2), &scored, &[HELD_OUT], &items); + assert_eq!( + (t.count(), t.chrf(), t.meaning()), + (1, Some(60.0), Some(0.6)) + ); + assert_eq!( + t.items(), + 1, + "model-written and withdrawn Test items are excluded" + ); + assert!(t.max_chrf().unwrap() > 99.0); + assert_eq!(t.coverage_by_n()[2].share, Some(1.0)); + + // A minted string that is not the Test text reads low. + let t = TestReadings::at_checkpoint(Checkpoint::for_test(2), &scored, &["zzz"], &items); + assert!(t.max_chrf().unwrap() < 20.0); + assert_eq!(t.coverage_by_n()[2].share, Some(0.0)); + + let empty = TestReadings::at_checkpoint(Checkpoint::for_test(2), &[], &[HELD_OUT], &[]); + assert_eq!( + (empty.count(), empty.chrf(), empty.meaning()), + (0, None, None) + ); + assert_eq!((empty.items(), empty.max_chrf()), (0, None)); + } + + #[test] + fn validation_and_test_leak_agree_in_method() { + // The same text and minted strings, once in Validation and once in + // Test, read the same numbers. + let words: Vec<&str> = HELD_OUT.split(' ').collect(); + let chunks: Vec = words.chunks(3).map(|c| c.join(" ")).collect(); + let minted: Vec<&str> = chunks.iter().map(String::as_str).collect(); + let train_item = in_split("item:t:1", UNRELATED, Split::Train, false); + let v = leak_diagnostic( + &minted, + &[ + in_split("item:v:1", HELD_OUT, Split::Validation, false), + train_item.clone(), + ], + ); + let test = in_split("item:c:1", HELD_OUT, Split::Test, false); + let t = + TestReadings::at_checkpoint(Checkpoint::for_test(1), &[], &minted, &[test, train_item]); + assert_eq!(t.items(), v.items); + assert_eq!(t.max_chrf(), v.max_chrf.as_ref().map(|p| p.chrf)); + assert_eq!(t.coverage_by_n(), v.coverage_by_n.as_slice()); + } +} diff --git a/crates/weksa/src/split.rs b/crates/weksa/src/split.rs index 37e4006..3b86957 100644 --- a/crates/weksa/src/split.rs +++ b/crates/weksa/src/split.rs @@ -1,6 +1,7 @@ //! The train/validation/test split is derived from an item's group and nothing //! else. No stored assignment exists, so no writer can reassign one. An item is -//! read in its live split; Test is read only under a Checkpoint. +//! read in its live split; Test is read only under a Checkpoint, and only the +//! lib can make one (convergence-freeze adds the door). use crate::corpus::{CorpusItem, Source}; @@ -63,40 +64,51 @@ impl SplitScheme { } } -/// Permission to read Test once. The loop declares one at a convergence -/// checkpoint (convergence-test); `TestReadings::at_checkpoint` consumes it, so -/// one declaration buys one Test reading, and `EncoderOffer::for_checkpoint` -/// borrows it first. +/// Permission to read Test once. Only the weksa lib can make one: there is no +/// public constructor, so nothing outside it (the binary, a driver crate, a +/// doctest, an integration test) can read Test. The one door is added by +/// convergence-freeze, which builds a `Checkpoint` from the convergence window +/// over the run's persisted round records (convergence-test). +/// `TestReadings::at_checkpoint` consumes it, so one `Checkpoint` buys one Test +/// reading. +/// +/// ``` +/// fn f(c: &weksa::split::Checkpoint) -> u32 { c.round() } +/// ``` +/// +/// No public constructor: +/// +/// ```compile_fail +/// let _ = weksa::split::Checkpoint::declare(1); +/// ``` +/// +/// One reading per checkpoint: /// /// ``` /// use weksa::score::TestReadings; /// use weksa::split::Checkpoint; -/// let c = Checkpoint::declare(3); -/// assert_eq!(c.round(), 3); -/// let _ = TestReadings::at_checkpoint(c, &[], &[], &[]); +/// fn f(c: Checkpoint) { let _ = TestReadings::at_checkpoint(c, &[], &[], &[]); } /// ``` /// -/// One declaration, one reading: -/// /// ```compile_fail /// use weksa::score::TestReadings; /// use weksa::split::Checkpoint; -/// let c = Checkpoint::declare(3); -/// let _ = TestReadings::at_checkpoint(c, &[], &[], &[]); -/// let _ = TestReadings::at_checkpoint(c, &[], &[], &[]); +/// fn f(c: Checkpoint) { +/// let _ = TestReadings::at_checkpoint(c, &[], &[], &[]); +/// let _ = TestReadings::at_checkpoint(c, &[], &[], &[]); +/// } /// ``` /// /// ```compile_fail -/// use weksa::split::Checkpoint; -/// let c = Checkpoint::declare(3); -/// let _again = c.clone(); +/// fn f(c: weksa::split::Checkpoint) { let _ = c.clone(); } /// ``` pub struct Checkpoint { round: u32, } impl Checkpoint { - pub fn declare(round: u32) -> Self { + #[cfg(test)] + pub(crate) fn for_test(round: u32) -> Self { Self { round } } @@ -104,3 +116,49 @@ impl Checkpoint { self.round } } + +/// Item builders for the unit tests that live in `src`: the twins of +/// `tests/common`'s `group_in_bucket`, `item_in`, `cast` and `model`. +#[cfg(test)] +pub(crate) mod fixture { + use super::{Split, SplitScheme}; + use crate::corpus::{CorpusItem, ModelFamily, Source}; + use crate::packet::EntityId; + + /// The first group `g` whose V1 bucket is in `split`. + pub(crate) fn group_in(split: Split) -> String { + (0..100_000) + .map(|i| format!("g{i}")) + .find(|g| SplitScheme::V1.of_bucket(SplitScheme::V1.bucket(g)) == split) + .unwrap() + } + + pub(crate) fn item(key: &str, text: &str, group: &str, model_written: bool) -> CorpusItem { + let source = if model_written { + Source::ModelWritten { + family: ModelFamily::Unrecorded, + generator: None, + prompt_digest: None, + } + } else { + Source::CastAuthored { + persona: "nibu".into(), + fixture: "f.yaml".into(), + author: "Ada Writer".into(), + } + }; + CorpusItem { + id: key.into(), + text: text.into(), + language: "en-US".into(), + speaker: EntityId::parse("entity:nibu").unwrap(), + names: vec![], + group: group.into(), + prior_turns: vec![], + source, + admitted_on: "2026-10-29".into(), + supersedes: None, + withdrawn: false, + } + } +} diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs index a27b31d..c801d7a 100644 --- a/crates/weksa/tests/context.rs +++ b/crates/weksa/tests/context.rs @@ -1,16 +1,15 @@ mod common; use common::{cast, ent, entities_in, group_in_bucket, item_in, model, world}; -use weksa::context::{EncoderOffer, OfferRefusal, Prompt, TrainItems}; -use weksa::split::Checkpoint; +use weksa::context::{ + ENCODE_INSTRUCTIONS as ENCODE, EncoderOffer, MINT_INSTRUCTIONS as MINT, OfferRefusal, Prompt, + TrainItems, +}; use weksa::{ CorpusItem, CorpusItemId, CorpusStore, NameSpan, PacketId, PacketStore, StoreError, StoreRefusal, Turn, }; -const MINT: &str = "Mint glosses from the lines below."; -const ENCODE: &str = "Encode the line below."; - struct Corpus { _dir: tempfile::TempDir, store: CorpusStore, @@ -109,15 +108,15 @@ fn train_items_are_in_key_order() { let none = CorpusStore::open(&dir.path().join("c.cc")).unwrap(); let none = TrainItems::from_corpus(&none).unwrap(); assert_eq!((none.len(), none.is_empty()), (0, true)); - assert_eq!(Prompt::mint(MINT, &none).text(), MINT); - assert!(Prompt::mint(MINT, &none).item_keys().is_empty()); + assert_eq!(Prompt::mint(&none).text(), MINT); + assert!(Prompt::mint(&none).item_keys().is_empty()); } #[test] fn mint_prompt_holds_train_text_only() { let c = corpus(); let t = TrainItems::from_corpus(&c.store).unwrap(); - let p = Prompt::mint(MINT, &t); + let p = Prompt::mint(&t); assert!(p.text().starts_with(MINT)); assert!(p.text().contains("alphamarker")); for other in ["betamarker", "gammamarker", "deltamarker", "epsilonmarker"] { @@ -145,25 +144,24 @@ fn prompts_render_deterministically() { .unwrap(); let t = TrainItems::from_corpus(&store).unwrap(); assert_eq!( - Prompt::mint(MINT, &t).text(), + Prompt::mint(&t).text(), format!( "{MINT}\n\n[item:a:1] entity:nibu: Hello there.\n\n[item:a:2] entity:nibu: Second line." ) ); - assert_eq!(Prompt::mint(MINT, &t).item_keys(), ["item:a:1", "item:a:2"]); + assert_eq!(Prompt::mint(&t).item_keys(), ["item:a:1", "item:a:2"]); let offer = EncoderOffer::for_round(&store.get("item:a:2").unwrap().unwrap()).unwrap(); assert_eq!( - Prompt::encode(ENCODE, &offer, &t).text(), + Prompt::encode(&offer, &t).text(), format!("{ENCODE}\n\n[item:a:2] entity:nibu: Second line.\n\nentities: entity:nibu") ); - assert_eq!(Prompt::encode(ENCODE, &offer, &t).item_keys(), ["item:a:2"]); + assert_eq!(Prompt::encode(&offer, &t).item_keys(), ["item:a:2"]); } #[test] fn encoder_offer_split_gate() { let c = corpus(); - let cp = Checkpoint::declare(1); let train = get(&c, "item:a:1"); let validation = get(&c, "item:b:1"); let test = get(&c, "item:c:1"); @@ -171,7 +169,6 @@ fn encoder_offer_split_gate() { let withdrawn = get(&c, "item:a:2"); assert!(withdrawn.withdrawn); let round = |i: &CorpusItem| EncoderOffer::for_round(i).err(); - let check = |i: &CorpusItem| EncoderOffer::for_checkpoint(i, &cp).err(); assert_eq!(round(&train), None); assert_eq!(round(&validation), None); @@ -179,17 +176,10 @@ fn encoder_offer_split_gate() { assert_eq!(round(&no_split), Some(OfferRefusal::NoSplit)); assert_eq!(round(&withdrawn), Some(OfferRefusal::Withdrawn)); - assert_eq!(check(&train), None); - assert_eq!(check(&validation), None); - assert_eq!(check(&test), None); - assert_eq!(check(&no_split), Some(OfferRefusal::NoSplit)); - assert_eq!(check(&withdrawn), Some(OfferRefusal::Withdrawn)); - // Withdrawn wins over NoSplit. let mut both = no_split.clone(); both.withdrawn = true; assert_eq!(round(&both), Some(OfferRefusal::Withdrawn)); - assert_eq!(check(&both), Some(OfferRefusal::Withdrawn)); // A withdrawn Test item is Withdrawn, not TestBeforeCheckpoint. let mut test_gone = test.clone(); test_gone.withdrawn = true; @@ -256,7 +246,7 @@ fn encoder_offer_entities_are_the_items_own() { "a sibling's entity is not the item's" ); assert_eq!(offer.key(), "item:a:1"); - let p = Prompt::encode(ENCODE, &offer, &t); + let p = Prompt::encode(&offer, &t); assert!( p.text() .ends_with("entities: entity:brokkr, entity:hero, entity:kay") @@ -359,7 +349,7 @@ fn prior_turns_print_only_text_a_live_train_item_holds() { store.withdraw("item:a:1").unwrap(); let t = TrainItems::from_corpus(&store).unwrap(); - let mint = Prompt::mint(MINT, &t); + let mint = Prompt::mint(&t); for secret in ["withdrawnsecret", "testsecret", "validation reply"] { assert!(!mint.text().contains(secret), "mint printed {secret}"); } @@ -370,7 +360,7 @@ fn prior_turns_print_only_text_a_live_train_item_holds() { let encode = |key: &str| { let item = store.get(key).unwrap().unwrap(); - Prompt::encode(ENCODE, &EncoderOffer::for_round(&item).unwrap(), &t) + Prompt::encode(&EncoderOffer::for_round(&item).unwrap(), &t) .text() .to_string() }; @@ -407,6 +397,6 @@ fn prior_turn_match_uses_the_stores_text_key() { }]; store.add(reply.clone(), &entities).unwrap(); let t = TrainItems::from_corpus(&store).unwrap(); - let p = Prompt::encode(ENCODE, &EncoderOffer::for_round(&reply).unwrap(), &t); + let p = Prompt::encode(&EncoderOffer::for_round(&reply).unwrap(), &t); assert!(p.text().contains("entity:brokkr: context line")); } diff --git a/crates/weksa/tests/score.rs b/crates/weksa/tests/score.rs index cdf2586..ae20bce 100644 --- a/crates/weksa/tests/score.rs +++ b/crates/weksa/tests/score.rs @@ -10,10 +10,9 @@ use common::{cast, group_in_bucket, item_in, model}; use serde_json::Value; use weksa::CorpusItem; use weksa::score::{ - Embedder, ItemScore, OllamaEmbedder, ScoreInput, Scorer, TestReadings, aggregate, chrf, cosine, + Embedder, ItemScore, OllamaEmbedder, ScoreInput, Scorer, aggregate, chrf, cosine, leak_diagnostic, }; -use weksa::split::Checkpoint; /// (reference, hypothesis, sacrebleu 2.4.3 sentence_chrf), map 491536c. const VECTORS: [(&str, &str, f64); 6] = [ @@ -877,78 +876,6 @@ fn leak_diagnostic_reads_validation_only() { assert!(on_validation.max_chrf.unwrap().chrf > 99.0); } -#[test] -fn test_readings_at_checkpoint() { - let train_item = train("item:t:1", UNRELATED); - let test = item_in("item:c:1", HELD_OUT, &group_in_bucket(90), cast()); - let test_model = item_in( - "item:c:2", - "model words here", - &group_in_bucket(90), - model(), - ); - let mut gone = item_in("item:c:3", "withdrawn words", &group_in_bucket(90), cast()); - gone.withdrawn = true; - let val = validation("item:v:1", "gardening is quiet work"); - let items = [ - train_item.clone(), - test.clone(), - test_model.clone(), - gone.clone(), - val.clone(), - ]; - let scored = [ - (&train_item, score(10.0, Some(0.5))), - (&val, score(40.0, Some(0.8))), - (&test, score(60.0, Some(0.6))), - (&test_model, score(100.0, Some(1.0))), - (&gone, score(90.0, Some(0.9))), - ]; - let t = TestReadings::at_checkpoint(Checkpoint::declare(2), &scored, &[HELD_OUT], &items); - assert_eq!( - (t.count(), t.chrf(), t.meaning()), - (1, Some(60.0), Some(0.6)) - ); - assert_eq!( - t.items(), - 1, - "model-written and withdrawn Test items are excluded" - ); - assert!(t.max_chrf().unwrap() > 99.0); - assert_eq!(t.coverage_by_n()[2].share, Some(1.0)); - - // A minted string that is not the Test text reads low; the same method as - // the Validation reading, so the same minted text reads alike in each. - let t = TestReadings::at_checkpoint(Checkpoint::declare(2), &scored, &["zzz"], &items); - assert!(t.max_chrf().unwrap() < 20.0); - assert_eq!(t.coverage_by_n()[2].share, Some(0.0)); - - let empty = TestReadings::at_checkpoint(Checkpoint::declare(2), &[], &[HELD_OUT], &[]); - assert_eq!( - (empty.count(), empty.chrf(), empty.meaning()), - (0, None, None) - ); - assert_eq!((empty.items(), empty.max_chrf()), (0, None)); -} - -#[test] -fn validation_and_test_leak_agree_in_method() { - // The same text and minted strings, once in Validation and once in Test, - // read the same numbers. - let minted = chunks(3, " "); - let minted = refs(&minted); - let train_item = train("item:t:1", UNRELATED); - let v = leak_diagnostic( - &minted, - &[validation("item:v:1", HELD_OUT), train_item.clone()], - ); - let test = item_in("item:c:1", HELD_OUT, &group_in_bucket(90), cast()); - let t = TestReadings::at_checkpoint(Checkpoint::declare(1), &[], &minted, &[test, train_item]); - assert_eq!(t.items(), v.items); - assert_eq!(t.max_chrf(), v.max_chrf.as_ref().map(|p| p.chrf)); - assert_eq!(t.coverage_by_n(), v.coverage_by_n.as_slice()); -} - #[test] fn held_out_train_union_excludes_withdrawn_train_items() { let mut gone = train("item:t:1", HELD_OUT); From d5b11088ad8bf42f86192b2299d344e0c99e4558 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 02:43:31 +0000 Subject: [PATCH 030/175] Checkpoint offers each item once; prior turns print the held Train text for_checkpoint takes &mut Checkpoint and refuses a repeated key (OfferedAtCheckpoint) after the gate; TrainItems::held replaces holds and block prints the matched Train item's text; the text-key fold is pinned against the store's own S6 refusal. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/context.rs | 72 +++++++++++++++++++++-------- crates/weksa/src/split.rs | 20 +++++++- crates/weksa/tests/context.rs | 86 +++++++++++++++++++++++++++++++++-- 3 files changed, 153 insertions(+), 25 deletions(-) diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index 11b9a51..8914724 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -69,13 +69,14 @@ impl TrainItems { self.items.is_empty() } - /// Whether a live Train item of `group` holds `text`, compared by the - /// store's own text key. - fn holds(&self, group: &str, text: &str) -> bool { + /// The text of the live Train item of `group` whose store text key equals + /// `text`'s (store S6 makes it unique). + fn held(&self, group: &str, text: &str) -> Option<&str> { let key = normalized(text); self.items .iter() - .any(|item| item.group == group && normalized(&item.text) == key) + .find(|item| item.group == group && normalized(&item.text) == key) + .map(|item| item.text.as_str()) } pub fn keys(&self) -> impl Iterator { @@ -91,6 +92,8 @@ pub enum OfferRefusal { NoSplit, /// A Test item is offered only under a [`Checkpoint`]. TestBeforeCheckpoint, + /// An item is offered at most once under one [`Checkpoint`]. + OfferedAtCheckpoint, } /// One item the encoder may be shown, with the entities a packet for it may @@ -109,12 +112,17 @@ impl EncoderOffer { } } - /// A convergence checkpoint offers any live item, Test included. + /// A convergence checkpoint offers any live item, Test included, each item + /// at most once under one `Checkpoint`. The gate runs first, so a refused + /// item records no key. pub fn for_checkpoint( item: &CorpusItem, - _checkpoint: &Checkpoint, + checkpoint: &mut Checkpoint, ) -> Result { Self::gate(item)?; + if !checkpoint.first_offer(&item.id) { + return Err(OfferRefusal::OfferedAtCheckpoint); + } Ok(Self::of(item)) } @@ -180,20 +188,19 @@ pub struct Prompt { /// What a prompt prints of an item: its line, and its prior turns only as the /// one owner allows. A prior turn is a copy of text, not a reference, so it can -/// outlive a withdrawal or quote a held-out line; its text is printed only when -/// a live Train item of the item's group holds it, and the speaker alone is -/// printed otherwise. Mint prints no prior turns (`prior: None`). +/// outlive a withdrawal or quote a held-out line; what is printed is the text of +/// the live Train item of the item's group that holds it (never the turn's own +/// bytes), and the speaker alone otherwise. Mint prints no prior turns (`prior: None`). fn block(item: &CorpusItem, prior: Option<&TrainItems>) -> String { let mut lines: Vec = prior .into_iter() .flat_map(|train| { - item.prior_turns.iter().map(move |t| { - if train.holds(&item.group, &t.text) { - format!("{}: {}", t.speaker, t.text) - } else { - format!("{}: {ELIDED}", t.speaker) - } - }) + item.prior_turns + .iter() + .map(move |t| match train.held(&item.group, &t.text) { + Some(text) => format!("{}: {text}", t.speaker), + None => format!("{}: {ELIDED}", t.speaker), + }) }) .collect(); lines.push(format!("[{}] {}: {}", item.id, item.speaker, item.text)); @@ -245,8 +252,9 @@ mod tests { #[test] fn checkpoint_offer_gate() { - let cp = Checkpoint::for_test(1); - let check = |i: &CorpusItem| EncoderOffer::for_checkpoint(i, &cp).err(); + // A fresh checkpoint per call: this test is the gate, not the once rule. + let check = + |i: &CorpusItem| EncoderOffer::for_checkpoint(i, &mut Checkpoint::for_test(1)).err(); let train = item("item:a:1", "t", &group_in(Split::Train), false); let validation = item("item:b:1", "v", &group_in(Split::Validation), false); let test = item("item:c:1", "x", &group_in(Split::Test), false); @@ -263,4 +271,32 @@ mod tests { both.withdrawn = true; assert_eq!(check(&both), Some(OfferRefusal::Withdrawn)); } + + #[test] + fn checkpoint_offers_each_item_once() { + let mut cp = Checkpoint::for_test(1); + let offer = |cp: &mut Checkpoint, i: &CorpusItem| EncoderOffer::for_checkpoint(i, cp).err(); + let a = item("item:c:1", "a", &group_in(Split::Test), false); + let b = item("item:c:3", "b", &group_in(Split::Test), false); + let v = item("item:b:1", "v", &group_in(Split::Validation), false); + assert_eq!(offer(&mut cp, &a), None); + assert_eq!(offer(&mut cp, &a), Some(OfferRefusal::OfferedAtCheckpoint)); + assert_eq!(offer(&mut cp, &b), None); + assert_eq!(offer(&mut cp, &v), None); + assert_eq!(offer(&mut cp, &v), Some(OfferRefusal::OfferedAtCheckpoint)); + + // A refused item records no key: its refusal stays the gate's. + let mut gone = item("item:c:2", "w", &group_in(Split::Test), false); + gone.withdrawn = true; + assert_eq!(offer(&mut cp, &gone), Some(OfferRefusal::Withdrawn)); + assert_eq!(offer(&mut cp, &gone), Some(OfferRefusal::Withdrawn)); + let mut live = gone.clone(); + live.withdrawn = false; + assert_eq!(offer(&mut cp, &live), None, "the refusal recorded no key"); + + // The checkpoint is spent by its reading; a new one offers afresh. + let _ = crate::score::TestReadings::at_checkpoint(cp, &[], &[], &[]); + let mut fresh = Checkpoint::for_test(2); + assert_eq!(offer(&mut fresh, &a), None); + } } diff --git a/crates/weksa/src/split.rs b/crates/weksa/src/split.rs index 3b86957..5d3a5ee 100644 --- a/crates/weksa/src/split.rs +++ b/crates/weksa/src/split.rs @@ -3,6 +3,8 @@ //! read in its live split; Test is read only under a Checkpoint, and only the //! lib can make one (convergence-freeze adds the door). +use std::collections::BTreeSet; + use crate::corpus::{CorpusItem, Source}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -70,7 +72,8 @@ impl SplitScheme { /// convergence-freeze, which builds a `Checkpoint` from the convergence window /// over the run's persisted round records (convergence-test). /// `TestReadings::at_checkpoint` consumes it, so one `Checkpoint` buys one Test -/// reading. +/// reading. One `Checkpoint` also offers each item at most once +/// (`EncoderOffer::for_checkpoint`). /// /// ``` /// fn f(c: &weksa::split::Checkpoint) -> u32 { c.round() } @@ -82,6 +85,10 @@ impl SplitScheme { /// let _ = weksa::split::Checkpoint::declare(1); /// ``` /// +/// ```compile_fail +/// let _ = weksa::split::Checkpoint { round: 1, offered: Default::default() }; +/// ``` +/// /// One reading per checkpoint: /// /// ``` @@ -104,12 +111,21 @@ impl SplitScheme { /// ``` pub struct Checkpoint { round: u32, + offered: BTreeSet, } impl Checkpoint { #[cfg(test)] pub(crate) fn for_test(round: u32) -> Self { - Self { round } + Self { + round, + offered: BTreeSet::new(), + } + } + + /// Records `key` as offered; false when it already was. + pub(crate) fn first_offer(&mut self, key: &str) -> bool { + self.offered.insert(key.to_string()) } pub fn round(&self) -> u32 { diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs index c801d7a..8fcad16 100644 --- a/crates/weksa/tests/context.rs +++ b/crates/weksa/tests/context.rs @@ -381,22 +381,98 @@ fn prior_turns_print_only_text_a_live_train_item_holds() { ); } -#[test] -fn prior_turn_match_uses_the_stores_text_key() { +/// A scratch Train store holding `train_text`, and an encode prompt for a +/// sibling whose one prior turn is `prior_text`. +fn encode_with_prior(train_text: &str, prior_text: &str) -> (String, Result<(), StoreError>) { let dir = tempfile::tempdir().unwrap(); let entities = entities_in(dir.path()); let mut store = CorpusStore::open(&dir.path().join("c.cc")).unwrap(); let g = group_in_bucket(10); store - .add(item_in("item:a:1", "Context Line", &g, cast()), &entities) + .add(item_in("item:a:1", train_text, &g, cast()), &entities) .unwrap(); let mut reply = item_in("item:a:2", "reply", &g, cast()); reply.prior_turns = vec![Turn { speaker: common::eid("brokkr"), - text: "context line".into(), + text: prior_text.into(), }]; store.add(reply.clone(), &entities).unwrap(); let t = TrainItems::from_corpus(&store).unwrap(); let p = Prompt::encode(&EncoderOffer::for_round(&reply).unwrap(), &t); - assert!(p.text().contains("entity:brokkr: context line")); + let text = p.text().to_string(); + // The store's own rule, as the oracle for key equality; after the prompt is + // built, because an admitted probe would itself become a held line. + let probe = store.add(item_in("item:a:9", prior_text, &g, cast()), &entities); + (text, probe.map(|_| ())) } + +#[test] +fn prior_turn_prints_the_held_train_text() { + let (text, _) = encode_with_prior("Context Line", "context-line!! :)"); + assert!( + text.contains("entity:brokkr: Context Line\n[item:a:2]"), + "{text}" + ); + for turn_bytes in ["!!", ":)", "context-line"] { + assert!(!text.contains(turn_bytes), "{turn_bytes} in {text}"); + } +} + +#[test] +fn prior_turn_match_uses_the_stores_text_key() { + let (text, _) = encode_with_prior("Context Line", "context line"); + assert!(text.contains("entity:brokkr: Context Line\n")); + assert!(!text.contains("context line")); +} + +#[test] +fn prior_turn_fold_matches_the_store() { + let rows = [ + ("case", "Context Line", "context line"), + ( + "punctuation and spacing", + "Context Line", + "context, line!", + ), + ( + "NFKC fullwidth letters", + "Context", + "\u{ff23}\u{ff4f}\u{ff4e}\u{ff54}\u{ff45}\u{ff58}\u{ff54}", + ), + ("NFKC fullwidth digits", "room 12", "room \u{ff11}\u{ff12}"), + ("dotless i", "Ilik", "\u{131}l\u{131}k"), + ("dotted capital I", "Istanbul", "\u{130}stanbul"), + ( + "katakana vs hiragana", + "\u{3072}\u{3089}\u{304c}\u{306a}", + "\u{30d2}\u{30e9}\u{30ac}\u{30ca}", + ), + ("voiced katakana wa", "\u{308f}\u{3099}", "\u{30f7}"), + ("Arabic-Indic digits", "room 12", "room \u{661}\u{662}"), + ]; + for (fold, train_text, prior_text) in rows { + let (text, probe) = encode_with_prior(train_text, prior_text); + assert!( + matches!( + probe, + Err(StoreError::Refused(StoreRefusal::S6TextHeld { .. })) + ), + "{fold}: the store does not hold these as one text: {probe:?}" + ); + assert!( + text.contains(&format!("entity:brokkr: {train_text}\n[item:a:2]")), + "{fold}: {text}" + ); + assert!(!text.contains(ELIDED_MARK), "{fold}: {text}"); + } + // A text differing by a letter is a different key: the store admits it and + // the prior turn is elided. + let (text, probe) = encode_with_prior("Context Line", "Context Lane"); + assert!(probe.is_ok(), "{probe:?}"); + assert!( + text.contains("entity:brokkr: [elided]\n[item:a:2]"), + "{text}" + ); +} + +const ELIDED_MARK: &str = "[elided]"; From 3d83ba5fd2f5253b97b79d051f018f3d27327323 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 02:43:36 +0000 Subject: [PATCH 031/175] Map: checkpoint-gate lines (7a, 7, 11) Co-Authored-By: Claude Sonnet 5.5 --- docs/targets/interlingua-lowering-map.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index d2917de..2f4467a 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -402,11 +402,11 @@ One line per cut; specs exist for the cuts about to run (2d and 4, siblings on ` 4b. `speaker-scene` (spec r1; `speaker-scene-seeds` is its cut 4c: CLI, inventory revision 3, cast speakers and scenes; `Party::Unnamed` covers an addressee with no entity; Speaker and Scene share one `SpeakerStateStore`): slots `Speaker` and `Scene` with their seed features; `weksa.speaker` (`speaker:@`) and `weksa.scene` (`scene:@`, participants and ordered-pair relations) documents with `Annotator` `Operator` and `Model { family, generator }`, validated by the packet's shared feature check, append-only per key; their stores, `weksa speaker` and `weksa scene` commands, published schemas; Model-annotated cast speakers and scenes for the committed items. Waits on 2d and 4; `pd-speaker-annotation` blocks only the public-domain annotator. 5. `language-ontology`: `weksa.community` from `data/cultural-ontology/` (owns follow-up `edda-referent-yaml-parse`), `weksa.ontology_entry` for en-US and ja-JP from `data/target-language-ontology/` (YAML deleted), speaker communities, deterministic `select`, the lowering view (packet without trace, speaker, scene, selected entries). Waits on 4. 6. `pd-importer`: public-domain play and fiction dialogue into the corpus with speaker, scene group, edition provenance, and Speaker and Scene annotations by the ruled annotator. Waits on 4, follow-up `gutenberg-terms` and question `pd-speaker-annotation`. -7a. `loop-context-builder`: `context.rs` (`TrainItems`, `EncoderOffer`, `Prompt`: corpus text reaches a model call only through `Prompt::mint` over live Train items or `Prompt::encode` over one offered item; instructions are `&'static str`), `SplitScheme::live`, `CorpusItem::cited_entities` (one owner of the id set `PacketStore::admit` and the offer share), and checkpoint-only Test readings (`Checkpoint`, `TestReadings`; `Aggregate` and `leak_diagnostic` carry no Test). Library only. Merges follow-ups `loop-context-builder-2` and `scorer-test-readings-type`. (spec r1) -7. `golf-loop-en`: the loop driver, the packet store bound to one inventory store (`packet-store-inventory-binding`), `ModelPort` with the `claude -p` adapter (it takes only `&Prompt`), encoder (packet with stance) and lowerer prompts, round records, Score documents, the composite and accept rule, `weksa score`, the detached unit on Yggdrasil. Waits on 3, 5, 6 and follow-up `claude-on-yggdrasil`. +7a. `loop-context-builder`: `context.rs` (`TrainItems`, `EncoderOffer`, `Prompt`: corpus text reaches a model call only through `Prompt::mint` over live Train items or `Prompt::encode` over one offered item; instructions are the consts `MINT_INSTRUCTIONS` and `ENCODE_INSTRUCTIONS` in `context.rs`; `Checkpoint` has no public constructor and offers each item once; prior turns print the held Train text), `SplitScheme::live`, `CorpusItem::cited_entities` (one owner of the id set `PacketStore::admit` and the offer share), and checkpoint-only Test readings (`Checkpoint`, `TestReadings`; `Aggregate` and `leak_diagnostic` carry no Test). Library only. Merges follow-ups `loop-context-builder-2` and `scorer-test-readings-type`. (spec r1) +7. `golf-loop-en`: the loop driver, the template wording in `context.rs` with its digest pinned per run (follow-up `template-digest-in-round-record`), the packet store bound to one inventory store (`packet-store-inventory-binding`), `ModelPort` with the `claude -p` adapter (it takes only `&Prompt`), encoder (packet with stance) and lowerer prompts, round records, Score documents, the composite and accept rule, `weksa score`, the detached unit on Yggdrasil. Waits on 3, 5, 6 and follow-up `claude-on-yggdrasil`. 8. `style-axis`: a style-embedding score behind the scorer port. Waits on 3. 9. `ja-panel`: ja-JP into the loop's panel. Waits on 7. 10. `conlang-lexicon`: the first conlang's lexicon placed at concept embeddings, its morphology data, its ontology entries and the gloss reader. Waits on 3, 5 and question `first-conlang`. -11. `convergence-freeze`: the convergence test, the freeze, the re-encode into training pairs. Waits on 7, 9 and 10. +11. `convergence-freeze`: the convergence test, `Checkpoint`'s one door (built from the convergence window over the persisted round records), the freeze, the re-encode into training pairs. Waits on 7, 9 and 10. 12. `distil`: fine-tune a lowerer on the frozen training set. Waits on 11 and a host question asked then. 13. `live-lowering`: agents emit packets with stance; Weksa reads speaker state from persona state (follow-up `persona-state-speech-community`) and lowers for the reader. Waits on 11. From 71e6058b640b049eed15d1fafd80d1dc2551d649 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 02:51:35 +0000 Subject: [PATCH 032/175] with_scene reads the item's speaker and group scene from the store; V17 addressee names a referent The view keeps its corpus item's speaker and group (LoweringInput::of takes the item; admissible is gone), and with_scene takes the SpeakerStateStore, so neither speaker nor scene comes from the caller and only admitted scenes reach a view. An addressee that names a predication is refused as V17. Pins scene numbering per group and the no-fallback case for a named addressee outside the participants; tests/scene.rs fixture now speaks as the item's speaker. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/store.rs | 5 +- crates/weksa/src/validate.rs | 126 +++---- crates/weksa/tests/display.rs | 2 + crates/weksa/tests/scene.rs | 404 ++++++++++++----------- crates/weksa/tests/speaker_state.rs | 27 +- crates/weksa/tests/validate.rs | 23 ++ docs/interlingua-standard.md | 15 +- docs/targets/interlingua-lowering-map.md | 4 +- 8 files changed, 351 insertions(+), 255 deletions(-) diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index 0263712..67f8f1a 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -21,7 +21,7 @@ use crate::inventory::{ }; use crate::packet::{EntityId, Packet}; use crate::speaker::{Annotator, MAX_PARTICIPANTS, Party, Scene, Speaker, scene_key, speaker_key}; -use crate::validate::{LoweringInput, Refusal, admissible, feature_refusals}; +use crate::validate::{LoweringInput, Refusal, feature_refusals, validate}; /// A broken store rule, named. #[derive(Clone, Debug, PartialEq, Eq)] @@ -959,7 +959,7 @@ impl PacketStore { corpus: &CorpusStore, entities: &EntityStore, ) -> Result { - let view = admissible(&packet, &inventory.inventory()?).map_err(StoreError::Invalid)?; + validate(&packet, &inventory.inventory()?).map_err(StoreError::Invalid)?; let item = corpus.resolve(&packet.provenance.corpus_item)?; if item.withdrawn { return Err(StoreRefusal::WithdrawnCorpusItem { @@ -983,6 +983,7 @@ impl PacketStore { .into()); } } + let view = LoweringInput::of(&packet, &item); let key = packet.id.to_string(); if let Some(existing) = self.get(&key)? { if existing == packet { diff --git a/crates/weksa/src/validate.rs b/crates/weksa/src/validate.rs index 9641062..06600e7 100644 --- a/crates/weksa/src/validate.rs +++ b/crates/weksa/src/validate.rs @@ -1,13 +1,14 @@ //! The validator owns the inventory and form rules a packet must meet. Every //! path that admits a packet applies them, but `validate` only judges: it //! returns `Ok(())` or the refusals. [`LoweringInput`], the only thing a -//! lowerer sees, comes from `PacketStore::admit` alone, which builds it through -//! the crate-private `admissible`. The stance is checked like every feature -//! list, in slot `Stance`. The rules are V1..V8, V10..V12 and V14..V16: the +//! lowerer sees, comes from `PacketStore::admit` alone, which builds it from +//! the admitted packet and its corpus item through the crate-private +//! `LoweringInput::of`. The stance is checked like every feature +//! list, in slot `Stance`. The rules are V1..V8, V10..V12 and V14..V17: the //! shape of an id or a grade is its type's parse, not a rule, so V9 is retired -//! and not renumbered; V13 is retired too. [`LoweringInput::with_scene`] takes -//! a [`Scene`](crate::speaker::Scene) document, and `feature_refusals` is the -//! one feature check it and the speaker-state store share. +//! and not renumbered; V13 is retired too. [`LoweringInput::with_scene`] reads the +//! scene of the view's own item group from the speaker-state store, and +//! `feature_refusals` is the one feature check it and that store share. //! //! `validate` owns form and capacity and makes no leak decision. A packet's one //! free text is its trace (`notes` and `rejected_readings`): `validate` does not @@ -27,12 +28,13 @@ use std::collections::BTreeSet; use std::fmt; +use crate::corpus::CorpusItem; use crate::inventory::{Family, InvId, Inventory, InventoryEntry, Slot, ValueKind}; use crate::packet::{ Constraints, Context, Discourse, EntityId, Extension, FeatValue, FeatureValue, NodeId, Packet, PacketKind, Predication, ProvenanceSource, Referent, Relation, SCHEMA_VERSION, Stance, }; -use crate::speaker::Scene; +use crate::store::{SpeakerStateStore, StoreError}; /// A broken rule, named. #[derive(Clone, Debug, PartialEq, Eq)] @@ -110,8 +112,13 @@ pub enum Refusal { V15ConsentSource { source: ProvenanceSource, }, - /// V16: `with_scene` was given a speaker the scene does not list. + /// V16: `with_scene` found the item's speaker absent from its group's scene. V16SpeakerNotInScene, + /// V17: an addressee that names a predication. An act is not someone + /// spoken to, so the addressee names a referent. + V17AddresseeNotAReferent { + id: NodeId, + }, } impl Refusal { @@ -135,6 +142,7 @@ impl Refusal { } Refusal::V15ConsentSource { .. } => "V15", Refusal::V16SpeakerNotInScene => "V16", + Refusal::V17AddresseeNotAReferent { .. } => "V17", } } } @@ -161,6 +169,7 @@ impl fmt::Display for Refusal { Refusal::V14ConsentNotAPredication { .. } => "consent names no predication", Refusal::V15ConsentSource { .. } => "consent set by a source that only read wording", Refusal::V16SpeakerNotInScene => "the speaker is not a participant of the scene", + Refusal::V17AddresseeNotAReferent { .. } => "the addressee names no referent", }; write!(f, "{}: {what}", self.rule()) } @@ -180,28 +189,6 @@ impl fmt::Display for Refusal { /// fn lower(p: &Packet, i: &Inventory) -> Result> { weksa::validate(p, i) } /// ``` pub fn validate(packet: &Packet, inventory: &Inventory) -> Result<(), Vec> { - admissible(packet, inventory).map(|_| ()) -} - -/// The one implementation of the rules. Crate-private: `PacketStore::admit` is -/// the only caller that keeps the view. -/// -/// The first block is the same call through the public name, so the second -/// fails for the private path and nothing else. -/// -/// ``` -/// use weksa::{Inventory, packet::Packet}; -/// fn judge(p: &Packet, i: &Inventory) { let _ = weksa::validate::validate(p, i); } -/// ``` -/// -/// ```compile_fail -/// use weksa::{Inventory, packet::Packet}; -/// fn lower(p: &Packet, i: &Inventory) { let _ = weksa::validate::admissible(p, i); } -/// ``` -pub(crate) fn admissible( - packet: &Packet, - inventory: &Inventory, -) -> Result> { let mut out = Vec::new(); if packet.schema_version != SCHEMA_VERSION { out.push(Refusal::V1SchemaVersion { @@ -221,6 +208,7 @@ pub(crate) fn admissible( } let nodes = check_nodes(packet, &mut out); check_refs(packet, &nodes, &mut out); + check_addressee(packet, &nodes, &mut out); check_repeats(packet, &mut out); check_consent(packet, &nodes, &mut out); if inventory.has_revision(packet.inventory_rev) { @@ -235,11 +223,7 @@ pub(crate) fn admissible( rev: packet.inventory_rev, }); } - if out.is_empty() { - Ok(LoweringInput::of(packet)) - } else { - Err(out) - } + if out.is_empty() { Ok(()) } else { Err(out) } } fn check_nodes(packet: &Packet, out: &mut Vec) -> BTreeSet { @@ -285,6 +269,17 @@ fn check_refs(packet: &Packet, nodes: &BTreeSet, out: &mut Vec) } } +/// V17: the addressee, when it names a node, names a referent. A dangling id +/// is V4's. +fn check_addressee(packet: &Packet, nodes: &BTreeSet, out: &mut Vec) { + if let Some(a) = &packet.discourse.addressee + && nodes.contains(a) + && !packet.referents.iter().any(|r| &r.id == a) + { + out.push(Refusal::V17AddresseeNotAReferent { id: a.clone() }); + } +} + /// V14 and V15: the addressee's consent is a typed fact with one origin. A /// dangling id is V4's, so V14 judges only a node that exists. fn check_consent(packet: &Packet, nodes: &BTreeSet, out: &mut Vec) { @@ -560,11 +555,17 @@ fn fits(kind: Option<&ValueKind>, value: &FeatValue) -> bool { /// fn read(l: weksa::LoweringInput) { let _ = l.scene; } /// ``` /// -/// Scene values come only from a `weksa.scene` document: the raw-values door -/// is gone. The first block is the same call with a document; the second hands -/// it feature values and fails for that alone. +/// Scene values come only from the store: the call names neither speaker nor +/// scene. The first block is the call with the store; the second hands it a +/// scene value and a speaker and fails for that alone. /// /// ``` +/// fn pick(l: weksa::LoweringInput, i: &weksa::Inventory, s: &weksa::SpeakerStateStore) { +/// let _ = l.with_scene(i, s); +/// } +/// ``` +/// +/// ```compile_fail /// fn pick( /// l: weksa::LoweringInput, /// i: &weksa::Inventory, @@ -618,11 +619,16 @@ pub struct LoweringInput { constraints: Constraints, extensions: Vec, stance: Stance, + /// The item's speaker and group, as the corpus store holds them: the + /// only source of whose relation `with_scene` selects, and from which scene. + speaker: EntityId, + group: String, scene: Vec, } impl LoweringInput { - fn of(p: &Packet) -> Self { + /// The view of an admitted packet and the corpus item it was minted for. + pub(crate) fn of(p: &Packet, item: &CorpusItem) -> Self { Self { inventory_rev: p.inventory_rev, kind: p.kind, @@ -634,38 +640,44 @@ impl LoweringInput { constraints: p.constraints.clone(), extensions: p.extensions.clone(), stance: p.stance.clone(), + speaker: item.speaker.clone(), + group: item.group.clone(), scene: Vec::new(), } } /// Adds the speaker's relation to the addressee, the only path for scene - /// values into a view; they come only from a `weksa.scene` document. The - /// speaker must be a participant of `scene` (V16). With no addressee, or - /// one that names no referent, nothing is selected; otherwise the relation - /// from the speaker to the addressee referent's entity (to `Unnamed` when - /// the referent has none) is checked like every feature list, in slot - /// `Scene` at this view's inventory revision, and replaces any earlier - /// scene. The English lowerer ignores it. + /// values into a view. The speaker is the item's speaker and the scene is + /// the latest the store holds for the item's group; the caller names + /// neither. No scene for the group selects nothing; a scene that does not + /// list the item's speaker is V16. With no addressee nothing is selected; + /// otherwise the relation from the speaker to the addressee referent's + /// entity (to `Unnamed` when the referent has none) is checked like every + /// feature list, in slot `Scene` at this view's inventory revision, and + /// replaces any earlier scene. The English lowerer ignores it. pub fn with_scene( mut self, inventory: &Inventory, - scene: &Scene, - speaker: &EntityId, - ) -> Result> { - if !scene.participants.contains(speaker) { - return Err(vec![Refusal::V16SpeakerNotInScene]); + state: &SpeakerStateStore, + ) -> Result { + self.scene = Vec::new(); + let Some(scene) = state.scene(&self.group)? else { + return Ok(self); + }; + if !scene.participants.contains(&self.speaker) { + return Err(StoreError::Invalid(vec![Refusal::V16SpeakerNotInScene])); } - let addressee = self + let selected = self .discourse .addressee .as_ref() - .and_then(|id| self.referents.iter().find(|r| &r.id == id)); - let selected = addressee - .map(|r| scene.relation(speaker, r.entity.as_ref())) - .unwrap_or_default(); + .and_then(|a| self.referents.iter().find(|r| &r.id == a)) + .map_or(&[][..], |r| { + scene.relation(&self.speaker, r.entity.as_ref()) + }); let out = feature_refusals(inventory, self.inventory_rev, selected, Slot::Scene); if !out.is_empty() { - return Err(out); + return Err(StoreError::Invalid(out)); } self.scene = selected.to_vec(); Ok(self) diff --git a/crates/weksa/tests/display.rs b/crates/weksa/tests/display.rs index 44063fa..7a1eaf0 100644 --- a/crates/weksa/tests/display.rs +++ b/crates/weksa/tests/display.rs @@ -122,6 +122,8 @@ fn refusal_display_names_the_rule_and_no_identifier() { Refusal::V15ConsentSource { source: ProvenanceSource::ImportedCorpus, }, + Refusal::V16SpeakerNotInScene, + Refusal::V17AddresseeNotAReferent { id: node() }, ]; for r in &all { assert!(r.to_string().starts_with(r.rule()), "{r:?}"); diff --git a/crates/weksa/tests/scene.rs b/crates/weksa/tests/scene.rs index bc64acc..46885ae 100644 --- a/crates/weksa/tests/scene.rs +++ b/crates/weksa/tests/scene.rs @@ -1,244 +1,274 @@ -//! The speaker's relation to the addressee as a lowering input: the relation of -//! a `weksa.scene` document from the speaker to the addressee referent's -//! entity, typed feature values of slot `Scene` checked against the inventory -//! at the view's revision, never wording. +//! The speaker's relation to the addressee as a lowering input. `with_scene` +//! names neither speaker nor scene: the view knows its corpus item, so the +//! speaker is the item's speaker and the scene is the store's latest for the +//! item's group. The relation is the one from that speaker to the addressee +//! referent's entity. Typed feature values only, never wording. mod common; -use std::path::Path; - -use common::{eid, grade, iv, lab, nd}; +use common::{cast, eid, entities_in, grade, item_in, lab, nd}; use weksa::inventory::{proof_revision, seed, speaker_scene_revision}; use weksa::packet::{FeatValue, FeatureValue, Packet, proof_packet}; use weksa::speaker::{Annotator, PairRelation, Party, Scene}; use weksa::{ - CorpusStore, EntityId, EntityStore, Inventory, InventoryStore, LoweringInput, PacketStore, - Refusal, lower, + CorpusStore, EntityStore, Inventory, InventoryStore, LoweringInput, NameSpan, PacketStore, + Refusal, SpeakerStateStore, StoreError, lower, }; -fn store() -> (InventoryStore, Inventory, tempfile::TempDir) { +struct W { + dir: tempfile::TempDir, + inventory: InventoryStore, + corpus: CorpusStore, + entities: EntityStore, + state: SpeakerStateStore, +} + +/// Group `g` holds `item:nibu:1`: Nibu says "Brokkr, come here." Group `h` +/// holds `item:nibu:2`, also spoken by Nibu. +fn world() -> W { let dir = tempfile::tempdir().unwrap(); - let mut store = InventoryStore::open(&dir.path().join("i.cc")).unwrap(); + let mut inventory = InventoryStore::open(&dir.path().join("i.cc")).unwrap(); for (rev, entries) in [seed(), proof_revision(), speaker_scene_revision()] { - store.admit_revision(rev, entries).unwrap(); + inventory.admit_revision(rev, entries).unwrap(); + } + let entities = entities_in(dir.path()); + let mut corpus = CorpusStore::open(&dir.path().join("c.cc")).unwrap(); + let mut first = item_in("item:nibu:1", "Brokkr, come here.", "g", cast()); + first.names = vec![NameSpan { + entity: eid("brokkr"), + start: 0, + end: 6, + }]; + corpus.add(first, &entities).unwrap(); + corpus + .add(item_in("item:nibu:2", "Go away.", "h", cast()), &entities) + .unwrap(); + let state = SpeakerStateStore::open(&dir.path().join("s.cc")).unwrap(); + W { + dir, + inventory, + corpus, + entities, + state, } - let inv = store.inventory().unwrap(); - (store, inv, dir) } -fn admitted(packet: &Packet, inventory: &InventoryStore) -> LoweringInput { - let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); - let corpus = CorpusStore::open(&root.join("data/corpus/corpus.cc")).unwrap(); - let entities = EntityStore::open(&root.join("data/entity/entity.cc")).unwrap(); - let dir = tempfile::tempdir().unwrap(); - PacketStore::open(&dir.path().join("p.cc")) - .unwrap() - .admit(packet.clone(), inventory, &corpus, &entities) - .unwrap() +fn inv(w: &W) -> Inventory { + w.inventory.inventory().unwrap() } -/// The proof packet at `rev`. Its referent `n1` has no entity and `n3` is -/// `entity:nibu`; `addressee` names which one is spoken to, or none. -fn packet(rev: u32, addressee: Option<&str>) -> Packet { - let mut p = proof_packet(); - p.inventory_rev = rev; - p.discourse.addressee = addressee.map(nd); - p -} - -fn feat(inv: &Inventory, gloss: &str, value: FeatValue) -> FeatureValue { - let id = inv - .entries() +fn feat(gloss: &str, value: FeatValue) -> FeatureValue { + let feature = seed() + .1 + .into_iter() + .chain(speaker_scene_revision().1) .find(|e| e.gloss == gloss) .unwrap_or_else(|| panic!("no feature {gloss}")) .id; - FeatureValue { feature: id, value } + FeatureValue { feature, value } } -fn rules(r: Result>) -> Vec<&'static str> { - r.unwrap_err().iter().map(Refusal::rule).collect() +fn intimacy(v: u16) -> Vec { + vec![feat("intimacy", FeatValue::Grade(grade(v)))] } -fn speaker() -> EntityId { - eid("brokkr") +fn status() -> Vec { + vec![ + feat("relative_status", FeatValue::Option(lab("above"))), + feat("intimacy", FeatValue::Grade(grade(200))), + feat("in_group", FeatValue::Flag(false)), + ] } -/// Brokkr speaks; the scene holds a relation to Nibu and one to an unnamed -/// party, with different values. -fn scene(inv: &Inventory) -> Scene { - scene_with(vec![ - PairRelation { - from: speaker(), - to: Party::Named(eid("nibu")), - features: vec![ - feat(inv, "relative_status", FeatValue::Option(lab("above"))), - feat(inv, "intimacy", FeatValue::Grade(grade(200))), - feat(inv, "in_group", FeatValue::Flag(false)), - ], - }, - PairRelation { - from: speaker(), - to: Party::Unnamed, - features: vec![feat(inv, "intimacy", FeatValue::Grade(grade(7)))], - }, - ]) +fn relation(from: &str, to: Party, features: Vec) -> PairRelation { + PairRelation { + from: eid(from), + to, + features, + } } -fn scene_with(relations: Vec) -> Scene { +/// Nibu's relations to Brokkr and to an unnamed party, and Brokkr's to Nibu. +fn scene(group: &str, n: u32, participants: &[&str], relations: Vec) -> Scene { Scene { - group: "g".into(), - n: 1, + group: group.into(), + n, inventory_rev: 3, - participants: vec![speaker(), eid("nibu")], + participants: participants.iter().map(|p| eid(p)).collect(), relations, annotator: Annotator::Operator, } } -fn selected(view: LoweringInput, inv: &Inventory, s: &Scene) -> Vec { - view.with_scene(inv, s, &speaker()) +fn standard(group: &str, n: u32) -> Scene { + scene( + group, + n, + &["nibu", "brokkr"], + vec![ + relation("nibu", Party::Named(eid("brokkr")), status()), + relation("nibu", Party::Unnamed, intimacy(7)), + relation("brokkr", Party::Named(eid("nibu")), intimacy(99)), + ], + ) +} + +fn put(w: &mut W, s: Scene) { + w.state + .put_scene(s, &w.inventory, &w.corpus, &w.entities) + .unwrap(); +} + +/// The line of `item:nibu:1` at `rev`, its addressee node `addressee` (`n1` +/// has no entity, `n3` is Brokkr), through the packet door. +fn view_of(w: &W, rev: u32, addressee: Option<&str>) -> LoweringInput { + let mut p: Packet = proof_packet(); + p.inventory_rev = rev; + p.referents[2].entity = Some(eid("brokkr")); + p.discourse.addressee = addressee.map(nd); + // A packet key holds its packet for good, so each view gets a fresh store. + let scratch = tempfile::tempdir().unwrap(); + PacketStore::open(&scratch.path().join("p.cc")) .unwrap() - .scene() - .to_vec() + .admit(p, &w.inventory, &w.corpus, &w.entities) + .unwrap() +} + +fn rules(r: Result) -> Vec<&'static str> { + match r.map(|_| ()).unwrap_err() { + StoreError::Invalid(rs) => rs.iter().map(Refusal::rule).collect(), + other => panic!("not a rule refusal: {other}"), + } } #[test] fn a_scene_relation_reaches_the_lowering_view_and_changes_no_english() { - let (store, inv, _dir) = store(); - let plain = admitted(&packet(3, Some("n3")), &store); + let mut w = world(); + put(&mut w, standard("g", 1)); + let plain = view_of(&w, 3, Some("n3")); assert!(plain.scene().is_empty()); - let s = scene(&inv); - let with = plain.clone().with_scene(&inv, &s, &speaker()).unwrap(); - assert_eq!(with.scene(), s.relations[0].features.as_slice()); - assert!(!with.scene().is_empty()); - let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); - let entities = EntityStore::open(&root.join("data/entity/entity.cc")).unwrap(); + let with = plain.clone().with_scene(&inv(&w), &w.state).unwrap(); + assert_eq!(with.scene(), status().as_slice()); assert_eq!( - lower(&with, &inv, &entities).unwrap(), - lower(&plain, &inv, &entities).unwrap() + lower(&with, &inv(&w), &w.entities).unwrap(), + lower(&plain, &inv(&w), &w.entities).unwrap() ); } #[test] fn with_scene_selects_by_addressee() { - let (store, inv, _dir) = store(); - let s = scene(&inv); - // A referent with an entity selects the relation to that entity. - let named = selected(admitted(&packet(3, Some("n3")), &store), &inv, &s); - assert_eq!(named, s.relations[0].features); - // A referent without one selects the relation to an unnamed party. - let unnamed = selected(admitted(&packet(3, Some("n1")), &store), &inv, &s); - assert_eq!(unnamed, s.relations[1].features); - // No addressee selects nothing. - assert!(selected(admitted(&packet(3, None), &store), &inv, &s).is_empty()); - // A named addressee the scene lacks selects nothing, though an unnamed - // relation exists. - let only_unnamed = scene_with(vec![s.relations[1].clone()]); - assert!( - selected( - admitted(&packet(3, Some("n3")), &store), - &inv, - &only_unnamed - ) - .is_empty() - ); - // Another speaker's relations are not selected. - let view = admitted(&packet(3, Some("n3")), &store); - let theirs = Scene { - participants: vec![speaker(), eid("nibu"), eid("hero")], - ..s.clone() + let mut w = world(); + put(&mut w, standard("g", 1)); + let take = |w: &W, addressee| { + view_of(w, 3, addressee) + .with_scene(&inv(w), &w.state) + .unwrap() + .scene() + .to_vec() }; - assert!( - view.with_scene(&inv, &theirs, &eid("hero")) - .unwrap() - .scene() - .is_empty() + // A referent with an entity selects the relation to that entity; a + // referent without one, the relation to an unnamed party; and the + // speaker's relation, never Brokkr's to Nibu. + assert_eq!(take(&w, Some("n3")), status()); + assert_eq!(take(&w, Some("n1")), intimacy(7)); + // No addressee selects nothing. + assert!(take(&w, None).is_empty()); +} + +#[test] +fn a_named_addressee_the_scene_lacks_never_falls_back_to_unnamed() { + let mut w = world(); + // Brokkr is not a participant and Nibu has only an unnamed relation. + put( + &mut w, + scene( + "g", + 1, + &["nibu", "player"], + vec![relation("nibu", Party::Unnamed, intimacy(7))], + ), ); - // A replacement replaces: an empty selection clears the earlier one. - let first = admitted(&packet(3, Some("n3")), &store) - .with_scene(&inv, &s, &speaker()) + let got = view_of(&w, 3, Some("n3")) + .with_scene(&inv(&w), &w.state) .unwrap(); - assert!( - first - .with_scene(&inv, &only_unnamed, &speaker()) - .unwrap() - .scene() - .is_empty() + assert!(got.scene().is_empty()); + // A participant without a relation behaves the same. + put( + &mut w, + scene( + "g", + 2, + &["nibu", "brokkr"], + vec![relation("nibu", Party::Unnamed, intimacy(7))], + ), ); + let got = view_of(&w, 3, Some("n3")) + .with_scene(&inv(&w), &w.state) + .unwrap(); + assert!(got.scene().is_empty()); +} + +#[test] +fn the_scene_is_the_items_group_and_its_latest_revision() { + let mut w = world(); + // Group h holds a different relation for the same pair. + put( + &mut w, + scene( + "h", + 1, + &["nibu", "brokkr"], + vec![relation("nibu", Party::Named(eid("brokkr")), intimacy(1))], + ), + ); + // No scene for the item's group selects nothing, whatever other groups hold. + let got = view_of(&w, 3, Some("n3")) + .with_scene(&inv(&w), &w.state) + .unwrap(); + assert!(got.scene().is_empty()); + // Revisions: the highest n is read. + put(&mut w, standard("g", 1)); + let mut second = standard("g", 2); + second.relations[0].features = intimacy(5); + put(&mut w, second); + let got = view_of(&w, 3, Some("n3")) + .with_scene(&inv(&w), &w.state) + .unwrap(); + assert_eq!(got.scene(), intimacy(5).as_slice()); + // A replacement replaces: a store without the scene clears the selection. + let empty = SpeakerStateStore::open(&w.dir.path().join("empty.cc")).unwrap(); + let cleared = got.with_scene(&inv(&w), &empty).unwrap(); + assert!(cleared.scene().is_empty()); } #[test] fn with_scene_refusals() { - let (store, inv, _dir) = store(); - let view = admitted(&packet(3, Some("n3")), &store); - let to_nibu = |features| { - scene_with(vec![PairRelation { - from: speaker(), - to: Party::Named(eid("nibu")), - features, - }]) - }; - let take = |s: &Scene| view.clone().with_scene(&inv, s, &speaker()); - // A speaker the scene does not list. + let mut w = world(); + put(&mut w, standard("g", 1)); + // The scene was admitted at revision 3; a view of revision 2 predates the + // Scene features, so the selected relation's ids do not resolve there. assert_eq!( - view.clone() - .with_scene(&inv, &scene(&inv), &eid("hero")) - .map(|_| ()) - .unwrap_err(), - [Refusal::V16SpeakerNotInScene] - ); - // A label the feature does not declare. - let lordly = feat(&inv, "relative_status", FeatValue::Option(lab("lordly"))); - assert_eq!(rules(take(&to_nibu(vec![lordly]))), ["V7"]); - // A value of the wrong kind. - let flag = feat(&inv, "intimacy", FeatValue::Flag(true)); - assert_eq!(rules(take(&to_nibu(vec![flag]))), ["V7"]); - // A stance feature in a relation. - let anger = feat(&inv, "anger", FeatValue::Grade(grade(1))); - assert_eq!(rules(take(&to_nibu(vec![anger]))), ["V8"]); - // A speaker feature in a relation. - let age = feat(&inv, "education", FeatValue::Grade(grade(1))); - assert_eq!(rules(take(&to_nibu(vec![age]))), ["V8"]); - // Twice in one list. - let twice = vec![ - feat(&inv, "intimacy", FeatValue::Grade(grade(1))), - feat(&inv, "intimacy", FeatValue::Grade(grade(2))), - ]; - assert_eq!(rules(take(&to_nibu(twice))), ["V11"]); - // Revision 2 predates the Scene features, so their ids do not resolve there. - let old = admitted(&packet(2, Some("n3")), &store); - assert_eq!( - rules(old.with_scene(&inv, &scene(&inv), &speaker())), + rules(view_of(&w, 2, Some("n3")).with_scene(&inv(&w), &w.state)), ["V5", "V5", "V5"] ); - // An unselected relation is not checked: only the selected one is. - let bad_elsewhere = scene_with(vec![ - PairRelation { - from: speaker(), - to: Party::Unnamed, - features: vec![feat(&inv, "anger", FeatValue::Grade(grade(1)))], - }, - PairRelation { - from: speaker(), - to: Party::Named(eid("nibu")), - features: vec![], - }, - ]); - assert!(take(&bad_elsewhere).unwrap().scene().is_empty()); -} - -#[test] -fn feature_ids_are_stable() { - let (_store, inv, _dir) = store(); - let ids: Vec<_> = [ - "relative_status", - "intimacy", - "in_group", - "socioeconomic_standing", - "education", - "age_cohort", - ] - .map(|g| feat(&inv, g, FeatValue::Flag(true)).feature) - .into(); - let want: Vec<_> = (19..=24).map(|n| iv(&format!("feature:{n}"))).collect(); - assert_eq!(ids, want); + // A scene that does not list the item's speaker: group k admitted while + // Brokkr was its only speaker, then an item of Nibu's joins it. + let mut item = item_in("item:brokkr:1", "Hello.", "k", cast()); + item.speaker = eid("brokkr"); + w.corpus.add(item, &w.entities).unwrap(); + put(&mut w, scene("k", 1, &["brokkr"], vec![])); + w.corpus + .add( + item_in("item:nibu:3", "Hello there.", "k", cast()), + &w.entities, + ) + .unwrap(); + let mut p = proof_packet(); + p.inventory_rev = 3; + p.provenance.corpus_item = weksa::CorpusItemId::parse("item:nibu:3").unwrap(); + p.referents[2].entity = Some(eid("nibu")); + let view = PacketStore::open(&w.dir.path().join("p3.cc")) + .unwrap() + .admit(p, &w.inventory, &w.corpus, &w.entities) + .unwrap(); + assert_eq!(rules(view.with_scene(&inv(&w), &w.state)), ["V16"]); } diff --git a/crates/weksa/tests/speaker_state.rs b/crates/weksa/tests/speaker_state.rs index 864d65a..dfb4c59 100644 --- a/crates/weksa/tests/speaker_state.rs +++ b/crates/weksa/tests/speaker_state.rs @@ -22,7 +22,7 @@ struct W { } /// Group `g` holds two live items (speakers nibu and brokkr) and one withdrawn -/// item (speaker hero). +/// item (speaker hero); group `g2` holds one live item of nibu's. fn world() -> W { let dir = tempfile::tempdir().unwrap(); let mut inventory = InventoryStore::open(&dir.path().join("i.cc")).unwrap(); @@ -35,9 +35,14 @@ fn world() -> W { ("item:nibu:1", "Come here.", "nibu"), ("item:nibu:2", "Go there.", "brokkr"), ("item:nibu:3", "Stay away.", "hero"), + ("item:nibu:4", "Elsewhere.", "g2"), ] { let mut item = item_in(key, text, "g", cast()); item.speaker = eid(speaker); + if key == "item:nibu:4" { + item.group = "g2".into(); + item.speaker = eid("nibu"); + } corpus.add(item, &entities).unwrap(); } corpus.withdraw("item:nibu:3").unwrap(); @@ -219,6 +224,26 @@ fn scene_revisions_append() { assert_eq!(w.state.scene("g").unwrap().unwrap().n, 10); let listed: Vec = w.state.all_scenes().unwrap().iter().map(|s| s.n).collect(); assert_eq!(listed, (1..=10).collect::>()); + // Numbering is per group: the first scene of another group is revision 1 + // whatever g holds, and its second is 2. + let other = |n| Scene { + group: "g2".into(), + participants: vec![eid("nibu")], + relations: vec![], + ..scene(n, 1) + }; + assert_eq!( + refused(put_scene(&mut w, other(2))), + not_next(&scene_key("g2", 2), 1, 2) + ); + assert_eq!(put_scene(&mut w, other(1)).unwrap(), AddOutcome::Added); + assert_eq!( + refused(put_scene(&mut w, other(3))), + not_next(&scene_key("g2", 3), 2, 3) + ); + assert_eq!(put_scene(&mut w, other(2)).unwrap(), AddOutcome::Added); + assert_eq!(w.state.scene("g2").unwrap().unwrap().n, 2); + assert_eq!(w.state.scene("g").unwrap().unwrap().n, 10); } #[test] diff --git a/crates/weksa/tests/validate.rs b/crates/weksa/tests/validate.rs index c5bc172..b46cd85 100644 --- a/crates/weksa/tests/validate.rs +++ b/crates/weksa/tests/validate.rs @@ -739,3 +739,26 @@ fn a_source_that_only_read_wording_cannot_set_consent() { assert_eq!(refusals(&p), vec![Refusal::V15ConsentSource { source }]); } } + +// V17: the addressee is someone spoken to, a referent. + +#[test] +fn the_addressee_names_a_referent_not_a_predication() { + let mut p = good(); + p.discourse.addressee = Some(nd("n2")); + assert_eq!( + refusals(&p), + vec![Refusal::V17AddresseeNotAReferent { id: nd("n2") }] + ); + // A referent passes, and a dangling id is V4's alone. + p.discourse.addressee = Some(nd("n1")); + assert_eq!(refusals(&p), vec![]); + p.discourse.addressee = Some(nd("n99")); + assert_eq!( + refusals(&p), + vec![Refusal::V4DanglingRef { + id: nd("n99"), + at: "addressee" + }] + ); +} diff --git a/docs/interlingua-standard.md b/docs/interlingua-standard.md index 830a2f7..3a300fd 100644 --- a/docs/interlingua-standard.md +++ b/docs/interlingua-standard.md @@ -163,6 +163,7 @@ names the rule. - V15: an `addressee_consent` is set only by provenance `Authored` or `GameTelemetry`; a decomposition of text or imported corpus text cannot mint it. +- V17: an `addressee` names a referent, not a predication. V9 is retired and not renumbered: the form of a key or a grade is its type, and an extension key's family is V6. `PacketStore::admit` returns the lowering view of a @@ -190,7 +191,7 @@ ids. ### `discourse` -Why the packet is said: `speech_act` is a speech-act id, `addressee` a node id, +Why the packet is said: `speech_act` is a speech-act id, `addressee` a referent's node id (V17), and `features` carry source-side nuance on content (hedge, certainty, irony and whatever the inventory grows). This is source-side communicative pressure, not target-language register. `addressee_consent` is no feature: it names the @@ -226,11 +227,13 @@ latest plus one, and the latest revision is the highest `n`. Each carries an `Annotator`, `Operator` or `Model` with an attestation string for its generator. Both hold typed feature values only, never text. A feature of slot `Stance`, `Speaker` or `Scene` sits in that slot alone (each layer has one writer and one -lifetime), which `admit_revision` enforces. `LoweringInput::with_scene` takes a -scene document and the speaker; the speaker must be a participant (V16), and -the relation selected is the speaker's to the entity of the addressee referent -(to `Unnamed` when that referent has no entity, to nothing when the packet has -no addressee). V13 is retired. +lifetime), which `admit_revision` enforces. `LoweringInput::with_scene` takes the +store and names neither speaker nor scene: the view knows its corpus item, so +the speaker is the item's speaker and the scene is the store's latest for the +item's group. The speaker must be a participant (V16), and the relation +selected is the speaker's to the entity of the addressee referent (to +`Unnamed` when that referent has no entity, to nothing when the packet has no +addressee or the scene holds no such relation). V13 is retired. ### `referents` diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 2b1959c..3cdc59e 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -399,8 +399,8 @@ One line per cut; specs exist for the cuts about to run (2d and 4, siblings on ` 2d. `entity-door-lows`: invisible format characters refused in entity forms (ZWJ, ZWNJ, CGJ and variation selectors kept); the form length bound per question `form-word-bound` (recommended: bytes only); attested origins (`Cast { persona, fixture }`, `Live { persona }`) and `CastAuthored.author`; empty provenance strings refused at the store; S6 under NFKC plus case fold and no-alphanumeric texts refused; `next_key` refuses at `u32::MAX`; CLI flags that do not apply refuse and no error echoes a value. Owns follow-ups `entity-door-lows` and `s6-unicode-normal-form`. (spec r1) 3b. `lowerer-first-cut` (follow-up `golf-loop-lowerer-cut`): the section above. `lower`, `weksa lower`, `weksa score`, `weksa packet proof`, inventory rev 2 for `nibu-proof-001`. 4. `speaker-state`: packet `stance` (felt affect and restraint, slot `Stance`, packet schema v4); formality, warmth and intensity out of the seed and the standard; only `PacketStore::admit` yields a `LoweringInput`, and it refuses packets for withdrawn items (follow-up `lowering-input-from-packet-store`). (spec r1) -4a. `speaker-relation-input`: slot `Scene` and its first seed features (`relative_status`, `intimacy`, `in_group`; inventory revision 3, now `speaker_scene_revision`); `LoweringInput::with_scene` carries the speaker's relation to the addressee as typed `Scene` feature values. Since 4b it takes a `weksa.scene` document and selects the relation by the addressee referent's entity (V16 when the speaker is not a participant; V13 is retired). The English lowerer does not render it. (PR for follow-up `chat-s3352-3352`) -4b. `speaker-scene` (spec r1; `speaker-scene-seeds` is its cut 4c: CLI, inventory revision 3, cast speakers and scenes; `Party::Unnamed` covers an addressee with no entity; Speaker and Scene share one `SpeakerStateStore`): slots `Speaker` and `Scene` with their seed features; `weksa.speaker` (`speaker:@`) and `weksa.scene` (`scene:@`, participants and ordered-pair relations) documents with `Annotator` `Operator` and `Model { family, generator }`, validated by the packet's shared feature check, append-only per key; their stores, `weksa speaker` and `weksa scene` commands, published schemas; Model-annotated cast speakers and scenes for the committed items. Waits on 2d and 4; `pd-speaker-annotation` blocks only the public-domain annotator. Part 1 landed (branch `weksa-lowering/speaker-scene`): `speaker.rs`, `SpeakerStateStore`, slots `Speaker` and `Scene` exclusive to their own feature lists, `speaker_scene_revision()` (Speaker features 22-24), `with_scene` on a Scene document, schemas. The CLI, the committed inventory revision 3 and the seeds stay with 4c. +4a. `speaker-relation-input`: slot `Scene` and its first seed features (`relative_status`, `intimacy`, `in_group`; inventory revision 3, now `speaker_scene_revision`); `LoweringInput::with_scene` carries the speaker's relation to the addressee as typed `Scene` feature values. Since 4b it takes the `SpeakerStateStore`, reads the scene of the view's own item group for the item's speaker, and selects the relation by the addressee referent's entity (V16 when the item's speaker is not a participant; V17 refuses an addressee that names a predication; V13 is retired). The English lowerer does not render it. (PR for follow-up `chat-s3352-3352`) +4b. `speaker-scene` (spec r1; `speaker-scene-seeds` is its cut 4c: CLI, inventory revision 3, cast speakers and scenes; `Party::Unnamed` covers an addressee with no entity; Speaker and Scene share one `SpeakerStateStore`): slots `Speaker` and `Scene` with their seed features; `weksa.speaker` (`speaker:@`) and `weksa.scene` (`scene:@`, participants and ordered-pair relations) documents with `Annotator` `Operator` and `Model { family, generator }`, validated by the packet's shared feature check, append-only per key; their stores, `weksa speaker` and `weksa scene` commands, published schemas; Model-annotated cast speakers and scenes for the committed items. Waits on 2d and 4; `pd-speaker-annotation` blocks only the public-domain annotator. Part 1 landed (branch `weksa-lowering/speaker-scene`): `speaker.rs`, `SpeakerStateStore`, slots `Speaker` and `Scene` exclusive to their own feature lists, `speaker_scene_revision()` (Speaker features 22-24), `with_scene` on the store, schemas. The CLI, the committed inventory revision 3 and the seeds stay with 4c. 5. `language-ontology`: `weksa.community` from `data/cultural-ontology/` (owns follow-up `edda-referent-yaml-parse`), `weksa.ontology_entry` for en-US and ja-JP from `data/target-language-ontology/` (YAML deleted), speaker communities, deterministic `select`, the lowering view (packet without trace, speaker, scene, selected entries). Waits on 4. 6. `pd-importer`: public-domain play and fiction dialogue into the corpus with speaker, scene group, edition provenance, and Speaker and Scene annotations by the ruled annotator. Waits on 4, follow-up `gutenberg-terms` and question `pd-speaker-annotation`. 7a. `loop-context-builder`: `context.rs` (`TrainItems`, `EncoderOffer`, `Prompt`: corpus text reaches a model call only through `Prompt::mint` over live Train items or `Prompt::encode` over one offered item; instructions are `&'static str`), `SplitScheme::live`, `CorpusItem::cited_entities` (one owner of the id set `PacketStore::admit` and the offer share), and checkpoint-only Test readings (`Checkpoint`, `TestReadings`; `Aggregate` and `leak_diagnostic` carry no Test). Library only. Merges follow-ups `loop-context-builder-2` and `scorer-test-readings-type`. (spec r1) From 43f2e118f219951eaf2c8f1cf468e282bf38d82f Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 02:59:11 +0000 Subject: [PATCH 033/175] Checkpoint: offers borrow it; door pinned by source shape; doc states the real guard EncoderOffer<'cp> borrows its Checkpoint so a Test offer cannot be encoded after the reading spends it. tests/checkpoint_door.rs scans split.rs for any public minting item under any name. The split.rs doc says operator tools read corpus text and the guarded invariant is the round path. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/context.rs | 43 +++++++++-- crates/weksa/src/score/mod.rs | 2 +- crates/weksa/src/split.rs | 27 +++++-- crates/weksa/tests/checkpoint_door.rs | 101 ++++++++++++++++++++++++++ 4 files changed, 160 insertions(+), 13 deletions(-) create mode 100644 crates/weksa/tests/checkpoint_door.rs diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index 8914724..a4f2749 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -6,6 +6,8 @@ //! [`ENCODE_INSTRUCTIONS`], consts in this module, and nothing else: `mint` and //! `encode` take none, so no run-time text can become instructions. +use std::marker::PhantomData; + use crate::corpus::CorpusItem; use crate::packet::EntityId; use crate::split::{Checkpoint, Split, SplitScheme}; @@ -98,14 +100,42 @@ pub enum OfferRefusal { /// One item the encoder may be shown, with the entities a packet for it may /// cite. Built per item, so a call sees one item and nothing of its group. -pub struct EncoderOffer { +/// +/// An offer made under a [`Checkpoint`] borrows it, so it is dead before +/// `TestReadings::at_checkpoint` can spend the checkpoint, and a Test offer is +/// never encoded after (or beside) the reading. A round offer borrows nothing. +/// +/// ``` +/// use weksa::context::{EncoderOffer, TrainItems}; +/// use weksa::split::Checkpoint; +/// use weksa::{CorpusItem, score::TestReadings}; +/// fn f(mut c: Checkpoint, i: &CorpusItem, t: &TrainItems) { +/// if let Ok(o) = EncoderOffer::for_checkpoint(i, &mut c) { +/// let _ = weksa::context::Prompt::encode(&o, t); +/// } +/// let _ = TestReadings::at_checkpoint(c, &[], &[], &[]); +/// } +/// ``` +/// +/// ```compile_fail +/// use weksa::context::{EncoderOffer, TrainItems}; +/// use weksa::split::Checkpoint; +/// use weksa::{CorpusItem, score::TestReadings}; +/// fn f(mut c: Checkpoint, i: &CorpusItem, t: &TrainItems) { +/// let o = EncoderOffer::for_checkpoint(i, &mut c).unwrap(); +/// let _ = TestReadings::at_checkpoint(c, &[], &[], &[]); +/// let _ = weksa::context::Prompt::encode(&o, t); +/// } +/// ``` +pub struct EncoderOffer<'cp> { item: CorpusItem, entities: Vec, + _checkpoint: PhantomData<&'cp mut Checkpoint>, } -impl EncoderOffer { +impl<'cp> EncoderOffer<'cp> { /// A round offers Train and Validation items. - pub fn for_round(item: &CorpusItem) -> Result { + pub fn for_round(item: &CorpusItem) -> Result, OfferRefusal> { match Self::gate(item)? { Split::Test => Err(OfferRefusal::TestBeforeCheckpoint), _ => Ok(Self::of(item)), @@ -117,7 +147,7 @@ impl EncoderOffer { /// item records no key. pub fn for_checkpoint( item: &CorpusItem, - checkpoint: &mut Checkpoint, + checkpoint: &'cp mut Checkpoint, ) -> Result { Self::gate(item)?; if !checkpoint.first_offer(&item.id) { @@ -133,13 +163,14 @@ impl EncoderOffer { SplitScheme::V1.live(item).ok_or(OfferRefusal::NoSplit) } - fn of(item: &CorpusItem) -> Self { + fn of<'a>(item: &CorpusItem) -> EncoderOffer<'a> { let mut entities: Vec = item.cited_entities().cloned().collect(); entities.sort(); entities.dedup(); - Self { + EncoderOffer { item: item.clone(), entities, + _checkpoint: PhantomData, } } diff --git a/crates/weksa/src/score/mod.rs b/crates/weksa/src/score/mod.rs index c531e89..2b55f3b 100644 --- a/crates/weksa/src/score/mod.rs +++ b/crates/weksa/src/score/mod.rs @@ -158,7 +158,7 @@ pub fn aggregate(scored: &[(&CorpusItem, ItemScore)]) -> Aggregate { /// The Test split's readings, scalars only: no item key, no minted string, no /// pair. The checkpoint decision (convergence-test) needs numbers; a key beside /// a minted string is the handle an edit step would act on. The only producer -/// is `at_checkpoint`, which consumes a `Checkpoint`, which only the lib can make. +/// is `at_checkpoint`, which consumes a `Checkpoint`; only the lib can make one. /// /// ``` /// use weksa::score::TestReadings; diff --git a/crates/weksa/src/split.rs b/crates/weksa/src/split.rs index 5d3a5ee..e50dc7b 100644 --- a/crates/weksa/src/split.rs +++ b/crates/weksa/src/split.rs @@ -1,7 +1,11 @@ //! The train/validation/test split is derived from an item's group and nothing //! else. No stored assignment exists, so no writer can reassign one. An item is -//! read in its live split; Test is read only under a Checkpoint, and only the -//! lib can make one (convergence-freeze adds the door). +//! read in its live split. The round path (round records, prompts, `aggregate`, +//! `leak_diagnostic`) carries no Test reading; a typed `TestReadings` and a Test +//! `EncoderOffer` need a Checkpoint, and only the lib can make one +//! (convergence-freeze adds the door). Operator tools (`weksa corpus dump`, +//! `weksa score`, `CorpusStore`) read corpus text of any split; that is not +//! gated here. use std::collections::BTreeSet; @@ -66,14 +70,18 @@ impl SplitScheme { } } -/// Permission to read Test once. Only the weksa lib can make one: there is no -/// public constructor, so nothing outside it (the binary, a driver crate, a -/// doctest, an integration test) can read Test. The one door is added by +/// Permission to read Test once, as typed `TestReadings` or as Test +/// `EncoderOffer`s. Only the weksa lib can make one: there is no public +/// constructor, so no code outside it can obtain those. This is not a seal on +/// corpus text: operator tools and `CorpusStore` can read any item, and the +/// guarded invariant is that the round path holds no Test reading. The one door +/// is added by /// convergence-freeze, which builds a `Checkpoint` from the convergence window /// over the run's persisted round records (convergence-test). /// `TestReadings::at_checkpoint` consumes it, so one `Checkpoint` buys one Test /// reading. One `Checkpoint` also offers each item at most once -/// (`EncoderOffer::for_checkpoint`). +/// (`EncoderOffer::for_checkpoint`), and an offer borrows its Checkpoint, so it +/// cannot outlive the reading that spends it. /// /// ``` /// fn f(c: &weksa::split::Checkpoint) -> u32 { c.round() } @@ -85,6 +93,13 @@ impl SplitScheme { /// let _ = weksa::split::Checkpoint::declare(1); /// ``` /// +/// The test constructor is not exported (its name today; the door scan in +/// `tests/checkpoint_door.rs` pins the shape under any name): +/// +/// ```compile_fail +/// let _ = weksa::split::Checkpoint::for_test(1); +/// ``` +/// /// ```compile_fail /// let _ = weksa::split::Checkpoint { round: 1, offered: Default::default() }; /// ``` diff --git a/crates/weksa/tests/checkpoint_door.rs b/crates/weksa/tests/checkpoint_door.rs new file mode 100644 index 0000000..2de6668 --- /dev/null +++ b/crates/weksa/tests/checkpoint_door.rs @@ -0,0 +1,101 @@ +//! The Checkpoint door is pinned by source shape, not by a name: no public item +//! of the lib can mint one, and nothing may derive, implement or default one. +//! `Checkpoint::for_test` is `cfg(test)`; a doctest names it today, and this +//! scan catches a rename. + +/// Why `src` lets code outside the lib make a `Checkpoint`, if it does. +fn door_leaks(split_rs: &str) -> Vec { + let mut leaks = Vec::new(); + let lines: Vec<&str> = split_rs.lines().collect(); + let mut in_impl = false; + let mut in_struct = false; + for (n, line) in lines.iter().enumerate() { + let t = line.trim(); + if t.starts_with("impl Checkpoint") { + in_impl = true; + } else if t.starts_with("impl") && t.contains("Checkpoint") { + leaks.push(format!("{}: trait impl: {t}", n + 1)); + } + if t.starts_with("pub struct Checkpoint") { + let derived = lines[..n] + .iter() + .rev() + .take_while(|l| l.trim().starts_with("#[") || l.trim().starts_with("///")) + .any(|l| l.contains("derive")); + if derived { + leaks.push(format!("{}: derive on Checkpoint", n + 1)); + } + in_struct = true; + continue; + } + if in_struct { + if t.starts_with('}') { + in_struct = false; + } else if t.starts_with("pub ") { + leaks.push(format!("{}: public field: {t}", n + 1)); + } + } + if in_impl && t.starts_with("pub ") && t.contains("fn ") { + let mints = t.contains("-> Self") || t.contains("-> Checkpoint"); + let test_only = lines[..n] + .iter() + .rev() + .take_while(|l| l.trim().starts_with("#[")) + .any(|l| l.contains("cfg(test)")); + if mints && !test_only { + leaks.push(format!("{}: pub constructor: {t}", n + 1)); + } + } + } + leaks +} + +#[test] +fn no_public_item_mints_a_checkpoint() { + let src = + std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/src/split.rs")).unwrap(); + assert_eq!(door_leaks(&src), Vec::::new()); + // No other module returns one either. + for entry in std::fs::read_dir(concat!(env!("CARGO_MANIFEST_DIR"), "/src")).unwrap() { + let path = entry.unwrap().path(); + if path.extension().is_some_and(|e| e == "rs") && !path.ends_with("split.rs") { + let text = std::fs::read_to_string(&path).unwrap(); + let minting = text + .lines() + .filter(|l| l.contains("-> Checkpoint") || l.contains("-> Result Self { x }\n}\n"; + assert_eq!(door_leaks(stray).len(), 1); + assert_eq!(door_leaks("impl Default for Checkpoint {}\n").len(), 1); + assert_eq!(door_leaks("impl From for Checkpoint {}\n").len(), 1); + assert_eq!( + door_leaks("#[derive(Clone)]\npub struct Checkpoint {\n").len(), + 1 + ); + assert_eq!( + door_leaks("pub struct Checkpoint {\n pub round: u32,\n}\n").len(), + 1 + ); + assert!(door_leaks("pub struct Checkpoint {\n round: u32,\n}\n").is_empty()); +} From 7a33a8d70f0757c46bb1753d56e3629e92ea3f20 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 02:59:40 +0000 Subject: [PATCH 034/175] Map: say what the Checkpoint gate guards (7a) Co-Authored-By: Claude Sonnet 5.5 --- docs/targets/interlingua-lowering-map.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 2f4467a..f011c53 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -402,7 +402,7 @@ One line per cut; specs exist for the cuts about to run (2d and 4, siblings on ` 4b. `speaker-scene` (spec r1; `speaker-scene-seeds` is its cut 4c: CLI, inventory revision 3, cast speakers and scenes; `Party::Unnamed` covers an addressee with no entity; Speaker and Scene share one `SpeakerStateStore`): slots `Speaker` and `Scene` with their seed features; `weksa.speaker` (`speaker:@`) and `weksa.scene` (`scene:@`, participants and ordered-pair relations) documents with `Annotator` `Operator` and `Model { family, generator }`, validated by the packet's shared feature check, append-only per key; their stores, `weksa speaker` and `weksa scene` commands, published schemas; Model-annotated cast speakers and scenes for the committed items. Waits on 2d and 4; `pd-speaker-annotation` blocks only the public-domain annotator. 5. `language-ontology`: `weksa.community` from `data/cultural-ontology/` (owns follow-up `edda-referent-yaml-parse`), `weksa.ontology_entry` for en-US and ja-JP from `data/target-language-ontology/` (YAML deleted), speaker communities, deterministic `select`, the lowering view (packet without trace, speaker, scene, selected entries). Waits on 4. 6. `pd-importer`: public-domain play and fiction dialogue into the corpus with speaker, scene group, edition provenance, and Speaker and Scene annotations by the ruled annotator. Waits on 4, follow-up `gutenberg-terms` and question `pd-speaker-annotation`. -7a. `loop-context-builder`: `context.rs` (`TrainItems`, `EncoderOffer`, `Prompt`: corpus text reaches a model call only through `Prompt::mint` over live Train items or `Prompt::encode` over one offered item; instructions are the consts `MINT_INSTRUCTIONS` and `ENCODE_INSTRUCTIONS` in `context.rs`; `Checkpoint` has no public constructor and offers each item once; prior turns print the held Train text), `SplitScheme::live`, `CorpusItem::cited_entities` (one owner of the id set `PacketStore::admit` and the offer share), and checkpoint-only Test readings (`Checkpoint`, `TestReadings`; `Aggregate` and `leak_diagnostic` carry no Test). Library only. Merges follow-ups `loop-context-builder-2` and `scorer-test-readings-type`. (spec r1) +7a. `loop-context-builder`: `context.rs` (`TrainItems`, `EncoderOffer`, `Prompt`: corpus text reaches a model call only through `Prompt::mint` over live Train items or `Prompt::encode` over one offered item; instructions are the consts `MINT_INSTRUCTIONS` and `ENCODE_INSTRUCTIONS` in `context.rs`; `Checkpoint` has no public constructor, offers each item once, and its offers borrow it (the gate is typed `TestReadings` and Test offers on the round path; operator tools such as `weksa corpus dump` and `weksa score` still read corpus text of any split); prior turns print the held Train text), `SplitScheme::live`, `CorpusItem::cited_entities` (one owner of the id set `PacketStore::admit` and the offer share), and checkpoint-only Test readings (`Checkpoint`, `TestReadings`; `Aggregate` and `leak_diagnostic` carry no Test). Library only. Merges follow-ups `loop-context-builder-2` and `scorer-test-readings-type`. (spec r1) 7. `golf-loop-en`: the loop driver, the template wording in `context.rs` with its digest pinned per run (follow-up `template-digest-in-round-record`), the packet store bound to one inventory store (`packet-store-inventory-binding`), `ModelPort` with the `claude -p` adapter (it takes only `&Prompt`), encoder (packet with stance) and lowerer prompts, round records, Score documents, the composite and accept rule, `weksa score`, the detached unit on Yggdrasil. Waits on 3, 5, 6 and follow-up `claude-on-yggdrasil`. 8. `style-axis`: a style-embedding score behind the scorer port. Waits on 3. 9. `ja-panel`: ja-JP into the loop's panel. Waits on 7. From 62059337a3ff21ac4cf8c4f91d06400014afd7a2 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 03:04:48 +0000 Subject: [PATCH 035/175] Door scan: say it is a line-based tripwire and what it misses Docs only; no behaviour change. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/split.rs | 5 +++-- crates/weksa/tests/checkpoint_door.rs | 14 ++++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/crates/weksa/src/split.rs b/crates/weksa/src/split.rs index e50dc7b..7bfdd47 100644 --- a/crates/weksa/src/split.rs +++ b/crates/weksa/src/split.rs @@ -93,8 +93,9 @@ impl SplitScheme { /// let _ = weksa::split::Checkpoint::declare(1); /// ``` /// -/// The test constructor is not exported (its name today; the door scan in -/// `tests/checkpoint_door.rs` pins the shape under any name): +/// The test constructor is not exported (this doctest names it today; the +/// line-based tripwire in `tests/checkpoint_door.rs` catches only common +/// single-line forms of a new public constructor): /// /// ```compile_fail /// let _ = weksa::split::Checkpoint::for_test(1); diff --git a/crates/weksa/tests/checkpoint_door.rs b/crates/weksa/tests/checkpoint_door.rs index 2de6668..663d8da 100644 --- a/crates/weksa/tests/checkpoint_door.rs +++ b/crates/weksa/tests/checkpoint_door.rs @@ -1,7 +1,13 @@ -//! The Checkpoint door is pinned by source shape, not by a name: no public item -//! of the lib can mint one, and nothing may derive, implement or default one. -//! `Checkpoint::for_test` is `cfg(test)`; a doctest names it today, and this -//! scan catches a rename. +//! A line-based tripwire on the Checkpoint door, not a proof. The real defence +//! is that `Checkpoint::for_test` is `cfg(test)`, pinned by the compile_fail +//! doctest on its current name in `split.rs`. This scan reads `split.rs` line by +//! line and catches the common single-line forms: a `pub fn` in `impl +//! Checkpoint` whose signature line says `-> Self` or `-> Checkpoint` without +//! `cfg(test)` above it, a trait impl for `Checkpoint`, a derive on it, a `pub` +//! field in its braces, and a `-> Checkpoint` line in any other `src` file. It +//! misses: signatures wrapped over several lines, `Option`, `Box`, `Result` or +//! tuple returns from `split.rs`, tuple structs with `pub` fields, and +//! constructors made by macros. /// Why `src` lets code outside the lib make a `Checkpoint`, if it does. fn door_leaks(split_rs: &str) -> Vec { From bb152d02ce8319e03af8f6e557bb6b2619b71103 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 03:10:21 +0000 Subject: [PATCH 036/175] with_scene records what it read: SceneRead says no scene, no addressee, self-addressed, no relation or selected at a revision The view no longer collapses five cases into one empty feature list. Scene::relation returns None for an absent relation. Pins the item speaker over the first participant and V16 without an addressee. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/lib.rs | 2 +- crates/weksa/src/speaker.rs | 13 ++- crates/weksa/src/validate.rs | 86 +++++++++++++++----- crates/weksa/tests/scene.rs | 153 ++++++++++++++++++++++++++--------- 4 files changed, 192 insertions(+), 62 deletions(-) diff --git a/crates/weksa/src/lib.rs b/crates/weksa/src/lib.rs index e147524..f8043a3 100644 --- a/crates/weksa/src/lib.rs +++ b/crates/weksa/src/lib.rs @@ -38,4 +38,4 @@ pub use store::{ AddOutcome, CorpusStore, EntityBound, EntityStore, InventoryStore, PacketStore, SpeakerStateStore, StoreError, StoreRefusal, }; -pub use validate::{LoweringInput, Refusal, validate}; +pub use validate::{LoweringInput, Refusal, SceneRead, validate}; diff --git a/crates/weksa/src/speaker.rs b/crates/weksa/src/speaker.rs index 51367e7..c51a6b1 100644 --- a/crates/weksa/src/speaker.rs +++ b/crates/weksa/src/speaker.rs @@ -83,9 +83,14 @@ pub struct Scene { impl Scene { /// The features of the relation from `speaker` to the named `addressee`, or - /// to `Unnamed` when there is none; empty when the scene holds no such - /// relation. A named addressee the scene lacks never falls back to `Unnamed`. - pub fn relation(&self, speaker: &EntityId, addressee: Option<&EntityId>) -> &[FeatureValue] { + /// to `Unnamed` when there is none; `None` when the scene holds no such + /// relation (a relation with no features is `Some(&[])`). A named + /// addressee the scene lacks never falls back to `Unnamed`. + pub fn relation( + &self, + speaker: &EntityId, + addressee: Option<&EntityId>, + ) -> Option<&[FeatureValue]> { self.relations .iter() .find(|r| { @@ -96,7 +101,7 @@ impl Scene { _ => false, } }) - .map_or(&[], |r| r.features.as_slice()) + .map(|r| r.features.as_slice()) } } diff --git a/crates/weksa/src/validate.rs b/crates/weksa/src/validate.rs index 06600e7..d8989ca 100644 --- a/crates/weksa/src/validate.rs +++ b/crates/weksa/src/validate.rs @@ -499,6 +499,28 @@ fn fits(kind: Option<&ValueKind>, value: &FeatValue) -> bool { } } +/// What `with_scene` found for a view: the scene revision `rev` it read, or why +/// it selected no relation. Each case is distinct so a missing annotation or a +/// store at the wrong path cannot look like a scene that held nothing. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SceneRead { + /// `with_scene` has not run. + Unread, + /// The store holds no scene for the item's group. + NoScene, + /// The scene was read and the packet names no addressee. + NoAddressee { rev: u32 }, + /// The addressee referent is the speaker themself. + SelfAddressed { rev: u32 }, + /// The scene holds no relation from the speaker to this addressee. + NoRelation { rev: u32 }, + /// The relation was found; its features may be none. + Selected { + rev: u32, + features: Vec, + }, +} + /// What the lowerer sees: a validated packet's meaning, with no packet id, no /// provenance (so no corpus item key, encoder or source) and no trace. Its /// fields are private and the constructor is crate-private, called only by @@ -623,7 +645,7 @@ pub struct LoweringInput { /// only source of whose relation `with_scene` selects, and from which scene. speaker: EntityId, group: String, - scene: Vec, + scene: SceneRead, } impl LoweringInput { @@ -642,44 +664,61 @@ impl LoweringInput { stance: p.stance.clone(), speaker: item.speaker.clone(), group: item.group.clone(), - scene: Vec::new(), + scene: SceneRead::Unread, } } - /// Adds the speaker's relation to the addressee, the only path for scene + /// Reads the speaker's relation to the addressee, the only path for scene /// values into a view. The speaker is the item's speaker and the scene is /// the latest the store holds for the item's group; the caller names - /// neither. No scene for the group selects nothing; a scene that does not - /// list the item's speaker is V16. With no addressee nothing is selected; - /// otherwise the relation from the speaker to the addressee referent's - /// entity (to `Unnamed` when the referent has none) is checked like every - /// feature list, in slot `Scene` at this view's inventory revision, and - /// replaces any earlier scene. The English lowerer ignores it. + /// neither. A scene that does not list the item's speaker is V16, with or + /// without an addressee. What the read found is recorded on the view as a + /// [`SceneRead`], never an empty feature list standing for five cases: + /// no scene for the group, no addressee, an addressee who is the speaker, + /// no such relation, or the relation from the speaker to the addressee + /// referent's entity (to `Unnamed` when the referent has none), whose + /// features are checked like every feature list in slot `Scene` at this + /// view's inventory revision. A read replaces any earlier one. The English + /// lowerer ignores it. pub fn with_scene( mut self, inventory: &Inventory, state: &SpeakerStateStore, ) -> Result { - self.scene = Vec::new(); + self.scene = SceneRead::Unread; let Some(scene) = state.scene(&self.group)? else { + self.scene = SceneRead::NoScene; return Ok(self); }; + let rev = scene.n; if !scene.participants.contains(&self.speaker) { return Err(StoreError::Invalid(vec![Refusal::V16SpeakerNotInScene])); } - let selected = self - .discourse - .addressee - .as_ref() - .and_then(|a| self.referents.iter().find(|r| &r.id == a)) - .map_or(&[][..], |r| { - scene.relation(&self.speaker, r.entity.as_ref()) - }); - let out = feature_refusals(inventory, self.inventory_rev, selected, Slot::Scene); + let Some(addressee) = self.discourse.addressee.as_ref() else { + self.scene = SceneRead::NoAddressee { rev }; + return Ok(self); + }; + let entity = self + .referents + .iter() + .find(|r| &r.id == addressee) + .and_then(|r| r.entity.as_ref()); + if entity == Some(&self.speaker) { + self.scene = SceneRead::SelfAddressed { rev }; + return Ok(self); + } + let Some(features) = scene.relation(&self.speaker, entity) else { + self.scene = SceneRead::NoRelation { rev }; + return Ok(self); + }; + let out = feature_refusals(inventory, self.inventory_rev, features, Slot::Scene); if !out.is_empty() { return Err(StoreError::Invalid(out)); } - self.scene = selected.to_vec(); + self.scene = SceneRead::Selected { + rev, + features: features.to_vec(), + }; Ok(self) } @@ -716,6 +755,13 @@ impl LoweringInput { /// The features of the scene relation from the speaker to the addressee; /// empty until `with_scene`, or when the scene holds none. pub fn scene(&self) -> &[FeatureValue] { + match &self.scene { + SceneRead::Selected { features, .. } => features, + _ => &[], + } + } + /// What `with_scene` read; `Unread` until it ran. + pub fn scene_read(&self) -> &SceneRead { &self.scene } } diff --git a/crates/weksa/tests/scene.rs b/crates/weksa/tests/scene.rs index 46885ae..f27bc76 100644 --- a/crates/weksa/tests/scene.rs +++ b/crates/weksa/tests/scene.rs @@ -11,7 +11,7 @@ use weksa::packet::{FeatValue, FeatureValue, Packet, proof_packet}; use weksa::speaker::{Annotator, PairRelation, Party, Scene}; use weksa::{ CorpusStore, EntityStore, Inventory, InventoryStore, LoweringInput, NameSpan, PacketStore, - Refusal, SpeakerStateStore, StoreError, lower, + Refusal, SceneRead, SpeakerStateStore, StoreError, lower, }; struct W { @@ -121,9 +121,15 @@ fn put(w: &mut W, s: Scene) { /// The line of `item:nibu:1` at `rev`, its addressee node `addressee` (`n1` /// has no entity, `n3` is Brokkr), through the packet door. fn view_of(w: &W, rev: u32, addressee: Option<&str>) -> LoweringInput { + view_to(w, rev, addressee, "brokkr") +} + +/// As `view_of`, with referent `n3` naming `entity` (Nibu is the item's +/// speaker, so he may be named on the line; Brokkr is named by its text). +fn view_to(w: &W, rev: u32, addressee: Option<&str>, entity: &str) -> LoweringInput { let mut p: Packet = proof_packet(); p.inventory_rev = rev; - p.referents[2].entity = Some(eid("brokkr")); + p.referents[2].entity = Some(eid(entity)); p.discourse.addressee = addressee.map(nd); // A packet key holds its packet for good, so each view gets a fresh store. let scratch = tempfile::tempdir().unwrap(); @@ -140,14 +146,27 @@ fn rules(r: Result) -> Vec<&'static str> { } } +fn read(w: &W, view: LoweringInput) -> SceneRead { + view.with_scene(&inv(w), &w.state) + .unwrap() + .scene_read() + .clone() +} + +fn selected(rev: u32, features: Vec) -> SceneRead { + SceneRead::Selected { rev, features } +} + #[test] fn a_scene_relation_reaches_the_lowering_view_and_changes_no_english() { let mut w = world(); put(&mut w, standard("g", 1)); let plain = view_of(&w, 3, Some("n3")); + assert_eq!(plain.scene_read(), &SceneRead::Unread); assert!(plain.scene().is_empty()); let with = plain.clone().with_scene(&inv(&w), &w.state).unwrap(); assert_eq!(with.scene(), status().as_slice()); + assert_eq!(with.scene_read(), &selected(1, status())); assert_eq!( lower(&with, &inv(&w), &w.entities).unwrap(), lower(&plain, &inv(&w), &w.entities).unwrap() @@ -155,23 +174,54 @@ fn a_scene_relation_reaches_the_lowering_view_and_changes_no_english() { } #[test] -fn with_scene_selects_by_addressee() { +fn with_scene_says_which_of_five_things_it_found() { let mut w = world(); + // No scene for the group. + assert_eq!(read(&w, view_of(&w, 3, Some("n3"))), SceneRead::NoScene); put(&mut w, standard("g", 1)); - let take = |w: &W, addressee| { - view_of(w, 3, addressee) - .with_scene(&inv(w), &w.state) - .unwrap() - .scene() - .to_vec() - }; // A referent with an entity selects the relation to that entity; a // referent without one, the relation to an unnamed party; and the // speaker's relation, never Brokkr's to Nibu. - assert_eq!(take(&w, Some("n3")), status()); - assert_eq!(take(&w, Some("n1")), intimacy(7)); - // No addressee selects nothing. - assert!(take(&w, None).is_empty()); + assert_eq!(read(&w, view_of(&w, 3, Some("n3"))), selected(1, status())); + assert_eq!( + read(&w, view_of(&w, 3, Some("n1"))), + selected(1, intimacy(7)) + ); + // The scene was read and the packet names no addressee. + assert_eq!( + read(&w, view_of(&w, 3, None)), + SceneRead::NoAddressee { rev: 1 } + ); + // The addressee is the speaker. + assert_eq!( + read(&w, view_to(&w, 3, Some("n3"), "nibu")), + SceneRead::SelfAddressed { rev: 1 } + ); + // The scene holds no relation to this addressee. + put( + &mut w, + scene( + "g", + 2, + &["nibu", "brokkr"], + vec![relation("nibu", Party::Unnamed, intimacy(7))], + ), + ); + assert_eq!( + read(&w, view_of(&w, 3, Some("n3"))), + SceneRead::NoRelation { rev: 2 } + ); + // A relation with no features is found, not absent. + put( + &mut w, + scene( + "g", + 3, + &["nibu", "brokkr"], + vec![relation("nibu", Party::Named(eid("brokkr")), vec![])], + ), + ); + assert_eq!(read(&w, view_of(&w, 3, Some("n3"))), selected(3, vec![])); } #[test] @@ -187,10 +237,10 @@ fn a_named_addressee_the_scene_lacks_never_falls_back_to_unnamed() { vec![relation("nibu", Party::Unnamed, intimacy(7))], ), ); - let got = view_of(&w, 3, Some("n3")) - .with_scene(&inv(&w), &w.state) - .unwrap(); - assert!(got.scene().is_empty()); + assert_eq!( + read(&w, view_of(&w, 3, Some("n3"))), + SceneRead::NoRelation { rev: 1 } + ); // A participant without a relation behaves the same. put( &mut w, @@ -201,10 +251,32 @@ fn a_named_addressee_the_scene_lacks_never_falls_back_to_unnamed() { vec![relation("nibu", Party::Unnamed, intimacy(7))], ), ); - let got = view_of(&w, 3, Some("n3")) - .with_scene(&inv(&w), &w.state) - .unwrap(); - assert!(got.scene().is_empty()); + assert_eq!( + read(&w, view_of(&w, 3, Some("n3"))), + SceneRead::NoRelation { rev: 2 } + ); +} + +#[test] +fn the_speaker_is_the_items_not_the_first_participant() { + let mut w = world(); + // Brokkr is listed first and has an unnamed relation of his own. + put( + &mut w, + scene( + "g", + 1, + &["brokkr", "nibu"], + vec![ + relation("brokkr", Party::Unnamed, intimacy(99)), + relation("nibu", Party::Unnamed, intimacy(7)), + ], + ), + ); + assert_eq!( + read(&w, view_of(&w, 3, Some("n1"))), + selected(1, intimacy(7)) + ); } #[test] @@ -220,11 +292,8 @@ fn the_scene_is_the_items_group_and_its_latest_revision() { vec![relation("nibu", Party::Named(eid("brokkr")), intimacy(1))], ), ); - // No scene for the item's group selects nothing, whatever other groups hold. - let got = view_of(&w, 3, Some("n3")) - .with_scene(&inv(&w), &w.state) - .unwrap(); - assert!(got.scene().is_empty()); + // No scene for the item's group, whatever other groups hold. + assert_eq!(read(&w, view_of(&w, 3, Some("n3"))), SceneRead::NoScene); // Revisions: the highest n is read. put(&mut w, standard("g", 1)); let mut second = standard("g", 2); @@ -233,10 +302,12 @@ fn the_scene_is_the_items_group_and_its_latest_revision() { let got = view_of(&w, 3, Some("n3")) .with_scene(&inv(&w), &w.state) .unwrap(); + assert_eq!(got.scene_read(), &selected(2, intimacy(5))); assert_eq!(got.scene(), intimacy(5).as_slice()); - // A replacement replaces: a store without the scene clears the selection. + // A read replaces the earlier one: a store without the scene says so. let empty = SpeakerStateStore::open(&w.dir.path().join("empty.cc")).unwrap(); let cleared = got.with_scene(&inv(&w), &empty).unwrap(); + assert_eq!(cleared.scene_read(), &SceneRead::NoScene); assert!(cleared.scene().is_empty()); } @@ -262,13 +333,21 @@ fn with_scene_refusals() { &w.entities, ) .unwrap(); - let mut p = proof_packet(); - p.inventory_rev = 3; - p.provenance.corpus_item = weksa::CorpusItemId::parse("item:nibu:3").unwrap(); - p.referents[2].entity = Some(eid("nibu")); - let view = PacketStore::open(&w.dir.path().join("p3.cc")) - .unwrap() - .admit(p, &w.inventory, &w.corpus, &w.entities) - .unwrap(); - assert_eq!(rules(view.with_scene(&inv(&w), &w.state)), ["V16"]); + for addressee in [Some("n3"), None] { + let mut p = proof_packet(); + p.inventory_rev = 3; + p.provenance.corpus_item = weksa::CorpusItemId::parse("item:nibu:3").unwrap(); + p.referents[2].entity = Some(eid("nibu")); + p.discourse.addressee = addressee.map(nd); + let scratch = tempfile::tempdir().unwrap(); + let view = PacketStore::open(&scratch.path().join("p.cc")) + .unwrap() + .admit(p, &w.inventory, &w.corpus, &w.entities) + .unwrap(); + assert_eq!( + rules(view.with_scene(&inv(&w), &w.state)), + ["V16"], + "{addressee:?}" + ); + } } From eff94fb4afdc67d9734cd4137bc1fad2c267c201 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 03:25:44 +0000 Subject: [PATCH 037/175] Pin LoweringInput::of's privacy with the right arity; scene() doc points to scene_read() Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/validate.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/weksa/src/validate.rs b/crates/weksa/src/validate.rs index d8989ca..a74066f 100644 --- a/crates/weksa/src/validate.rs +++ b/crates/weksa/src/validate.rs @@ -609,11 +609,13 @@ pub enum SceneRead { /// call, so the second fails for that call and nothing else. /// /// ``` -/// fn build(p: &weksa::packet::Packet) { let _ = p; } +/// fn build(p: &weksa::packet::Packet, item: &weksa::CorpusItem) { let _ = (p, item); } /// ``` /// /// ```compile_fail -/// fn build(p: &weksa::packet::Packet) { let _ = weksa::LoweringInput::of(p); } +/// fn build(p: &weksa::packet::Packet, item: &weksa::CorpusItem) { +/// let _ = weksa::LoweringInput::of(p, item); +/// } /// ``` /// /// A lowering view is never read from bytes: `Deserialize` would let a caller @@ -752,8 +754,9 @@ impl LoweringInput { pub fn stance(&self) -> &Stance { &self.stance } - /// The features of the scene relation from the speaker to the addressee; - /// empty until `with_scene`, or when the scene holds none. + /// The features of the scene relation from the speaker to the addressee. + /// Empty for every [`SceneRead`] but `Selected` (and for a selected + /// relation that has no features): ask [`scene_read`](Self::scene_read) which. pub fn scene(&self) -> &[FeatureValue] { match &self.scene { SceneRead::Selected { features, .. } => features, From e7b4d9a68b3337c304653d0ad46f775c1c45126d Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 03:45:49 +0000 Subject: [PATCH 038/175] RunStores is the one public producer of TrainItems and EncoderOffer: a run pins its corpus and entity stores by digest TrainItems::from_corpus and EncoderOffer::for_round become pub(crate). RunStores (run.rs) creates a run over three stores, records their canonical paths and FNV-1a 64 digests plus the prompt and lowerer source digests in run.cc, and reopens only from that record, refusing on drift. RunRecord and RoundRecord are published CultCache types. tests/context.rs and the context.rs doctests move to RunStores; the two mutated-clone gate cases become a unit test. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/context.rs | 85 +++- crates/weksa/src/lib.rs | 1 + crates/weksa/src/run.rs | 433 +++++++++++++++++++ crates/weksa/src/schema.rs | 3 + crates/weksa/src/score/mod.rs | 4 +- crates/weksa/src/split.rs | 13 +- crates/weksa/src/store.rs | 2 +- crates/weksa/tests/cli.rs | 2 + crates/weksa/tests/common/mod.rs | 22 + crates/weksa/tests/context.rs | 105 +++-- crates/weksa/tests/golf_loop.rs | 145 +++++++ schemas/cultnet/weksa.loop_round.schema.json | 227 ++++++++++ schemas/cultnet/weksa.loop_run.schema.json | 58 +++ 13 files changed, 1029 insertions(+), 71 deletions(-) create mode 100644 crates/weksa/src/run.rs create mode 100644 crates/weksa/tests/golf_loop.rs create mode 100644 schemas/cultnet/weksa.loop_round.schema.json create mode 100644 schemas/cultnet/weksa.loop_run.schema.json diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index a4f2749..2936d7a 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -3,8 +3,14 @@ //! [`TrainItems`] or by encoding one item an [`EncoderOffer`] holds: Train-only //! minting and per-item encoding (leak-owned-by-flow-and-scorer-3, //! loop-context-builder-2). Instruction text is [`MINT_INSTRUCTIONS`] and -//! [`ENCODE_INSTRUCTIONS`], consts in this module, and nothing else: `mint` and -//! `encode` take none, so no run-time text can become instructions. +//! [`ENCODE_INSTRUCTIONS`], consts in this module; an encode prompt also lists +//! the inventory at the run's revision. +//! +//! Item blocks are data from the run's pinned corpus, reached only through +//! `RunStores`, the one public producer of `TrainItems` and `EncoderOffer` +//! (forged-item-text-in-prompt). A forged item in a scratch store is a different +//! run with a different corpus digest, visible in its record; nothing inside a +//! process its forger controls can stop it. use std::marker::PhantomData; @@ -25,9 +31,12 @@ pub const ENCODE_INSTRUCTIONS: &str = "Encode the line below."; /// item. /// /// ``` -/// let dir = tempfile::tempdir().unwrap(); -/// let store = weksa::CorpusStore::open(&dir.path().join("c.cc")).unwrap(); -/// let t = weksa::context::TrainItems::from_corpus(&store).unwrap(); +/// # let dir = tempfile::tempdir().unwrap(); +/// # let p = |n: &str| dir.path().join(n); +/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } +/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e").unwrap(); +/// let t = run.train().unwrap(); /// assert!(t.is_empty()); /// ``` /// @@ -36,17 +45,23 @@ pub const ENCODE_INSTRUCTIONS: &str = "Encode the line below."; /// ``` /// /// ```compile_fail -/// let dir = tempfile::tempdir().unwrap(); -/// let store = weksa::CorpusStore::open(&dir.path().join("c.cc")).unwrap(); -/// let t = weksa::context::TrainItems::from_corpus(&store).unwrap(); +/// # let dir = tempfile::tempdir().unwrap(); +/// # let p = |n: &str| dir.path().join(n); +/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } +/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e").unwrap(); +/// let t = run.train().unwrap(); /// let _u = t.clone(); /// ``` /// /// ```compile_fail /// fn s(_: &T) {} -/// let dir = tempfile::tempdir().unwrap(); -/// let store = weksa::CorpusStore::open(&dir.path().join("c.cc")).unwrap(); -/// let t = weksa::context::TrainItems::from_corpus(&store).unwrap(); +/// # let dir = tempfile::tempdir().unwrap(); +/// # let p = |n: &str| dir.path().join(n); +/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } +/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e").unwrap(); +/// let t = run.train().unwrap(); /// s(&t); /// ``` pub struct TrainItems { @@ -54,7 +69,7 @@ pub struct TrainItems { } impl TrainItems { - pub fn from_corpus(corpus: &CorpusStore) -> anyhow::Result { + pub(crate) fn from_corpus(corpus: &CorpusStore) -> anyhow::Result { let items = corpus .all()? .into_iter() @@ -134,8 +149,9 @@ pub struct EncoderOffer<'cp> { } impl<'cp> EncoderOffer<'cp> { - /// A round offers Train and Validation items. - pub fn for_round(item: &CorpusItem) -> Result, OfferRefusal> { + /// A round offers Train and Validation items. `RunStores::offer` and + /// `RunStores::offers` are the public producers. + pub(crate) fn for_round(item: &CorpusItem) -> Result, OfferRefusal> { match Self::gate(item)? { Split::Test => Err(OfferRefusal::TestBeforeCheckpoint), _ => Ok(Self::of(item)), @@ -188,9 +204,12 @@ impl<'cp> EncoderOffer<'cp> { /// constructors are [`Prompt::mint`] and [`Prompt::encode`]. /// /// ``` -/// let dir = tempfile::tempdir().unwrap(); -/// let store = weksa::CorpusStore::open(&dir.path().join("c.cc")).unwrap(); -/// let t = weksa::context::TrainItems::from_corpus(&store).unwrap(); +/// # let dir = tempfile::tempdir().unwrap(); +/// # let p = |n: &str| dir.path().join(n); +/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } +/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e").unwrap(); +/// let t = run.train().unwrap(); /// let p = weksa::context::Prompt::mint(&t); /// assert_eq!(p.text(), weksa::context::MINT_INSTRUCTIONS); /// ``` @@ -198,17 +217,23 @@ impl<'cp> EncoderOffer<'cp> { /// Instructions are not a parameter: /// /// ```compile_fail -/// let dir = tempfile::tempdir().unwrap(); -/// let store = weksa::CorpusStore::open(&dir.path().join("c.cc")).unwrap(); -/// let t = weksa::context::TrainItems::from_corpus(&store).unwrap(); +/// # let dir = tempfile::tempdir().unwrap(); +/// # let p = |n: &str| dir.path().join(n); +/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } +/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e").unwrap(); +/// let t = run.train().unwrap(); /// let p = weksa::context::Prompt::mint("x", &t); /// ``` /// /// ```compile_fail /// fn s(_: &T) {} -/// let dir = tempfile::tempdir().unwrap(); -/// let store = weksa::CorpusStore::open(&dir.path().join("c.cc")).unwrap(); -/// let t = weksa::context::TrainItems::from_corpus(&store).unwrap(); +/// # let dir = tempfile::tempdir().unwrap(); +/// # let p = |n: &str| dir.path().join(n); +/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } +/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e").unwrap(); +/// let t = run.train().unwrap(); /// let p = weksa::context::Prompt::mint(&t); /// s(&p); /// ``` @@ -303,6 +328,20 @@ mod tests { assert_eq!(check(&both), Some(OfferRefusal::Withdrawn)); } + #[test] + fn round_offer_gate_precedence() { + let round = |i: &CorpusItem| EncoderOffer::for_round(i).err(); + let mut no_split = item("item:c:2", "m", &group_in(Split::Test), true); + no_split.withdrawn = true; + // Withdrawn wins over NoSplit. + assert_eq!(round(&no_split), Some(OfferRefusal::Withdrawn)); + // A withdrawn Test item is Withdrawn, not TestBeforeCheckpoint. + let mut test = item("item:c:1", "x", &group_in(Split::Test), false); + assert_eq!(round(&test), Some(OfferRefusal::TestBeforeCheckpoint)); + test.withdrawn = true; + assert_eq!(round(&test), Some(OfferRefusal::Withdrawn)); + } + #[test] fn checkpoint_offers_each_item_once() { let mut cp = Checkpoint::for_test(1); diff --git a/crates/weksa/src/lib.rs b/crates/weksa/src/lib.rs index f8043a3..1b8c3c8 100644 --- a/crates/weksa/src/lib.rs +++ b/crates/weksa/src/lib.rs @@ -12,6 +12,7 @@ pub mod inventory; pub mod lower; pub mod packet; pub mod phones; +pub mod run; pub mod schema; pub mod score; pub mod speaker; diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs new file mode 100644 index 0000000..8b892d0 --- /dev/null +++ b/crates/weksa/src/run.rs @@ -0,0 +1,433 @@ +//! One golf-loop run: the stores it pinned at open, the rounds it has recorded, +//! and the one writer of both (`RunStores::create` and, for rounds, `round`). +//! +//! A run pins its corpus and entity store files by canonical path and FNV-1a 64 +//! digest of their bytes. Every round reopens them only from that record and +//! refuses on drift, and every round record carries the digests. Nothing inside +//! a process whose writer controls the code can stop an item faked into a +//! scratch store; what this makes true is that a run over a scratch store is a +//! different run with a different corpus digest, visible in its record, and a +//! later reader compares the rounds of one run only. `RunStores` is the one +//! public producer of [`TrainItems`] and [`EncoderOffer`]: the CLI paths that +//! take caller stores (`validate`, `packet proof|dao`, `lower`) are unchanged. + +use std::fmt; +use std::io::ErrorKind; +use std::path::{Path, PathBuf}; + +use cultcache_rs::{CultCache, DatabaseEntry}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::context::{EncoderOffer, TrainItems}; +use crate::form::{is_attestation, is_slug}; +use crate::score::{CHRF_VERSION, SplitMean}; +use crate::split::{fnv1a64, hex16}; +use crate::store::{CorpusStore, EntityStore, InventoryStore, open_cache}; + +/// The digest of the prompt builder's source. Any edit to `context.rs`, +/// comments included, ends a run; code outside that file is not covered. +pub const PROMPT_SOURCE_DIGEST: u64 = fnv1a64(include_bytes!("context.rs")); + +/// The digest of the lowerer's source. Any edit to `lower.rs`, comments +/// included, ends a run; code outside that file is not covered. +pub const LOWERER_SOURCE_DIGEST: u64 = fnv1a64(include_bytes!("lower.rs")); + +/// What a run pinned at open. Key `run: