diff --git a/.voidbot/state/weksa.cc b/.voidbot/state/weksa.cc index be51ad6..7960180 100644 Binary files a/.voidbot/state/weksa.cc and b/.voidbot/state/weksa.cc differ diff --git a/AGENTS.md b/AGENTS.md index ce540a6..55d488a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -51,3 +51,13 @@ The live state architecture is ported from EpiphanyAgent in reduced form: Project truth does not belong in agent personality memory. If Weksa later grows role dossiers or heartbeat scheduling, those surfaces must remain separate from language-project truth. + +## Building And Verifying + +- `crates/weksa` depends on CultLib's `cultcache-rs` as a git dependency pinned by + `rev`, fetched from GitHub. With the default `CARGO_HOME` that fetch fails on a + Mind's host. Use the pre-fetched home: + `CARGO_HOME=/var/lib/mind-weksa-self/cargo-home cargo test --workspace`. +- `weksa packet proof ` admits the nibu proof packet under the key + `packet:6e6962752d303031` (hex of `nibu-001`), not `nibu-proof-001`. Use that key + with `weksa lower `. diff --git a/crates/weksa/src/corpus.rs b/crates/weksa/src/corpus.rs index 403cdb8..d67cc60 100644 --- a/crates/weksa/src/corpus.rs +++ b/crates/weksa/src/corpus.rs @@ -17,10 +17,13 @@ pub struct Turn { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub enum Source { - /// A line from a cast member's hand-authored fixture. + /// A line from a cast member's hand-authored fixture, with the human who + /// wrote it. A fixture that records no author is `ModelWritten` with family + /// `Unrecorded`, never this (`corpus-labels-tell-truth`). CastAuthored { persona: String, fixture: String, + author: String, }, OperatorWritten, OperatorApproved { diff --git a/crates/weksa/src/entity.rs b/crates/weksa/src/entity.rs index 4c51e84..33d22e1 100644 --- a/crates/weksa/src/entity.rs +++ b/crates/weksa/src/entity.rs @@ -13,19 +13,21 @@ use crate::packet::EntityId; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub struct LanguageForms { pub language: String, - /// 1 to 64 bytes each, no control character, no edge whitespace, none repeated. + /// 1 to 64 bytes each, no control or invisible format character, no edge + /// whitespace, none repeated. pub forms: Vec, } /// Where an entity was minted. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub enum EntityOrigin { - /// From a cast member's fixture. - Cast { persona: String }, + /// From a cast member's fixture: the persona and the fixture path the + /// entity came from. + Cast { persona: String, fixture: String }, /// From a public-domain work. PublicDomain { work: String }, - /// Minted live by a persona before it names the entity. - Live, + /// Minted live by the named persona before it names the entity. + Live { persona: String }, } /// Key: the entity id. Forms only grow: see `EntityStore::put`. diff --git a/crates/weksa/src/form.rs b/crates/weksa/src/form.rs index d8336fb..9de8eb8 100644 --- a/crates/weksa/src/form.rs +++ b/crates/weksa/src/form.rs @@ -86,3 +86,114 @@ pub(crate) fn is_slug(s: &str) -> bool { && b[0] != b'-' && b[b.len() - 1] != b'-' } + +/// A General_Category Cf code point, or the line and paragraph separators +/// U+2028 and U+2029, other than the joiners ZWNJ U+200C and ZWJ U+200D. The +/// ranges are Unicode 15.1 Cf; `tests/format_refused.txt` is the full list from +/// Python's `unicodedata` that the table is swept against. +pub(crate) fn is_format(c: char) -> bool { + matches!( + c, + '\u{ad}' + | '\u{600}'..='\u{605}' + | '\u{61c}' + | '\u{6dd}' + | '\u{70f}' + | '\u{890}'..='\u{891}' + | '\u{8e2}' + | '\u{180e}' + | '\u{200b}' + | '\u{200e}'..='\u{200f}' + | '\u{2028}'..='\u{202e}' + | '\u{2060}'..='\u{2064}' + | '\u{2066}'..='\u{206f}' + | '\u{feff}' + | '\u{fff9}'..='\u{fffb}' + | '\u{110bd}' + | '\u{110cd}' + | '\u{13430}'..='\u{1343f}' + | '\u{1bca0}'..='\u{1bca3}' + | '\u{1d173}'..='\u{1d17a}' + | '\u{e0001}' + | '\u{e0020}'..='\u{e007f}' + ) +} + +/// ZWNJ and ZWJ: Cf, but scripts and emoji sequences need them inside a form. +pub(crate) fn is_joiner(c: char) -> bool { + matches!(c, '\u{200c}' | '\u{200d}') +} + +/// A character a name or form may not hold: every `is_format` code point, and +/// the invisible fillers that are Default_Ignorable without being Cf (Hangul +/// U+115F, U+1160, U+3164, U+FFA0 and Khmer U+17B4, U+17B5). Joiners, +/// variation selectors, CGJ and the Mongolian selectors are not refused. +pub(crate) fn is_refused_format(c: char) -> bool { + is_format(c) + || matches!( + c, + '\u{115f}' | '\u{1160}' | '\u{17b4}' | '\u{17b5}' | '\u{3164}' | '\u{ffa0}' + ) +} + +/// A joiner, a variation selector (U+FE00..U+FE0F, U+E0100..U+E01EF), CGJ +/// U+034F or a Mongolian free variation selector (U+180B..U+180D, U+180F): it +/// shapes a neighbour and shows nothing alone. +pub(crate) fn is_joining_only(c: char) -> bool { + is_joiner(c) + || matches!( + c, + '\u{34f}' + | '\u{180b}'..='\u{180d}' + | '\u{180f}' + | '\u{fe00}'..='\u{fe0f}' + | '\u{e0100}'..='\u{e01ef}' + ) +} + +/// An attestation string (author, persona, fixture, note, work, url, generator, +/// digest): at most 200 bytes, at least one alphanumeric, no control, format or +/// joiner character. Whitespace between words is fine; it attests something. +pub(crate) fn is_attestation(s: &str) -> bool { + s.len() <= 200 + && s.chars().any(char::is_alphanumeric) + && !s + .chars() + .any(|c| c.is_control() || is_refused_format(c) || is_joiner(c)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The rule against every code point: `is_format` is exactly the listed + /// Cf-minus-joiners-plus-U+2028/9 set, so each range has both of its + /// edges and the gaps between ranges pinned. + #[test] + fn format_rule_matches_the_unicode_list_at_every_code_point() { + let listed: std::collections::BTreeSet = include_str!("../tests/format_refused.txt") + .lines() + .filter(|l| !l.starts_with('#')) + .map(|l| u32::from_str_radix(l, 16).unwrap()) + .collect(); + for code in 0..=0x10ffff_u32 { + if let Some(c) = char::from_u32(code) { + assert_eq!(is_format(c), listed.contains(&code), "U+{code:X}"); + } + } + assert!(!is_format('\u{200c}') && !is_format('\u{200d}')); + } + + #[test] + fn an_attestation_is_bounded_readable_text() { + assert!(is_attestation("a")); + assert!(is_attestation(&"x".repeat(200))); + assert!(!is_attestation(&"x".repeat(201))); + assert!(is_attestation("two words, 2 more")); + for bad in [ + "", " ", "\t", "\u{200b}", "\u{200d}", "\0", "--", "a\nb", "a\u{ad}b", + ] { + assert!(!is_attestation(bad), "{bad:?}"); + } + } +} diff --git a/crates/weksa/src/inventory.rs b/crates/weksa/src/inventory.rs index 792cad4..22538dd 100644 --- a/crates/weksa/src/inventory.rs +++ b/crates/weksa/src/inventory.rs @@ -473,3 +473,38 @@ pub fn seed() -> (InventoryRevision, Vec) { entries, ) } + +/// Revision 2: the concepts and frames the `nibu-proof-001` packet names, which +/// the seed leaves open. Hand-seeded like [`seed`] until the golf loop's +/// minting round owns concepts and frames. +pub fn proof_revision() -> (InventoryRevision, Vec) { + let mut entries: Vec = Vec::new(); + for (family, glosses) in [ + ( + Family::Concept, + ["endanger", "prevent", "player", "control", "ship_mind"].as_slice(), + ), + (Family::Frame, ["hazard_warning", "intervention"].as_slice()), + ] { + for (i, gloss) in glosses.iter().enumerate() { + entries.push(InventoryEntry { + id: family.id(NonZeroU32::MIN.saturating_add(i as u32)), + gloss: gloss.to_string(), + value_kind: None, + applies_to: vec![], + introduced_rev: 2, + retired_rev: None, + }); + } + } + let changes = entries.iter().map(|e| Change::Add(e.id)).collect(); + ( + InventoryRevision { + n: 2, + parent: Some(1), + changes, + frozen: false, + }, + entries, + ) +} diff --git a/crates/weksa/src/lib.rs b/crates/weksa/src/lib.rs index 479366b..0bfdb8c 100644 --- a/crates/weksa/src/lib.rs +++ b/crates/weksa/src/lib.rs @@ -7,6 +7,7 @@ pub mod corpus; pub mod entity; pub mod form; pub mod inventory; +pub mod lower; pub mod packet; pub mod schema; pub mod score; @@ -21,6 +22,7 @@ pub use inventory::{ Change, Family, InvId, Inventory, InventoryEntry, InventoryRevision, OptionLabel, OptionSet, Slot, ValueKind, }; +pub use lower::{LowerError, lower}; pub use packet::{ EncoderId, EntityId, Grade, NodeId, Packet, PacketId, PacketKind, SCHEMA_VERSION, Stance, }; diff --git a/crates/weksa/src/lower.rs b/crates/weksa/src/lower.rs new file mode 100644 index 0000000..d6fcdfb --- /dev/null +++ b/crates/weksa/src/lower.rs @@ -0,0 +1,205 @@ +//! The English lowerer: the validated lowering view of a packet to text, by a +//! fixed table keyed by inventory ids. It renders predications, referents and the +//! speech act, and nothing else: no style, no model, no corpus text. It takes a +//! [`LoweringInput`], so it cannot see the trace or the corpus item. A packet +//! carrying a feature, relation, extension, role fill or referent it does not +//! render is refused, because rendering part of a meaning would assert a different +//! one (`polarity: negative` dropped says the opposite). Context, kind and +//! constraints steer delivery and are not rendered; the concept of a predication +//! must resolve but the frame owns the verb. +//! +//! The table is a temporary owner. Each row speaks for an inventory entry and +//! names the gloss it speaks for, so a row cannot outlive a change of what its id +//! means. `language-ontology` moves the rows into ontology documents. + +use std::fmt; + +use crate::inventory::{Family, InvId, Inventory}; +use crate::packet::{NodeId, Predication}; +use crate::validate::LoweringInput; + +/// `(inventory id, the gloss the row speaks for, surface)`. +type Row = (&'static str, &'static str, &'static str); + +/// Referent concepts, as the noun phrase after "the". +const CONCEPTS: &[Row] = &[ + ("concept:3", "player", "player"), + ("concept:4", "control", "control"), + ("concept:5", "ship_mind", "ship mind"), +]; + +/// Frames: the clause a predication of that frame says. `{gloss}` is the filler +/// of the role whose inventory gloss it is. +const FRAMES: &[Row] = &[ + ( + "frame:1", + "hazard_warning", + "{agent} may endanger {patient} by touching {instrument}", + ), + ( + "frame:2", + "intervention", + "{agent} will prevent {patient} to protect {beneficiary}", + ), +]; + +/// Speech acts: the opener of the first sentence. +const SPEECH_ACTS: &[Row] = &[ + ("speech_act:1", "assert", ""), + ("speech_act:5", "warn", "Warning: "), +]; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum LowerError { + /// The packet carries a slot the lowerer does not render. + Unrendered { what: &'static str }, + /// No table row for an id the packet uses. + NoRow { id: InvId }, + /// The row's gloss is not the gloss the inventory gives its id at the + /// packet's revision, or the id does not resolve there. + GlossMismatch { id: InvId, row: &'static str }, + /// A frame template names a role the inventory has no live entry for. + UnknownRole { frame: InvId, role: String }, + /// The predication fills no role its frame's template needs. + MissingRole { predication: NodeId, role: String }, +} + +impl fmt::Display for LowerError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{self:?}") + } +} + +impl std::error::Error for LowerError {} + +fn row( + table: &'static [Row], + id: InvId, + inv: &Inventory, + rev: u32, +) -> Result<&'static str, LowerError> { + let (_, gloss, surface) = table + .iter() + .find(|(key, _, _)| *key == id.to_string()) + .ok_or(LowerError::NoRow { id })?; + match inv.resolve(&id, rev) { + Some(e) if e.gloss == *gloss => Ok(surface), + _ => Err(LowerError::GlossMismatch { id, row: gloss }), + } +} + +pub fn lower(input: &LoweringInput, inv: &Inventory) -> Result { + let rev = input.inventory_rev(); + let unrendered = |what| Err(LowerError::Unrendered { what }); + if input.predications().is_empty() { + return unrendered("no predication"); + } + if !input.discourse().features.is_empty() + || input.referents().iter().any(|r| !r.features.is_empty()) + || input.predications().iter().any(|p| !p.features.is_empty()) + { + return unrendered("features"); + } + if !input.relations().is_empty() { + return unrendered("relations"); + } + if !input.extensions().is_empty() { + return unrendered("extensions"); + } + let opener = match input.discourse().speech_act { + Some(act) => row(SPEECH_ACTS, act, inv, rev)?, + None => "", + }; + let mut sentences = Vec::new(); + for p in input.predications() { + sentences.push(clause(input, inv, p)?); + } + let mentioned = |r: &NodeId| { + input + .predications() + .iter() + .any(|p| p.roles.iter().any(|f| &f.filler == r)) + }; + if input.referents().iter().any(|r| !mentioned(&r.id)) { + return unrendered("a referent no role mentions"); + } + let mut out = String::new(); + for (i, clause) in sentences.iter().enumerate() { + if i > 0 { + out.push(' '); + } + if i == 0 { + out.push_str(opener); + } + let mut chars = clause.chars(); + if let Some(first) = chars.next() { + if i == 0 && !opener.is_empty() { + out.push(first); + } else { + out.extend(first.to_uppercase()); + } + out.push_str(chars.as_str()); + } + out.push('.'); + } + Ok(out) +} + +fn clause(input: &LoweringInput, inv: &Inventory, p: &Predication) -> Result { + let rev = input.inventory_rev(); + let template = row(FRAMES, p.frame, inv, rev)?; + let mut said: Vec<&NodeId> = Vec::new(); + let mut out = String::new(); + let mut rest = template; + while let Some(open) = rest.find('{') { + out.push_str(&rest[..open]); + let close = rest[open..].find('}').map_or(rest.len(), |c| open + c); + let name = &rest[open + 1..close]; + rest = &rest[(close + 1).min(rest.len())..]; + let role = inv + .entries() + .find(|e| e.family() == Family::Role && e.gloss == name && e.live_at(rev)) + .ok_or_else(|| LowerError::UnknownRole { + frame: p.frame, + role: name.to_string(), + })?; + let fill = + p.roles + .iter() + .find(|r| r.role == role.id) + .ok_or_else(|| LowerError::MissingRole { + predication: p.id.clone(), + role: name.to_string(), + })?; + out.push_str(&filler(input, inv, &fill.filler, &said)?); + said.push(&fill.filler); + } + out.push_str(rest); + if p.roles.len() != said.len() { + return Err(LowerError::Unrendered { + what: "a role the frame does not name", + }); + } + Ok(out) +} + +/// A role filler as a phrase: the addressee is "you", a predication is "that", +/// another referent is "the" and its concept word. The addressee said twice in +/// one predication is reflexive; no other referent is, because the packet has no +/// animacy to choose "themself" or "itself" by. +fn filler( + input: &LoweringInput, + inv: &Inventory, + node: &NodeId, + said: &[&NodeId], +) -> Result { + let again = said.contains(&node); + if let Some(r) = input.referents().iter().find(|r| &r.id == node) { + if input.discourse().addressee.as_ref() == Some(node) { + return Ok(if again { "yourself" } else { "you" }.to_string()); + } + let word = row(CONCEPTS, r.concept, inv, input.inventory_rev())?; + return Ok(format!("the {word}")); + } + Ok("that".to_string()) +} diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 48fce8f..f36ba27 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -8,10 +8,12 @@ use clap::{Args, Parser, Subcommand, ValueEnum}; use unicode_normalization::UnicodeNormalization; use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn}; use weksa::entity::{Entity, EntityOrigin, LanguageForms}; -use weksa::inventory::seed; +use weksa::inventory::{proof_revision, seed}; +use weksa::packet::proof_packet; +use weksa::score::ScoreInput; use weksa::{ - CorpusStore, EntityId, EntityStore, InventoryStore, PacketStore, SplitScheme, StoreError, - validate, + CorpusStore, EntityId, EntityStore, InventoryStore, OllamaEmbedder, PacketStore, Scorer, + SplitScheme, StoreError, lower, validate, }; #[derive(Parser)] @@ -43,6 +45,12 @@ enum Command { Inventory(InventoryCommand), #[command(subcommand)] Entity(EntityCommand), + #[command(subcommand)] + Packet(PacketCommand), + /// Lower a stored packet to English text. + Lower { store: PathBuf, key: String }, + /// Score output text (stdin, or --output) against corpus items, one embed call. + Score(ScoreArgs), /// Print the split of a corpus item. Split { item_key: String }, /// Write the published JSON Schemas. @@ -52,6 +60,26 @@ enum Command { }, } +#[derive(Subcommand)] +enum PacketCommand { + /// Admit the nibu-proof-001 packet through `PacketStore::admit`. + Proof { store: PathBuf }, +} + +#[derive(Args)] +struct ScoreArgs { + /// Corpus item keys, the references. + #[arg(required = true)] + items: Vec, + /// The text to score; stdin when absent. + #[arg(long)] + output: Option, + #[arg(long, default_value = "http://127.0.0.1:11434")] + ollama_url: String, + #[arg(long, default_value = "qwen3-embedding:0.6b")] + model: String, +} + #[derive(Subcommand)] enum CorpusCommand { Add(Box), @@ -68,6 +96,8 @@ enum InventoryCommand { Dump, /// Admit revision 1, the standard's enumerated values and the nuance features. Seed, + /// Admit revision 2, the concepts and frames the nibu-proof-001 packet names. + Proof, } #[derive(Subcommand)] @@ -88,9 +118,12 @@ struct PutArgs { forms: Vec, #[arg(long, value_enum)] origin: OriginKind, - /// The persona of a `cast` origin; refused with any other origin. + /// The persona of a `cast` or `live` origin; refused with any other origin. #[arg(long)] persona: Option, + /// The fixture path a `cast` entity came from; refused with any other origin. + #[arg(long)] + fixture: Option, /// The work of a `public-domain` origin; refused with any other origin. #[arg(long)] work: Option, @@ -107,9 +140,7 @@ impl PutArgs { fn entity(self) -> Result { let mut forms: Vec = Vec::new(); for f in &self.forms { - let (language, form) = f - .split_once('=') - .with_context(|| format!("--form {f:?} is not language=form"))?; + let (language, form) = f.split_once('=').context("--form is not language=form")?; match forms.iter_mut().find(|lf| lf.language == language) { Some(lf) => lf.forms.push(form.nfc().collect()), None => forms.push(LanguageForms { @@ -118,16 +149,19 @@ impl PutArgs { }), } } - let origin = match (self.origin, self.persona, self.work) { - (OriginKind::Cast, persona, None) => EntityOrigin::Cast { + let origin = match (self.origin, self.persona, self.fixture, self.work) { + (OriginKind::Cast, persona, fixture, None) => EntityOrigin::Cast { persona: need(persona, "persona")?, + fixture: need(fixture, "fixture")?, }, - (OriginKind::PublicDomain, None, work) => EntityOrigin::PublicDomain { + (OriginKind::PublicDomain, None, None, work) => EntityOrigin::PublicDomain { work: need(work, "work")?, }, - (OriginKind::Live, None, None) => EntityOrigin::Live, + (OriginKind::Live, persona, None, None) => EntityOrigin::Live { + persona: need(persona, "persona")?, + }, _ => bail!( - "--persona is only for --origin cast and --work only for --origin public-domain" + "--fixture is only for --origin cast, --persona only for --origin cast or live and --work only for --origin public-domain" ), }; Ok(Entity { @@ -202,6 +236,7 @@ struct AddArgs { note: Option, #[arg(long)] work: Option, + /// Who wrote the line: for `cast-authored` and `public-domain`. #[arg(long)] author: Option, #[arg(long)] @@ -226,10 +261,37 @@ fn need(value: Option, flag: &str) -> Result { impl AddArgs { fn source(&self) -> Result { let get = |v: &Option, flag: &str| need(v.clone(), flag); + let applies: &[&str] = match self.source { + SourceKind::CastAuthored => &["persona", "fixture", "author"], + SourceKind::OperatorWritten => &[], + SourceKind::OperatorApproved => &["note"], + SourceKind::PublicDomain => &["work", "author", "edition-url"], + SourceKind::ModelWritten => &["model-family", "generator", "prompt-digest"], + }; + let given = [ + ("persona", self.persona.is_some()), + ("fixture", self.fixture.is_some()), + ("author", self.author.is_some()), + ("note", self.note.is_some()), + ("work", self.work.is_some()), + ("edition-url", self.edition_url.is_some()), + ("model-family", self.family.is_some()), + ("generator", self.generator.is_some()), + ("prompt-digest", self.prompt_digest.is_some()), + ]; + let stray: Vec = given + .into_iter() + .filter(|(flag, given)| *given && !applies.contains(flag)) + .map(|(flag, _)| format!("--{flag}")) + .collect(); + if !stray.is_empty() { + bail!("{} does not apply to this --source", stray.join(", ")); + } Ok(match self.source { SourceKind::CastAuthored => Source::CastAuthored { persona: get(&self.persona, "persona")?, fixture: get(&self.fixture, "fixture")?, + author: get(&self.author, "author")?, }, SourceKind::OperatorWritten => Source::OperatorWritten, SourceKind::OperatorApproved => Source::OperatorApproved { @@ -258,9 +320,11 @@ impl AddArgs { .map(|t| { let (speaker, text) = t .split_once('=') - .with_context(|| format!("--prior-turn {t:?} is not speaker=text"))?; + .context("--prior-turn is not speaker=text")?; Ok(Turn { - speaker: speaker.parse()?, + speaker: speaker.parse().map_err(|_| { + anyhow::anyhow!("--prior-turn speaker is not entity:") + })?, text: text.nfc().collect(), }) }) @@ -269,13 +333,13 @@ impl AddArgs { .names .iter() .map(|n| { - let bad = || format!("--name {n:?} is not start..end=entity:"); - let (range, entity) = n.split_once('=').with_context(bad)?; - let (start, end) = range.split_once("..").with_context(bad)?; + let bad = || anyhow::anyhow!("--name is not start..end=entity:"); + let (range, entity) = n.split_once('=').ok_or_else(bad)?; + let (start, end) = range.split_once("..").ok_or_else(bad)?; Ok(NameSpan { - entity: entity.parse()?, - start: start.parse().with_context(bad)?, - end: end.parse().with_context(bad)?, + entity: entity.parse().map_err(|_| bad())?, + start: start.parse().map_err(|_| bad())?, + end: end.parse().map_err(|_| bad())?, }) }) .collect::>>()?; @@ -338,7 +402,7 @@ fn main() -> Result<()> { } Command::Corpus(CorpusCommand::Add(args)) => { let mut store = CorpusStore::open(&cli.corpus)?; - let key = store.next_key(&args.slug)?; + let key = store.next_key(&args.slug).map_err(refused)?; let entities = EntityStore::open(&cli.entities)?; let outcome = store .add((*args).item(key.clone())?, &entities) @@ -400,6 +464,77 @@ fn main() -> Result<()> { .map_err(refused)?; println!("seeded rev:1 with {count} entries"); } + Command::Inventory(InventoryCommand::Proof) => { + let (rev, entries) = proof_revision(); + let count = entries.len(); + InventoryStore::open(&cli.inventory)? + .admit_revision(rev, entries) + .map_err(refused)?; + println!("admitted rev:2 with {count} entries"); + } + Command::Packet(PacketCommand::Proof { store }) => { + let packet = proof_packet(); + let key = packet.id.to_string(); + PacketStore::open(&store)? + .admit( + packet, + &InventoryStore::open(&cli.inventory)?, + &CorpusStore::open(&cli.corpus)?, + &EntityStore::open(&cli.entities)?, + ) + .map_err(refused)?; + println!("admitted {key}"); + } + Command::Lower { store, key } => { + let inventory_store = InventoryStore::open(&cli.inventory)?; + let inventory = inventory_store.inventory()?; + let mut packets = PacketStore::open(&store)?; + let packet = packets + .get(&key)? + .with_context(|| format!("no packet {key}"))?; + // Admission is the only source of a LoweringInput; re-admitting the + // stored packet is a no-op write that re-runs every door check. + let view = packets + .admit( + packet, + &inventory_store, + &CorpusStore::open(&cli.corpus)?, + &EntityStore::open(&cli.entities)?, + ) + .map_err(refused)?; + println!("{}", lower(&view, &inventory)?); + } + Command::Score(args) => { + let output = match &args.output { + Some(path) => std::fs::read_to_string(path)?, + None => std::io::read_to_string(std::io::stdin())?, + }; + let corpus = CorpusStore::open(&cli.corpus)?; + let references = args + .items + .iter() + .map(|key| { + Ok(corpus + .get(key)? + .with_context(|| format!("no item {key}"))? + .text) + }) + .collect::>>()?; + let inputs: Vec = references + .iter() + .map(|reference| ScoreInput { + reference, + output: output.trim(), + }) + .collect(); + let scorer = Scorer::new(OllamaEmbedder::new(&args.ollama_url, &args.model)); + for (key, score) in args.items.iter().zip(scorer.score(&inputs)?) { + match score.meaning { + Some(m) => println!("{key} chrf={:.2} meaning={m:.4}", score.chrf), + None => println!("{key} chrf={:.2} meaning=none", score.chrf), + } + } + } Command::Split { item_key } => { let item = CorpusStore::open(&cli.corpus)? .get(&item_key)? diff --git a/crates/weksa/src/packet.rs b/crates/weksa/src/packet.rs index 73199a2..cae001b 100644 --- a/crates/weksa/src/packet.rs +++ b/crates/weksa/src/packet.rs @@ -347,3 +347,86 @@ pub struct Trace { pub notes: Vec, pub rejected_readings: Vec, } + +/// The `nibu-proof-001` packet: the meaning of `examples/english-lowering/nibu-proof-001/interlingua.yaml` +/// (warn the player off the damaged control; Nibu prevents it for them) as typed +/// ids of inventory revision 2, for the corpus item `item:nibu:1`. No features: +/// nothing here is style. +pub fn proof_packet() -> Packet { + fn id(s: &str) -> InvId { + InvId::parse(s).expect("proof ids are well formed") + } + fn node(s: &str) -> NodeId { + NodeId::parse(s).expect("proof nodes are well formed") + } + let referent = |n: &str, kind: &str, concept: &str, entity: Option<&str>| Referent { + id: node(n), + kind: id(kind), + concept: id(concept), + entity: entity.map(|e| EntityId::parse(e).expect("proof entity is well formed")), + features: vec![], + }; + let fill = |role: &str, filler: &str| RoleFill { + role: id(role), + filler: node(filler), + }; + Packet { + id: PacketId::parse("packet:6e6962752d303031").expect("proof id"), + schema_version: SCHEMA_VERSION, + inventory_rev: 2, + kind: PacketKind::DialogueLine, + provenance: Provenance { + source: ProvenanceSource::Authored, + corpus_item: CorpusItemId::parse("item:nibu:1").expect("proof item"), + encoder: EncoderId::parse("encoder:6f70657261746f72").expect("proof encoder"), + confidence: Grade::new(900).expect("proof grade"), + }, + context: Context::default(), + discourse: Discourse { + speech_act: Some(id("speech_act:5")), + addressee: Some(node("n1")), + features: vec![], + }, + referents: vec![ + referent("n1", "referent_kind:1", "concept:3", None), + referent("n2", "referent_kind:5", "concept:4", None), + referent("n3", "referent_kind:1", "concept:5", Some("entity:nibu")), + ], + predications: vec![ + Predication { + id: node("n4"), + concept: id("concept:1"), + frame: id("frame:1"), + roles: vec![ + fill("role:1", "n1"), + fill("role:4", "n2"), + fill("role:2", "n1"), + ], + features: vec![], + }, + Predication { + id: node("n5"), + concept: id("concept:2"), + frame: id("frame:2"), + roles: vec![ + fill("role:1", "n3"), + fill("role:2", "n4"), + fill("role:8", "n1"), + ], + features: vec![], + }, + ], + relations: vec![], + constraints: Constraints { + preserve: vec![node("n1"), node("n2")], + deterministic: true, + max_latency_ms: None, + }, + extensions: vec![], + trace: Trace { + notes: vec!["First proof packet for Nibu flavored-English lowering.".to_string()], + rejected_readings: vec![], + }, + stance: Stance::default(), + } +} diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index abf66fc..1caa6ac 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -6,13 +6,14 @@ use std::collections::BTreeSet; use std::fmt; use std::path::Path; +use caseless::default_case_fold_str; use cultcache_rs::{CultCache, SingleFileMessagePackBackingStore}; -use unicode_normalization::is_nfc; +use unicode_normalization::{UnicodeNormalization, char::is_combining_mark, is_nfc}; use unicode_segmentation::UnicodeSegmentation; -use crate::corpus::{CorpusItem, CorpusItemId, item_key, key_slug}; -use crate::entity::Entity; -use crate::form::is_language_tag; +use crate::corpus::{CorpusItem, CorpusItemId, Source, item_key, key_slug}; +use crate::entity::{Entity, EntityOrigin}; +use crate::form::{is_attestation, is_joining_only, is_language_tag, is_refused_format}; use crate::inventory::{Change, InvId, Inventory, InventoryEntry, InventoryRevision, revision_key}; use crate::packet::{EntityId, Packet}; use crate::validate::{LoweringInput, Refusal, admissible}; @@ -91,15 +92,12 @@ pub enum EntityBound { Forms, /// A form of more than 64 bytes. FormBytes, - /// A form of more than 4 words. - FormWords, } impl EntityBound { pub const MAX_LANGUAGES: usize = 8; pub const MAX_FORMS: usize = 8; pub const MAX_FORM_BYTES: usize = 64; - pub const MAX_FORM_WORDS: usize = 4; } #[derive(Debug)] @@ -188,7 +186,7 @@ impl CorpusStore { } /// The next free key for `slug`; withdrawn items keep their numbers. - pub fn next_key(&self, slug: &str) -> anyhow::Result { + pub fn next_key(&self, slug: &str) -> Result { let highest = self .all()? .iter() @@ -196,7 +194,10 @@ impl CorpusStore { .filter_map(|i| i.id.rsplit_once(':')?.1.parse::().ok()) .max() .unwrap_or(0); - Ok(item_key(slug, highest + 1)) + let next = highest + .checked_add(1) + .ok_or_else(|| malformed(slug, "slug has no free key"))?; + Ok(item_key(slug, next)) } /// The item `id` names, or `UnknownCorpusItem` when it holds nothing. @@ -239,8 +240,14 @@ impl CorpusStore { if !is_nfc(&item.text) || item.prior_turns.iter().any(|t| !is_nfc(&t.text)) { return Err(StoreRefusal::NotNfc { key: item.id }.into()); } - check_entities(&item, entities)?; let text = normalized(&item.text); + if !text.chars().any(char::is_alphanumeric) { + return Err(malformed(&item.id, "text has no alphanumeric character")); + } + if let Some(why) = empty_attestation(&item.source) { + return Err(malformed(&item.id, why)); + } + check_entities(&item, entities)?; for held in self.all()? { if normalized(&held.text) == text && item.supersedes.as_deref() != Some(&held.id) { return Err(StoreRefusal::S6TextHeld { @@ -353,15 +360,58 @@ fn check_entities(item: &CorpusItem, entities: &EntityStore) -> Result<(), Store Ok(()) } -/// Lowercased alphanumerics only, so case, spacing and punctuation variants of -/// a text compare equal. +/// The S6 key of a text: Unicode compatibility caseless matching (D146: NFKC, +/// default case fold, NFKC) keeping letters, digits and combining marks +/// (General_Category L, N, M) so tone marks and viramas keep words apart. Composed and decomposed, +/// fullwidth, ligature and case-fold variants (straße, STRASSE) and spacing and +/// punctuation variants compare equal. Homoglyphs across scripts (Cyrillic a for +/// Latin a) stay apart: no key here reads what a character looks like. fn normalized(text: &str) -> String { - text.chars() - .flat_map(char::to_lowercase) - .filter(|c| c.is_alphanumeric()) + let folded = default_case_fold_str(&text.nfkc().collect::()); + folded + .nfkc() + .filter(|c| c.is_alphanumeric() || is_combining_mark(*c)) .collect() } +/// The field that holds an unattested provenance string, if any: a string with +/// no alphanumeric, a control or format character, or over 200 bytes attests +/// nothing. +fn empty_attestation(source: &Source) -> Option<&'static str> { + let empty = |s: &String| !is_attestation(s); + let empty_some = |s: &Option| s.as_ref().is_some_and(empty); + match source { + Source::CastAuthored { persona, .. } if empty(persona) => { + Some("a cast-authored source names its persona") + } + Source::CastAuthored { fixture, .. } if empty(fixture) => { + Some("a cast-authored source names its fixture") + } + Source::CastAuthored { author, .. } if empty(author) => { + Some("a cast-authored source names its author") + } + Source::OperatorApproved { note } if empty(note) => { + Some("an operator-approved source carries its note") + } + Source::PublicDomain { work, .. } if empty(work) => { + Some("a public-domain source names its work") + } + Source::PublicDomain { author, .. } if empty(author) => { + Some("a public-domain source names its author") + } + Source::PublicDomain { edition_url, .. } if empty(edition_url) => { + Some("a public-domain source names its edition url") + } + Source::ModelWritten { generator, .. } if empty_some(generator) => { + Some("a model-written source names a non-empty generator") + } + Source::ModelWritten { prompt_digest, .. } if empty_some(prompt_digest) => { + Some("a model-written source carries a non-empty prompt digest") + } + _ => None, + } +} + pub struct EntityStore { cache: CultCache, } @@ -387,8 +437,9 @@ impl EntityStore { /// every stored language and form in order and origin as it was, and only /// appends: forms grow, never shrink, reorder or change. An entity has 1 to /// 8 languages, each a BCP 47-form tag named once, with 1 to 8 NFC forms of - /// 1 to 64 bytes and at most 4 words, no control character, no edge - /// whitespace, none repeated. + /// 1 to 64 bytes, no control or invisible format character (joiners and + /// variation selectors stay), no edge whitespace, none repeated; the origin + /// attests where it came from with strings that hold an alphanumeric, no control or format character and at most 200 bytes. pub fn put(&mut self, entity: Entity) -> Result { let key = entity.id.to_string(); check_entity_shape(&entity)?; @@ -421,6 +472,20 @@ fn check_entity_shape(entity: &Entity) -> Result<(), StoreError> { } .into() }; + let (persona, fixture, work) = match &entity.origin { + EntityOrigin::Cast { persona, fixture } => (Some(persona), Some(fixture), None), + EntityOrigin::Live { persona } => (Some(persona), None, None), + EntityOrigin::PublicDomain { work } => (None, None, Some(work)), + }; + if persona.is_some_and(|s| !is_attestation(s)) { + return Err(bad("an origin names its persona")); + } + if fixture.is_some_and(|s| !is_attestation(s)) { + return Err(bad("a cast origin names its fixture")); + } + if work.is_some_and(|s| !is_attestation(s)) { + return Err(bad("a public-domain origin names its work")); + } if entity.forms.is_empty() { return Err(bad("an entity has forms in at least one language")); } @@ -446,9 +511,6 @@ fn check_entity_shape(entity: &Entity) -> Result<(), StoreError> { if f.len() > EntityBound::MAX_FORM_BYTES { return Err(bound(EntityBound::FormBytes)); } - if f.split_whitespace().count() > EntityBound::MAX_FORM_WORDS { - return Err(bound(EntityBound::FormWords)); - } if !is_nfc(f) { return Err(StoreRefusal::NotNfc { key: entity.id.to_string(), @@ -460,6 +522,12 @@ fn check_entity_shape(entity: &Entity) -> Result<(), StoreError> { "a form is not empty and has no control character or edge whitespace", )); } + if f.chars().any(is_refused_format) { + return Err(bad("a form holds an invisible format character")); + } + if f.chars().all(is_joining_only) { + return Err(bad("a form is more than joiners and selectors")); + } if !seen.insert(f) { return Err(bad("a form is not repeated within its language")); } diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 68229fa..e05b24a 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -1,6 +1,6 @@ mod common; use std::process::{Command, Output}; -use weksa::{InventoryStore, PacketStore}; +use weksa::{CorpusStore, EntityStore, InventoryStore, PacketStore}; fn weksa(dir: &std::path::Path, args: &[&str]) -> Output { Command::new(env!("CARGO_BIN_EXE_weksa")) @@ -35,6 +35,8 @@ fn cast_entities(dir: &std::path::Path, slugs: &[&str]) { "cast", "--persona", slug, + "--fixture", + "f.yaml", ], ); assert!(put.status.success(), "{}", out(&put)); @@ -81,7 +83,7 @@ fn seed_add_dump_split_withdraw() { "{dump}" ); - // A cast-authored item needs its persona and fixture. + // A cast-authored item needs its persona, fixture and author. let missing = weksa(d.path(), &ADD); assert!( !missing.status.success() && out(&missing).contains("--persona"), @@ -89,7 +91,14 @@ fn seed_add_dump_split_withdraw() { out(&missing) ); let mut add = ADD.to_vec(); - add.extend(["--persona", "nibu", "--fixture", "f.yaml"]); + add.extend([ + "--persona", + "nibu", + "--fixture", + "f.yaml", + "--author", + "Ada Writer", + ]); let added = weksa(d.path(), &add); assert!( added.status.success() && out(&added).contains("Added item:nibu:1"), @@ -294,7 +303,15 @@ fn schema_writes_the_published_files() { fn entity_put_and_name_spans() { let d = tempfile::tempdir().unwrap(); let put = |args: &[&str]| { - let mut all = vec!["entity", "put", "entity:kay", "--origin", "cast"]; + let mut all = vec![ + "entity", + "put", + "entity:kay", + "--origin", + "cast", + "--fixture", + "f.yaml", + ]; all.extend_from_slice(args); weksa(d.path(), &all) }; @@ -386,24 +403,59 @@ fn entity_origin_flags_must_match_origin() { weksa(d.path(), &all) }; for mismatched in [ - &["--origin", "live", "--persona", "kay"][..], - &["--origin", "live", "--work", "edda"], - &["--origin", "cast", "--persona", "kay", "--work", "edda"], + &["--origin", "live"][..], + &["--origin", "live", "--persona", "kay", "--work", "edda"], &[ "--origin", - "public-domain", + "live", "--persona", "kay", + "--fixture", + "f.yaml", + ], + &["--origin", "cast", "--persona", "kay"], + &["--origin", "cast", "--fixture", "f.yaml"], + &[ + "--origin", + "cast", + "--persona", + "kay", + "--fixture", + "f.yaml", "--work", "edda", ], - &["--origin", "public-domain", "--persona", "kay"], + &["--origin", "public-domain"], + &[ + "--origin", + "public-domain", + "--work", + "edda", + "--persona", + "kay", + ], + &[ + "--origin", + "public-domain", + "--work", + "edda", + "--fixture", + "f.yaml", + ], + &["--origin", "cast", "--persona", "", "--fixture", "f.yaml"], + &["--origin", "cast", "--persona", "kay", "--fixture", ""], + &["--origin", "live", "--persona", ""], + &["--origin", "public-domain", "--work", ""], ] { let o = put(mismatched); assert!(!o.status.success(), "{mismatched:?}: {}", out(&o)); } assert!(!out(&weksa(d.path(), &["entity", "dump"])).contains("entity:kay")); - assert!(put(&["--origin", "live"]).status.success()); + assert!( + put(&["--origin", "live", "--persona", "kay"]) + .status + .success() + ); let work = weksa( d.path(), &[ @@ -436,6 +488,8 @@ fn intake_normalizes_to_nfc() { "fr=Rene\u{301}", "--origin", "live", + "--persona", + "rene", ], ); assert!(put.status.success(), "{}", out(&put)); @@ -467,3 +521,349 @@ fn intake_normalizes_to_nfc() { let dump = out(&weksa(d.path(), &["corpus", "dump"])); assert!(dump.contains("Ren\u{e9} part."), "{dump}"); } + +/// `corpus add` of "Mind the hatch." by `entity:nibu` under `source`, with +/// `extra` flags. +fn add_with(dir: &std::path::Path, source: &str, extra: &[&str]) -> Output { + let mut args = vec![ + "corpus", + "add", + "--slug", + "nibu", + "--text", + "Mind the hatch.", + "--speaker", + "entity:nibu", + "--group", + "scene-1", + "--admitted-on", + "2026-10-29", + "--source", + source, + ]; + args.extend_from_slice(extra); + weksa(dir, &args) +} + +#[test] +fn source_flags_must_match_the_source() { + let d = tempfile::tempdir().unwrap(); + cast_entities(d.path(), &["nibu"]); + // A provenance flag that does not apply to --source refuses, naming the flag. + for (source, extra, flag) in [ + ("operator-written", &["--persona", "p"][..], "--persona"), + ( + "operator-approved", + &["--note", "n", "--work", "w"], + "--work", + ), + ( + "public-domain", + &[ + "--work", + "w", + "--author", + "a", + "--edition-url", + "u", + "--note", + "n", + ], + "--note", + ), + ( + "cast-authored", + &[ + "--persona", + "p", + "--fixture", + "f", + "--author", + "a", + "--generator", + "g", + ], + "--generator", + ), + ( + "model-written", + &["--model-family", "unrecorded", "--edition-url", "u"], + "--edition-url", + ), + ( + "operator-written", + &["--model-family", "unrecorded"], + "--model-family", + ), + ( + "model-written", + &[ + "--model-family", + "unrecorded", + "--prompt-digest", + "d", + "--fixture", + "f", + ], + "--fixture", + ), + // A source missing a flag it needs. + ( + "cast-authored", + &["--persona", "p", "--fixture", "f"], + "--author", + ), + ( + "public-domain", + &["--work", "w", "--edition-url", "u"], + "--author", + ), + // Empty provenance strings. + ( + "model-written", + &["--model-family", "unrecorded", "--generator", ""], + "", + ), + ( + "model-written", + &["--model-family", "unrecorded", "--prompt-digest", ""], + "", + ), + ( + "cast-authored", + &["--persona", "", "--fixture", "f", "--author", "a"], + "", + ), + ("operator-approved", &["--note", ""], ""), + ] { + let o = add_with(d.path(), source, extra); + assert!(!o.status.success(), "{source} {extra:?}: {}", out(&o)); + assert!(out(&o).contains(flag), "{source} {extra:?}: {}", out(&o)); + } + assert!(!out(&weksa(d.path(), &["corpus", "dump"])).contains("item:")); + // A source whose flags all apply is admitted; --author reaches a + // cast-authored item. + let cast = add_with( + d.path(), + "cast-authored", + &["--persona", "p", "--fixture", "f", "--author", "Ada Writer"], + ); + assert!(cast.status.success(), "{}", out(&cast)); + let dump = out(&weksa(d.path(), &["corpus", "dump"])); + assert!(dump.contains("author: \"Ada Writer\""), "{dump}"); +} + +#[test] +fn malformed_intake_never_echoes_its_values() { + const CANARY: &str = "CANARY-7f3a"; + let d = tempfile::tempdir().unwrap(); + cast_entities(d.path(), &["nibu"]); + let shape = |value: &str| format!("{CANARY}{value}"); + let puts = [ + (vec![shape("")], "--form"), + (vec![shape("_x=Kay")], ""), + (vec![format!("en-US={CANARY}\u{7}")], ""), + (vec![format!("{CANARY}_x=Kay")], ""), + ]; + for (forms, flag) in puts { + let mut args = vec![ + "entity", + "put", + "entity:kay", + "--origin", + "live", + "--persona", + "kay", + ]; + for f in &forms { + args.extend(["--form", f]); + } + let o = weksa(d.path(), &args); + assert!(!o.status.success(), "{forms:?}: {}", out(&o)); + assert!(!out(&o).contains(CANARY), "{forms:?}: {}", out(&o)); + assert!(out(&o).contains(flag), "{forms:?}: {}", out(&o)); + } + let bad_turn = [shape(""), format!("{CANARY}=hello")]; + let bad_name = [ + shape(""), + format!("0..1={CANARY}"), + format!("{CANARY}..1=entity:nibu"), + format!("0..{CANARY}=entity:nibu"), + ]; + let cases = bad_turn + .iter() + .map(|t| ("--prior-turn", t)) + .chain(bad_name.iter().map(|n| ("--name", n))); + for (flag, value) in cases { + let o = add_with(d.path(), "operator-written", &[flag, value]); + assert!(!o.status.success(), "{flag} {value}: {}", out(&o)); + assert!(!out(&o).contains(CANARY), "{flag} {value}: {}", out(&o)); + assert!(out(&o).contains(flag), "{flag} {value}: {}", out(&o)); + } +} + +#[test] +fn every_decomposed_form_and_turn_is_stored_composed() { + let d = tempfile::tempdir().unwrap(); + cast_entities(d.path(), &["nibu"]); + let put = weksa( + d.path(), + &[ + "entity", + "put", + "entity:rene", + "--form", + "fr=Rene\u{301}", + "--form", + "fr=Zoe\u{308}", + "--origin", + "live", + "--persona", + "rene", + ], + ); + assert!(put.status.success(), "{}", out(&put)); + let rene = EntityStore::open(&d.path().join("e.cc")) + .unwrap() + .get(&"entity:rene".parse().unwrap()) + .unwrap() + .unwrap(); + assert_eq!(rene.forms[0].forms, ["Ren\u{e9}", "Zo\u{eb}"]); + let add = add_with( + d.path(), + "operator-written", + &["--prior-turn", "entity:rene=Cafe\u{301} now"], + ); + assert!(add.status.success(), "{}", out(&add)); + let item = CorpusStore::open(&d.path().join("c.cc")) + .unwrap() + .get("item:nibu:1") + .unwrap() + .unwrap(); + assert_eq!(item.prior_turns[0].text, "Caf\u{e9} now"); +} + +/// An Ollama stand-in: every text embeds to `[1, its length]`, so identical +/// texts have cosine 1. +fn fake_embedder() -> String { + use std::io::{Read, Write}; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + std::thread::spawn(move || { + for mut stream in listener.incoming().flatten() { + let mut buf = vec![0u8; 65536]; + let mut n = 0; + let body = loop { + n += stream.read(&mut buf[n..]).unwrap(); + let text = String::from_utf8_lossy(&buf[..n]).into_owned(); + if let Some((head, body)) = text.split_once("\r\n\r\n") { + let want: usize = head + .to_lowercase() + .split("content-length: ") + .nth(1) + .unwrap() + .split("\r\n") + .next() + .unwrap() + .parse() + .unwrap(); + if body.len() >= want { + break body.to_string(); + } + } + }; + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + let vectors: Vec> = body["input"] + .as_array() + .unwrap() + .iter() + .map(|t| vec![1.0, t.as_str().unwrap().len() as f64]) + .collect(); + let reply = serde_json::json!({ "embeddings": vectors }).to_string(); + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{reply}", + reply.len() + ) + .unwrap(); + } + }); + url +} + +#[test] +fn proof_packet_lowers_and_the_lowering_scores_against_items() { + let d = tempfile::tempdir().unwrap(); + cast_entities(d.path(), &["nibu"]); + for args in [&["inventory", "seed"][..], &["inventory", "proof"]] { + let run = weksa(d.path(), args); + assert!(run.status.success(), "{}", out(&run)); + } + let item = weksa( + d.path(), + &[ + "corpus", + "add", + "--slug", + "nibu", + "--text", + "Warning: you may endanger yourself by touching the control. The ship mind will prevent that to protect you.", + "--speaker", + "entity:nibu", + "--group", + "scene-1", + "--admitted-on", + "2026-11-01", + "--source", + "operator-written", + ], + ); + assert!(item.status.success(), "{}", out(&item)); + let admitted = weksa(d.path(), &["packet", "proof", "./p.cc"]); + assert!( + admitted.status.success() && out(&admitted).contains("admitted packet:6e6962752d303031"), + "{}", + out(&admitted) + ); + let lowered = weksa(d.path(), &["lower", "./p.cc", "packet:6e6962752d303031"]); + assert!(lowered.status.success(), "{}", out(&lowered)); + let text = String::from_utf8(lowered.stdout).unwrap(); + assert!( + text.starts_with("Warning: you may endanger yourself"), + "{text}" + ); + + let missing = weksa(d.path(), &["lower", "./p.cc", "packet:0000000000000000"]); + assert!(!missing.status.success() && out(&missing).contains("no packet")); + + std::fs::write(d.path().join("out.txt"), &text).unwrap(); + let url = fake_embedder(); + let scored = weksa( + d.path(), + &[ + "score", + "item:nibu:1", + "--output", + "out.txt", + "--ollama-url", + &url, + ], + ); + assert!(scored.status.success(), "{}", out(&scored)); + assert_eq!( + out(&scored).trim(), + "item:nibu:1 chrf=100.00 meaning=1.0000" + ); + let unknown = weksa( + d.path(), + &[ + "score", + "item:nibu:9", + "--output", + "out.txt", + "--ollama-url", + &url, + ], + ); + assert!(!unknown.status.success() && out(&unknown).contains("no item item:nibu:9")); +} diff --git a/crates/weksa/tests/common/mod.rs b/crates/weksa/tests/common/mod.rs index aecf5d0..07e95ee 100644 --- a/crates/weksa/tests/common/mod.rs +++ b/crates/weksa/tests/common/mod.rs @@ -67,6 +67,7 @@ pub fn cast() -> Source { Source::CastAuthored { persona: "nibu".into(), fixture: "f.yaml".into(), + author: "Ada Writer".into(), } } @@ -95,6 +96,7 @@ pub fn ent(slug: &str, language: &str, forms: &[&str]) -> Entity { }], origin: EntityOrigin::Cast { persona: slug.into(), + fixture: "f.yaml".into(), }, } } diff --git a/crates/weksa/tests/entities.rs b/crates/weksa/tests/entities.rs index 4f8f2f2..46aeb71 100644 --- a/crates/weksa/tests/entities.rs +++ b/crates/weksa/tests/entities.rs @@ -105,7 +105,11 @@ fn entity_forms_only_grow() { }, |e| e.forms.reverse(), |e| e.forms[1].language = "fi".into(), - |e| e.origin = EntityOrigin::Live, + |e| { + e.origin = EntityOrigin::Live { + persona: "kay".into(), + } + }, ]; assert_eq!( refusal(store.put(ent("kay", "en-US", &["Kay"]))), diff --git a/crates/weksa/tests/entity_door.rs b/crates/weksa/tests/entity_door.rs index a0b3e9e..bcd53e8 100644 --- a/crates/weksa/tests/entity_door.rs +++ b/crates/weksa/tests/entity_door.rs @@ -163,28 +163,33 @@ fn entity_capacity_is_bounded() { let make = |forms: Vec| Entity { id: eid("big"), forms, - origin: EntityOrigin::Live, + origin: EntityOrigin::Live { + persona: "p".into(), + }, }; - // A whole line is not a name: 7 words. - let line = "I will never leave you, not ever."; - let e = make(vec![lf("en-US", one(line))]); - assert_eq!( - refusal(w.entities.put(e.clone())), - bounds(&e, EntityBound::FormWords) - ); - // Four words is the edge; five is over. 64 bytes is the edge; 65 is over. - w.entities - .put(Entity { - id: eid("four"), - ..make(vec![lf("en-US", one("Mary of the Isles"))]) - }) - .unwrap(); - let five = make(vec![lf("en-US", one("Mary of the Isles too"))]); - assert_eq!( - refusal(w.entities.put(five.clone())), - bounds(&five, EntityBound::FormWords) - ); + // A form is bounded by bytes, not words: a spaced name of any word count and + // names in scripts that write no spaces are admitted (the Thai name counts + // five UAX #29 words, a Han name one per character). + for (i, name) in [ + "I will never leave you, not ever.", + "Mary of the Isles too", + "de la Cruz y Garcia", + "\u{e2a}\u{e21}\u{e0a}\u{e32}\u{e22} \u{e43}\u{e08}\u{e14}\u{e35}", + "\u{5c71}\u{7530}\u{592a}\u{90ce}", + ] + .into_iter() + .enumerate() + { + w.entities + .put(Entity { + id: eid(&format!("name-{i}")), + ..make(vec![lf("en-US", one(name))]) + }) + .unwrap(); + } + // 64 bytes is the edge; 65 is over, counted in bytes (21 Han characters are + // 63 bytes, 22 are 66). let wide = make(vec![lf("en-US", one(&"a".repeat(65)))]); assert_eq!( refusal(w.entities.put(wide.clone())), @@ -196,6 +201,17 @@ fn entity_capacity_is_bounded() { ..make(vec![lf("en-US", one(&"a".repeat(64)))]) }) .unwrap(); + w.entities + .put(Entity { + id: eid("han-edge"), + ..make(vec![lf("ja", one(&"\u{5c71}".repeat(21)))]) + }) + .unwrap(); + let han = make(vec![lf("ja", one(&"\u{5c71}".repeat(22)))]); + assert_eq!( + refusal(w.entities.put(han.clone())), + bounds(&han, EntityBound::FormBytes) + ); // 8 forms per language is the edge. let forms = |n: usize| (0..n).map(|i| format!("form{i}")).collect::>(); @@ -321,6 +337,7 @@ fn entities_are_append_only_and_replay_needs_them() { let mut persona = base.clone(); persona.origin = EntityOrigin::Cast { persona: "someone-else".into(), + fixture: "f.yaml".into(), }; assert_eq!(refusal(w.entities.put(persona)), not_append); let mut duplicate = base.clone(); @@ -346,3 +363,276 @@ fn entities_are_append_only_and_replay_needs_them() { } ); } + +/// One entity holding `form` as its only en-US form. +fn with_form(slug: &str, form: &str) -> Entity { + Entity { + id: eid(slug), + forms: vec![LanguageForms { + language: "en-US".into(), + forms: vec![form.into()], + }], + origin: EntityOrigin::Live { + persona: "p".into(), + }, + } +} + +#[test] +fn invisible_format_characters_are_refused_in_forms() { + let mut w = world(); + let hidden = |slug: &str| StoreRefusal::Malformed { + key: format!("entity:{slug}"), + why: "a form holds an invisible format character", + }; + let at = |code: u32| char::from_u32(code).unwrap(); + // Every Cf code point except the joiners, plus U+2028 and U+2029, listed + // from Python's unicodedata. The code-point sweep in form.rs pins the edges + // of the rule; this checks the door applies it, in each position. + let listed: Vec = include_str!("format_refused.txt") + .lines() + .filter(|l| !l.starts_with('#')) + .map(|l| u32::from_str_radix(l, 16).unwrap()) + .collect(); + assert_eq!(listed.len(), 170); + for (n, code) in listed.iter().enumerate() { + let c = at(*code); + let mut forms = vec![format!("Ka{c}y")]; + if !c.is_whitespace() { + forms.extend([format!("{c}Kay"), format!("Kay{c}")]); + } + for form in forms { + assert_eq!( + refusal(w.entities.put(with_form("fmt", &form))), + hidden("fmt"), + "U+{code:X} (#{n}) in {form:?}" + ); + } + } + // Default_Ignorable fillers that are not Cf: Hangul and Khmer. + for code in [0x115f, 0x1160, 0x17b4, 0x17b5, 0x3164, 0xffa0] { + assert_eq!( + refusal( + w.entities + .put(with_form("fmt", &format!("Ka{}y", at(code)))) + ), + hidden("fmt"), + "U+{code:X}" + ); + } + assert!(w.entities.get(&eid("fmt")).unwrap().is_none()); + + // The neighbours of the refused ranges, ZWNJ, ZWJ, CGJ, variation + // selectors and the Mongolian selectors (all Mn) are admitted in a form. + let admitted: [u32; 33] = [ + 0xac, + 0xae, + 0x34f, + 0x5ff, + 0x606, + 0x61b, + 0x61d, + 0x6dc, + 0x6de, + 0x70e, + 0x710, + 0x88f, + 0x892, + 0x8e1, + 0x8e3, + 0x180b, + 0x180c, + 0x180d, + 0x180f, + 0x200a, + 0x200c, + 0x200d, + 0x2010, + 0x202f, + 0x2065, + 0x2070, + 0xfe00, + 0xfe0f, + 0xfeff - 1, + 0xfffc, + 0x110be, + 0x1d172, + 0xe0100, + ]; + for (n, code) in admitted.into_iter().enumerate() { + let form = format!("Ka{}y", at(code)); + w.entities + .put(with_form(&format!("ok-{n}"), &form)) + .unwrap_or_else(|e| panic!("U+{code:X}: {e}")); + } + // ZWNJ in a Persian form, ZWJ in an emoji sequence, VS16 and a Mongolian + // free variation selector beside a letter. + for (n, form) in [ + "\u{645}\u{6cc}\u{200c}\u{62e}\u{648}\u{627}\u{647}\u{645}", + "\u{1f469}\u{200d}\u{1f4bb}", + "\u{2764}\u{fe0f}", + "\u{1820}\u{180b}", + "\u{1f600}\u{e0100}", + ] + .into_iter() + .enumerate() + { + w.entities + .put(with_form(&format!("script-{n}"), form)) + .unwrap(); + } + // A form of only joiners, variation selectors, CGJ or Mongolian selectors + // shows nothing and is refused. + let empty_looking = |slug: &str| StoreRefusal::Malformed { + key: format!("entity:{slug}"), + why: "a form is more than joiners and selectors", + }; + for form in [ + "\u{200d}", + "\u{200c}", + "\u{200c}\u{200d}", + "\u{fe0f}", + "\u{fe00}\u{fe0f}", + "\u{34f}", + "\u{180b}\u{180c}\u{180d}\u{180f}", + "\u{e0100}", + "\u{e01ef}\u{34f}\u{200d}", + ] { + assert_eq!( + refusal(w.entities.put(with_form("blank", form))), + empty_looking("blank"), + "{form:?}" + ); + } + assert!(w.entities.get(&eid("blank")).unwrap().is_none()); +} + +#[test] +fn an_origin_attests_where_the_entity_came_from() { + let mut w = world(); + let bad = |slug: &str, why: &'static str| StoreRefusal::Malformed { + key: format!("entity:{slug}"), + why, + }; + let with = |slug: &str, origin: EntityOrigin| Entity { + origin, + ..with_form(slug, "Kay") + }; + let cast = |persona: &str, fixture: &str| EntityOrigin::Cast { + persona: persona.into(), + fixture: fixture.into(), + }; + assert_eq!( + refusal(w.entities.put(with("a", cast("", "f.yaml")))), + bad("a", "an origin names its persona") + ); + assert_eq!( + refusal(w.entities.put(with("b", cast("kay", "")))), + bad("b", "a cast origin names its fixture") + ); + assert_eq!( + refusal(w.entities.put(with( + "c", + EntityOrigin::Live { + persona: String::new() + } + ))), + bad("c", "an origin names its persona") + ); + assert_eq!( + refusal(w.entities.put(with( + "d", + EntityOrigin::PublicDomain { + work: String::new() + } + ))), + bad("d", "a public-domain origin names its work") + ); + for slug in ["a", "b", "c", "d"] { + assert!(w.entities.get(&eid(slug)).unwrap().is_none()); + } + w.entities.put(with("e", cast("kay", "f.yaml"))).unwrap(); + w.entities + .put(with( + "f", + EntityOrigin::PublicDomain { + work: "edda".into(), + }, + )) + .unwrap(); +} + +#[test] +fn an_origin_attestation_holds_something_readable() { + let mut w = world(); + let with = |slug: &str, origin: EntityOrigin| Entity { + origin, + ..with_form(slug, "Kay") + }; + type Build = fn(String) -> EntityOrigin; + let origins: [(Build, &str); 4] = [ + ( + |v| EntityOrigin::Cast { + persona: v, + fixture: "f.yaml".into(), + }, + "an origin names its persona", + ), + ( + |v| EntityOrigin::Cast { + persona: "kay".into(), + fixture: v, + }, + "a cast origin names its fixture", + ), + ( + |v| EntityOrigin::Live { persona: v }, + "an origin names its persona", + ), + ( + |v| EntityOrigin::PublicDomain { work: v }, + "a public-domain origin names its work", + ), + ]; + let unreadable = [ + " ".to_string(), + "\t".into(), + "\n".into(), + "\u{200b}".into(), + "\u{200d}".into(), + "\0".into(), + "--".into(), + "ok\u{200b}ok".into(), + "ok\u{2028}ok".into(), + "ok\0ok".into(), + "x".repeat(201), + "x".repeat(2 * 1024 * 1024), + ]; + let mut n = 0; + for (build, why) in &origins { + for bad in &unreadable { + n += 1; + let slug = format!("o-{n}"); + assert_eq!( + refusal(w.entities.put(with(&slug, build(bad.clone())))), + StoreRefusal::Malformed { + key: format!("entity:{slug}"), + why + }, + "{why}: {:?}", + bad.chars().take(12).collect::() + ); + assert!(w.entities.get(&eid(&slug)).unwrap().is_none()); + } + } + for (m, (build, _)) in origins.iter().enumerate() { + for (k, good) in ["x".repeat(200), "two words, 2 more".to_string()] + .into_iter() + .enumerate() + { + w.entities + .put(with(&format!("g-{}", 2 * m + k), build(good))) + .unwrap(); + } + } +} diff --git a/crates/weksa/tests/format_refused.txt b/crates/weksa/tests/format_refused.txt new file mode 100644 index 0000000..b994f95 --- /dev/null +++ b/crates/weksa/tests/format_refused.txt @@ -0,0 +1,171 @@ +# Cf minus ZWNJ/ZWJ, plus U+2028 and U+2029; Python 3.13 unicodedata 15.1.0 over 0..10FFFF +AD +600 +601 +602 +603 +604 +605 +61C +6DD +70F +890 +891 +8E2 +180E +200B +200E +200F +2028 +2029 +202A +202B +202C +202D +202E +2060 +2061 +2062 +2063 +2064 +2066 +2067 +2068 +2069 +206A +206B +206C +206D +206E +206F +FEFF +FFF9 +FFFA +FFFB +110BD +110CD +13430 +13431 +13432 +13433 +13434 +13435 +13436 +13437 +13438 +13439 +1343A +1343B +1343C +1343D +1343E +1343F +1BCA0 +1BCA1 +1BCA2 +1BCA3 +1D173 +1D174 +1D175 +1D176 +1D177 +1D178 +1D179 +1D17A +E0001 +E0020 +E0021 +E0022 +E0023 +E0024 +E0025 +E0026 +E0027 +E0028 +E0029 +E002A +E002B +E002C +E002D +E002E +E002F +E0030 +E0031 +E0032 +E0033 +E0034 +E0035 +E0036 +E0037 +E0038 +E0039 +E003A +E003B +E003C +E003D +E003E +E003F +E0040 +E0041 +E0042 +E0043 +E0044 +E0045 +E0046 +E0047 +E0048 +E0049 +E004A +E004B +E004C +E004D +E004E +E004F +E0050 +E0051 +E0052 +E0053 +E0054 +E0055 +E0056 +E0057 +E0058 +E0059 +E005A +E005B +E005C +E005D +E005E +E005F +E0060 +E0061 +E0062 +E0063 +E0064 +E0065 +E0066 +E0067 +E0068 +E0069 +E006A +E006B +E006C +E006D +E006E +E006F +E0070 +E0071 +E0072 +E0073 +E0074 +E0075 +E0076 +E0077 +E0078 +E0079 +E007A +E007B +E007C +E007D +E007E +E007F diff --git a/crates/weksa/tests/lower.rs b/crates/weksa/tests/lower.rs new file mode 100644 index 0000000..f50f096 --- /dev/null +++ b/crates/weksa/tests/lower.rs @@ -0,0 +1,212 @@ +//! The English lowerer against the committed inventory (revision 2) and the +//! `nibu-proof-001` packet, and the rules that keep a table row from speaking +//! for a meaning it does not carry. + +mod common; +use std::path::Path; + +use common::{iv, nd}; +use weksa::inventory::proof_revision; +use weksa::packet::{FeatValue, FeatureValue, Packet, proof_packet}; +use weksa::{ + CorpusStore, EntityStore, Inventory, InventoryStore, LowerError, LoweringInput, PacketStore, + lower, validate, +}; + +const PROOF: &str = "Warning: you may endanger yourself by touching the control. \ + The ship mind will prevent that to protect you."; + +fn committed() -> (Inventory, tempfile::TempDir) { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let inv = InventoryStore::open(&root.join("data/interlingua/inventory.cc")).unwrap(); + (inv.inventory().unwrap(), tempfile::tempdir().unwrap()) +} + +/// A `LoweringInput` has one source, `PacketStore::admit`, so every packet +/// under test goes through a throwaway store backed by the committed stores. +fn view(packet: &Packet, _inv: &Inventory) -> LoweringInput { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let inventory = InventoryStore::open(&root.join("data/interlingua/inventory.cc")).unwrap(); + admitted(packet, &inventory) +} + +/// The same door against an inventory store the test built. +fn admitted(packet: &Packet, inventory: &InventoryStore) -> LoweringInput { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let corpus = CorpusStore::open(&root.join("data/corpus/corpus.cc")).unwrap(); + let entities = EntityStore::open(&root.join("data/entity/entity.cc")).unwrap(); + let dir = tempfile::tempdir().unwrap(); + PacketStore::open(&dir.path().join("p.cc")) + .unwrap() + .admit(packet.clone(), inventory, &corpus, &entities) + .unwrap() +} + +fn lowered(packet: &Packet, inv: &Inventory) -> Result { + lower(&view(packet, inv), inv) +} + +#[test] +fn the_proof_packet_is_admitted_by_the_store_and_lowers_to_its_sentences() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let (inv, dir) = committed(); + let inventory = InventoryStore::open(&root.join("data/interlingua/inventory.cc")).unwrap(); + let corpus = CorpusStore::open(&root.join("data/corpus/corpus.cc")).unwrap(); + let entities = EntityStore::open(&root.join("data/entity/entity.cc")).unwrap(); + let mut packets = PacketStore::open(&dir.path().join("p.cc")).unwrap(); + packets + .admit(proof_packet(), &inventory, &corpus, &entities) + .unwrap(); + let stored = packets.get("packet:6e6962752d303031").unwrap().unwrap(); + assert_eq!(lowered(&stored, &inv).unwrap(), PROOF); +} + +#[test] +fn only_the_addressee_is_you_and_is_reflexive() { + let (inv, _dir) = committed(); + let mut p = proof_packet(); + p.discourse.addressee = None; + assert_eq!( + lowered(&p, &inv).unwrap(), + "Warning: the player may endanger the player by touching the control. \ + The ship mind will prevent that to protect the player." + ); +} + +#[test] +fn no_speech_act_means_no_opener_and_a_capital() { + let (inv, _dir) = committed(); + let mut p = proof_packet(); + p.discourse.speech_act = None; + assert!(lowered(&p, &inv).unwrap().starts_with("You may endanger")); + p.discourse.speech_act = Some(iv("speech_act:1")); + assert!(lowered(&p, &inv).unwrap().starts_with("You may endanger")); +} + +#[test] +fn a_packet_carrying_what_the_lowerer_does_not_render_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let mut store = InventoryStore::open(&dir.path().join("i.cc")).unwrap(); + let (rev1, entries) = weksa::inventory::seed(); + store.admit_revision(rev1, entries).unwrap(); + let (rev2, entries) = proof_revision(); + store.admit_revision(rev2, entries).unwrap(); + let mut ext = common::entry("extension:1", 3); + ext.value_kind = Some(weksa::ValueKind::Grade); + common::admit(&mut store, 3, vec![ext]); + let inv = store.inventory().unwrap(); + let discourse = |p: &mut Packet| { + p.discourse.features.push(FeatureValue { + feature: iv("feature:6"), + value: FeatValue::Grade(common::grade(300)), + }) + }; + let predication = |p: &mut Packet| { + p.predications[0].features.push(FeatureValue { + feature: iv("feature:1"), + value: FeatValue::Option(common::lab("negative")), + }) + }; + let relation = |p: &mut Packet| { + p.relations.push(weksa::packet::Relation { + relation: iv("relation:8"), + from: nd("n4"), + to: nd("n5"), + features: vec![], + }) + }; + let extension = |p: &mut Packet| { + p.inventory_rev = 3; + p.extensions.push(weksa::packet::Extension { + key: iv("extension:1"), + value: FeatValue::Grade(common::grade(3)), + }) + }; + let referent_feature = |p: &mut Packet| { + p.referents[0].features.push(FeatureValue { + feature: iv("feature:6"), + value: FeatValue::Grade(common::grade(1)), + }) + }; + let no_predication = |p: &mut Packet| p.predications.clear(); + let extra_role = |p: &mut Packet| { + let filler = p.predications[0].roles[0].filler.clone(); + p.predications[0].roles.push(weksa::packet::RoleFill { + role: iv("role:3"), + filler, + }) + }; + let lone_referent = |p: &mut Packet| { + let mut r = p.referents[0].clone(); + r.id = nd("n99"); + p.referents.push(r) + }; + for (what, edit) in [ + ("features", &discourse as &dyn Fn(&mut Packet)), + ("features", &predication), + ("relations", &relation), + ("extensions", &extension), + ("no predication", &no_predication), + ("a role the frame does not name", &extra_role), + ("a referent no role mentions", &lone_referent), + ] { + let mut p = proof_packet(); + edit(&mut p); + assert_eq!( + lower(&admitted(&p, &store), &inv), + Err(LowerError::Unrendered { what }), + "{what}" + ); + } + // The seed lets no feature sit on a referent, so that arm is refused by validate. + let mut p = proof_packet(); + referent_feature(&mut p); + assert!(validate(&p, &inv).is_err()); +} + +#[test] +fn an_unknown_row_and_an_unfilled_role_are_refused_by_name() { + let (inv, _dir) = committed(); + let mut p = proof_packet(); + p.discourse.speech_act = Some(iv("speech_act:3")); + assert_eq!( + lowered(&p, &inv), + Err(LowerError::NoRow { + id: iv("speech_act:3") + }) + ); + let mut p = proof_packet(); + p.predications[0].roles.pop(); + assert_eq!( + lowered(&p, &inv), + Err(LowerError::MissingRole { + predication: nd("n4"), + role: "patient".into() + }) + ); +} + +#[test] +fn a_row_cannot_speak_for_an_id_whose_gloss_differs() { + // Revision 2 with concept:4 glossed "hatch" in place of "control": the row + // for concept:4 says "control" and must not render it. + let dir = tempfile::tempdir().unwrap(); + let mut store = InventoryStore::open(&dir.path().join("i.cc")).unwrap(); + let (rev1, entries) = weksa::inventory::seed(); + store.admit_revision(rev1, entries).unwrap(); + let (rev2, mut entries) = proof_revision(); + for e in &mut entries { + if e.id == iv("concept:4") { + e.gloss = "hatch".into(); + } + } + store.admit_revision(rev2, entries).unwrap(); + let inv = store.inventory().unwrap(); + assert_eq!( + lower(&admitted(&proof_packet(), &store), &inv), + Err(LowerError::GlossMismatch { + id: iv("concept:4"), + row: "control" + }) + ); +} diff --git a/crates/weksa/tests/split_schema.rs b/crates/weksa/tests/split_schema.rs index d69c5cb..d1e638a 100644 --- a/crates/weksa/tests/split_schema.rs +++ b/crates/weksa/tests/split_schema.rs @@ -22,6 +22,7 @@ fn cast() -> Source { Source::CastAuthored { persona: "p".into(), fixture: "f".into(), + author: "a".into(), } } diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs index c21e49a..07cbe0c 100644 --- a/crates/weksa/tests/stores.rs +++ b/crates/weksa/tests/stores.rs @@ -144,6 +144,17 @@ fn corpus_keys_are_canonical() { } // An alias of a stored key is not a corpus item id at all. assert!(weksa::CorpusItemId::parse("item:nibu:01").is_err()); + // The last free number is handed out; past u32::MAX the slug has no free + // key, and asking neither wraps nor panics. + store + .add(item("item:b:4294967294", "one before the last"), &ents(&d)) + .unwrap(); + assert_eq!(store.next_key("b").unwrap(), "item:b:4294967295"); + let full = "a".repeat(48); + assert_eq!( + refusal(store.next_key(&full)), + malformed(&full, "slug has no free key") + ); } #[test] @@ -264,6 +275,322 @@ fn corpus_admission_rules() { ); } +#[test] +fn s6_reads_text_under_compatibility_case_folding() { + let d = dir(); + let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + // Each pair is one text to the key: fullwidth, ligature, superscript and + // circled-digit compatibility forms and full case folds (straße, STRASSE). + let pairs = [ + ("Brokkr", "\u{ff22}\u{ff52}\u{ff4f}\u{ff4b}\u{ff4b}\u{ff52}"), + ("fine", "\u{fb01}ne"), + ("room 2", "room \u{b2}"), + ("gate 1", "gate \u{2460}"), + ("stra\u{df}e", "STRASSE"), + ( + "\u{3a3}\u{3af}\u{3c3}\u{3c5}\u{3c6}\u{3bf}\u{3c2}", + "\u{3c3}\u{3af}\u{3c3}\u{3c5}\u{3c6}\u{3bf}\u{3c3}", + ), + // The first NFKC step: a compatibility letter folds only after it is + // mapped (math-bold S has no case fold of its own; black-letter H is + // not a case variant of h until NFKC reads it as H). + ( + "sigrun", + "\u{1d412}\u{1d422}\u{1d420}\u{1d42b}\u{1d42e}\u{1d427}", + ), + ("h", "\u{210c}"), + // The second NFKC step: a full case fold can leave a decomposed or + // reordered sequence that only a second NFKC composes to the same key + // (dialytika tonos, caron under line, macron below). + ("\u{390}", "\u{3aa}\u{301}"), + ("\u{3b0}", "\u{3ab}\u{301}"), + ("\u{1e96}\u{30c}", "\u{21e}\u{331}"), + ]; + for (n, (a, b)) in pairs.into_iter().enumerate() { + let (first, second) = ( + format!("item:s6:{}", 2 * n + 1), + format!("item:s6:{}", 2 * n + 2), + ); + store.add(item(&first, a), &ents(&d)).unwrap(); + assert_eq!( + refusal(store.add(item(&second, b), &ents(&d))), + StoreRefusal::S6TextHeld { + key: second.clone(), + held_by: first.clone() + }, + "{a:?} vs {b:?}" + ); + assert!(store.get(&second).unwrap().is_none()); + } + // A decomposed text never reaches the key: the door refuses it as not NFC. + assert_eq!( + refusal(store.add(item("item:s6:90", "Bro\u{308}kkr"), &ents(&d))), + StoreRefusal::NotNfc { + key: "item:s6:90".into() + } + ); + // Cyrillic a is not Latin a: homoglyphs across scripts stay apart. + store.add(item("item:s6:91", "cat"), &ents(&d)).unwrap(); + store + .add(item("item:s6:92", "c\u{430}t"), &ents(&d)) + .unwrap(); +} + +#[test] +fn s6_keeps_tone_marks_and_viramas_apart() { + let d = dir(); + let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + // Minimal pairs that differ only by a combining mark: Thai mai tho and mai + // ek on one word, Thai tone on a short vowel, a Devanagari virama. + let pairs = [ + ("\u{e19}\u{e49}\u{e33}", "\u{e19}\u{e33}"), + ("\u{e44}\u{e21}\u{e49}", "\u{e44}\u{e21}\u{e48}"), + ("\u{915}\u{94d}\u{92f}", "\u{915}\u{92f}"), + ]; + for (n, (a, b)) in pairs.into_iter().enumerate() { + store + .add(item(&format!("item:mk:{}", 2 * n + 1), a), &ents(&d)) + .unwrap_or_else(|e| panic!("{a:?}: {e}")); + store + .add(item(&format!("item:mk:{}", 2 * n + 2), b), &ents(&d)) + .unwrap_or_else(|e| panic!("{b:?} held by {a:?}: {e}")); + } + // A mark alone is still no text. + for (n, text) in ["\u{e49}", "\u{94d}"].into_iter().enumerate() { + let key = format!("item:mk:{}", 90 + n); + assert_eq!( + refusal(store.add(item(&key, text), &ents(&d))), + malformed(&key, "text has no alphanumeric character") + ); + } +} + +#[test] +fn an_attestation_holds_something_readable() { + let d = dir(); + let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + let s = |x: &str| x.to_string(); + type Build = fn(String) -> Source; + let builders: [(Build, &'static str); 9] = [ + ( + |v| Source::CastAuthored { + persona: v, + fixture: "f".into(), + author: "a".into(), + }, + "a cast-authored source names its persona", + ), + ( + |v| Source::CastAuthored { + persona: "p".into(), + fixture: v, + author: "a".into(), + }, + "a cast-authored source names its fixture", + ), + ( + |v| Source::CastAuthored { + persona: "p".into(), + fixture: "f".into(), + author: v, + }, + "a cast-authored source names its author", + ), + ( + |v| Source::OperatorApproved { note: v }, + "an operator-approved source carries its note", + ), + ( + |v| Source::PublicDomain { + work: v, + author: "a".into(), + edition_url: "u".into(), + }, + "a public-domain source names its work", + ), + ( + |v| Source::PublicDomain { + work: "w".into(), + author: v, + edition_url: "u".into(), + }, + "a public-domain source names its author", + ), + ( + |v| Source::PublicDomain { + work: "w".into(), + author: "a".into(), + edition_url: v, + }, + "a public-domain source names its edition url", + ), + ( + |v| Source::ModelWritten { + family: ModelFamily::Unrecorded, + generator: Some(v), + prompt_digest: None, + }, + "a model-written source names a non-empty generator", + ), + ( + |v| Source::ModelWritten { + family: ModelFamily::Unrecorded, + generator: None, + prompt_digest: Some(v), + }, + "a model-written source carries a non-empty prompt digest", + ), + ]; + let unreadable = [ + s(" "), + s("\t"), + s("\n"), + s("\u{200b}"), + s("\u{200d}"), + s("\0"), + s("--"), + s("\u{e49}"), + s("ok\u{200b}ok"), + s("ok\u{200d}ok"), + s("ok\u{2028}ok"), + s("ok\nok"), + s("ok\0ok"), + "x".repeat(201), + "x".repeat(2 * 1024 * 1024), + ]; + let mut n = 0; + for (build, why) in &builders { + for bad in &unreadable { + n += 1; + let key = format!("item:att:{n}"); + let refused = item_in(&key, &format!("line {n}"), "scene-1", build(bad.clone())); + assert_eq!( + refusal(store.add(refused, &ents(&d))), + malformed(&key, why), + "{why}: {:?}", + bad.chars().take(12).collect::() + ); + } + } + assert!(store.all().unwrap().is_empty()); + // 200 bytes and plain words with spaces attest; 201 does not. + for (m, (build, _)) in builders.iter().enumerate() { + for (k, good) in ["x".repeat(200), s("two words, 2 more")] + .into_iter() + .enumerate() + { + let key = format!("item:att-ok:{}", 2 * m + k + 1); + store + .add( + item_in(&key, &format!("fine {m} {k}"), "scene-1", build(good)), + &ents(&d), + ) + .unwrap_or_else(|e| panic!("{key}: {e}")); + } + } +} + +#[test] +fn a_text_with_no_alphanumeric_character_is_refused() { + let d = dir(); + let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + for (n, text) in ["...", "?!", " ", "\u{2014}\u{2014}", "\u{ff01}"] + .into_iter() + .enumerate() + { + let key = format!("item:p:{}", n + 1); + assert_eq!( + refusal(store.add(item(&key, text), &ents(&d))), + malformed(&key, "text has no alphanumeric character"), + "{text:?}" + ); + } + assert!(store.all().unwrap().is_empty()); + store.add(item("item:p:1", "a..."), &ents(&d)).unwrap(); +} + +#[test] +fn an_item_source_attests_with_non_empty_strings() { + let d = dir(); + let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + let s = |x: &str| x.to_string(); + let cast = |persona: &str, fixture: &str, author: &str| Source::CastAuthored { + persona: s(persona), + fixture: s(fixture), + author: s(author), + }; + let model = |generator: Option<&str>, digest: Option<&str>| Source::ModelWritten { + family: ModelFamily::Unrecorded, + generator: generator.map(s), + prompt_digest: digest.map(s), + }; + let public = |work: &str, author: &str, url: &str| Source::PublicDomain { + work: s(work), + author: s(author), + edition_url: s(url), + }; + let cases: [(Source, &'static str); 9] = [ + ( + cast("", "f", "a"), + "a cast-authored source names its persona", + ), + ( + cast("p", "", "a"), + "a cast-authored source names its fixture", + ), + ( + cast("p", "f", ""), + "a cast-authored source names its author", + ), + ( + Source::OperatorApproved { note: s("") }, + "an operator-approved source carries its note", + ), + ( + public("", "a", "u"), + "a public-domain source names its work", + ), + ( + public("w", "", "u"), + "a public-domain source names its author", + ), + ( + public("w", "a", ""), + "a public-domain source names its edition url", + ), + ( + model(Some(""), None), + "a model-written source names a non-empty generator", + ), + ( + model(None, Some("")), + "a model-written source carries a non-empty prompt digest", + ), + ]; + for (n, (source, why)) in cases.into_iter().enumerate() { + let key = format!("item:prov:{}", n + 1); + let refused = item_in(&key, &format!("line {n}"), "scene-1", source); + assert_eq!(refusal(store.add(refused, &ents(&d))), malformed(&key, why)); + } + assert!(store.all().unwrap().is_empty()); + // The same sources with every string present are admitted. + for (n, source) in [ + cast("p", "f", "a"), + Source::OperatorApproved { note: s("n") }, + Source::OperatorWritten, + public("w", "a", "u"), + model(Some("g"), Some("d")), + model(None, None), + ] + .into_iter() + .enumerate() + { + let key = format!("item:prov:{}", n + 1); + let item = item_in(&key, &format!("line {n}"), "scene-1", source); + assert_eq!(store.add(item, &ents(&d)).unwrap(), AddOutcome::Added); + } +} + #[test] fn s5_supersede_keeps_the_group() { let d = dir(); @@ -906,6 +1233,23 @@ fn the_committed_stores_load_and_hold_clean_data() { "no item claims a recorded human author" ); assert_eq!(items.len(), 3); + // Every committed entity attests, with non-empty strings, where it came + // from, and a cast entity's fixture is a file of this repository. + let entities = EntityStore::open(&root.join("data/entity/entity.cc")) + .unwrap() + .all() + .unwrap(); + assert!(!entities.is_empty()); + for e in &entities { + match &e.origin { + weksa::entity::EntityOrigin::Cast { persona, fixture } => { + assert!(!persona.is_empty(), "{}", e.id); + assert!(root.join(fixture).is_file(), "{}: {fixture}", e.id); + } + weksa::entity::EntityOrigin::Live { persona } => assert!(!persona.is_empty()), + weksa::entity::EntityOrigin::PublicDomain { work } => assert!(!work.is_empty()), + } + } } #[test] @@ -926,14 +1270,16 @@ fn seed_reseeded() { .all() .unwrap(); assert_eq!(items.len(), 3); - // The committed inventory is the seed, reseeded: stance features in, the - // displayed-affect features out. + // The committed inventory is the seed, reseeded (stance features in, the + // displayed-affect features out), then the nibu-proof revision. let committed = InventoryStore::open(&root.join("data/interlingua/inventory.cc")) .unwrap() .inventory() .unwrap(); - let (rev, entries) = seed(); - assert!(committed.has_revision(rev.n)); + let (rev, mut entries) = seed(); + let (proof, proof_entries) = weksa::inventory::proof_revision(); + assert!(committed.has_revision(rev.n) && committed.has_revision(proof.n)); + entries.extend(proof_entries); let mut have: Vec<_> = committed.entries().cloned().collect(); let mut want = entries; have.sort_by_key(|e| e.id); diff --git a/data/entity/entity.cc b/data/entity/entity.cc index f561201..8f9a198 100644 --- a/data/entity/entity.cc +++ b/data/entity/entity.cc @@ -1 +1 @@ -cultcache.store.v1weksa.entityweksa.entityweksa.entity.v1weksa.entityL{"schemaName":"weksa.entity","schemaVersion":"weksa.entity.v1","members":[]}weksa.entityentity:brokkrweksa.entity2026-10-29T14:59:18Z-entity:brokkren-USBrokkrCastbrokkrentity:nibuweksa.entity2026-10-29T14:59:18Z'entity:nibuen-USNibuCastnibu \ No newline at end of file +cultcache.store.v1weksa.entityweksa.entityweksa.entity.v1weksa.entityL{"schemaName":"weksa.entity","schemaVersion":"weksa.entity.v1","members":[]}weksa.entityentity:brokkrweksa.entity2026-10-29T16:43:28Znentity:brokkren-USBrokkrCastbrokkr?examples/persona-lowering/brokkr-edda-voice/lowered-output.yamlentity:nibuweksa.entity2026-10-29T16:43:28Zeentity:nibuen-USNibuCastnibu`, an opaque digest of item, inventory revision and encoder (the item key lives in `provenance.corpus_item`, which the lowerer never sees); CultCache type `weksa.interlingua_packet`, carrying `schema_version` and `inventory_rev`. Every id inside is an opaque form type (`n` nodes, `encoder:<16 hex>`, canonical `:`). From cut `speaker-state` (schema v2) it also carries the utterance **stance**: the speaker's felt affect and restraint, as inventory features of slot `Stance` (see Speaker state below) | Minted by an encoder for one corpus item under one spec version; immutable; a new spec version means a new packet, never an edit. Admitted only through `PacketStore::admit`, which validates against the inventory store at the packet's revision | The encoder writes; the form types refuse malformed ids at construction and load, the validator refuses broken references and over-capacity packets: a feature at most once per slot, at most 64 nodes (`typed-packet`); nothing else writes packets | +| Packet | Key `packet:<16 hex>`, an opaque digest of item, inventory revision and encoder (the item key lives in `provenance.corpus_item`, which the lowerer never sees); CultCache type `weksa.interlingua_packet`, carrying `schema_version` and `inventory_rev`. Every id inside is an opaque form type (`n` nodes, `encoder:<16 hex>`, canonical `:`). From cut `speaker-state` (schema v4) it also carries the utterance **stance**: the speaker's felt affect and restraint, as inventory features of slot `Stance` (see Speaker state below) | Minted by an encoder for one corpus item under one spec version; immutable; a new spec version means a new packet, never an edit. Admitted only through `PacketStore::admit`, which validates against the inventory store at the packet's revision | The encoder writes; the form types refuse malformed ids at construction and load, the validator refuses broken references and over-capacity packets: a feature at most once per slot, at most 64 nodes (`typed-packet`); nothing else writes packets | | Schema version | `weksa.interlingua_packet.v` constant in `crates/weksa`; published JSON Schema at `schemas/cultnet/` | Bumped by a commit that changes the Rust types; the published schema is regenerated in the same commit and a test fails on drift | A cut (Hands, under a spec); never the loop | -| Inventory entry | Key `:`, `n` canonical decimal >= 1 (families: concept, frame, role, relation, speech_act, feature, referent_kind, extension); type `weksa.inventory_entry`, with a bounded gloss, value kind (option labels bounded), the slots a feature may sit in (`Discourse`, `Referent`, `Predication`, `Relation`; from cut `speaker-state` also `Stance`, `Speaker`, `Scene`), `introduced_rev` and optional `retired_rev`. Speaker axes, scene relations and affects are features of those slots, so the loop grows them like any closed-class family and convergence covers them | Added or retired by an inventory revision; never edited in place (a changed meaning is a retire plus an add); a retired id still resolves for packets of earlier revisions | The loop's accepted rounds, whose minting call the context builder assembles from Train items only; Self on merge to main. Admission checks form, not text | +| Inventory entry | Key `:`, `n` canonical decimal >= 1 (families: concept, frame, role, relation, speech_act, feature, referent_kind, extension); type `weksa.inventory_entry`, with a bounded gloss, value kind (option labels bounded), the slots a feature may sit in (`Discourse`, `Referent`, `Predication`, `Relation`; `Stance` from cut `speaker-state`, `Speaker` and `Scene` from cut `speaker-scene`), `introduced_rev` and optional `retired_rev`. Speaker axes, scene relations and affects are features of those slots, so the loop grows them like any closed-class family and convergence covers them | Added or retired by an inventory revision; never edited in place (a changed meaning is a retire plus an add); a retired id still resolves for packets of earlier revisions | The loop's accepted rounds, whose minting call the context builder assembles from Train items only; Self on merge to main. Admission checks form, not text | | Inventory revision | Key `rev:`, type `weksa.inventory_revision`, with parent, change list and `frozen` | Appended per accepted round; exactly one revision is ever marked frozen, by the convergence test | The loop's accept rule appends; the convergence test freezes; Self records the freeze as a ruling | | Target profile | Locale or language-project id (`en-US`, `ja-JP`, a conlang's slug) plus a revision `profile::` | Authored, revised by commit or by an accepted round that changed the lowering; each lowering names the revision it used | Profile authors and accepted rounds; never the lowerer's defaults (`persona-drives-register`) | | Language ontology entry (cut `language-ontology`) | Key `onto::`; type `weksa.ontology_entry`: conditions over feature values (stance, speaker, scene, packet features) and community ids, and the surface choice they select (layer: lexicon, grammar, discourse, phonology, orthography; a bounded descriptor and at most three bounded examples); introduced and retired by a profile revision | Added or retired by a profile revision, never edited; converted from `data/target-language-ontology/*.yaml`, which is deleted in the same cut so the YAML is not a second owner | The language's owner (the operator for her settings' languages) and accepted rounds; an ontology-minting call is assembled from Train items only, like an inventory-minting call, and its text is in the scorer's leak diagnostic. Selection is deterministic code (`select(profile, stance, speaker, scene)`); the lowerer realizes the selected choices and the trace lists their ids (`lowering-traced`) | | Community (cut `language-ontology`) | Key `community:`; type `weksa.community`, converted from `data/cultural-ontology/cultures/` and `referents/` (YAML deleted in the same cut) | Revised by commit; a speaker names the communities it belongs to by id | The cultural ontology's author (owner-attested evidence only; Discord-only content stays deleted, `discord-only-content-deleted`) | -| Speaker (cut `speaker-state`) | Key `speaker:@`; type `weksa.speaker`: stable axes as feature values of slot `Speaker` (origin, socioeconomic standing, age cohort, education and whatever the loop adds), at an inventory revision, and an `Annotator` | A new revision appends, never edits; a corpus item and every prior turn name a speaker slug, and a round records the speaker revisions it read | The annotator it names: `PersonaSnapshot` (a projection of a `gamecult.persona_state.v0` document, naming its source document and `updatedAt`), `Operator`, `WorkMetadata` (a public-domain work's own cast list and narration), or `Model { family, generator }` (the 2b `ModelFamily`) | -| Scene (cut `speaker-state`) | Key `scene:@`, the item's split group; type `weksa.scene`: participants (speaker slugs) and, per ordered pair, relation features of slot `Scene` (relative status, intimacy, in-group, audience and whatever the loop adds), at an inventory revision, and an `Annotator` | As Speaker. Keyed by the group, so a scene never straddles the split | As Speaker | +| Speaker (cut `speaker-scene`) | Key `speaker:@`; type `weksa.speaker`: stable axes as feature values of slot `Speaker` (origin, socioeconomic standing, age cohort, education and whatever the loop adds), at an inventory revision, and an `Annotator` | A new revision appends, never edits; a corpus item and every prior turn name a speaker slug, and a round records the speaker revisions it read | The annotator it names: `PersonaSnapshot` (a projection of a `gamecult.persona_state.v0` document, naming its source document and `updatedAt`), `Operator`, `WorkMetadata` (a public-domain work's own cast list and narration), or `Model { family, generator }` (the 2b `ModelFamily`) | +| Scene (cut `speaker-scene`) | Key `scene:@`, the item's split group; type `weksa.scene`: participants (speaker slugs) and, per ordered pair, relation features of slot `Scene` (relative status, intimacy, in-group, audience and whatever the loop adds), at an inventory revision, and an `Annotator` | As Speaker. Keyed by the group, so a scene never straddles the split | As Speaker | | Corpus item | Key `item::`, `n` canonical decimal >= 1 (no sign, no leading zero, from cut 2b); type `weksa.corpus_item` with text, language, speaker, split group, prior turns, source. Source `ModelWritten` carries a `ModelFamily` enum, the generator string as the fixture wrote it, and a prompt digest (cut 2b). The item carries no speaker state of its own: its speaker slug and group name the Speaker and Scene documents | Admitted once; the whole item is immutable (re-adding an identical item is a no-op, any difference is refused; a correction is a new item that `supersedes` the old); text already held under another key is refused unless the new item supersedes that key (cut 2b); withdrawn, never deleted, so old scores stay explainable | Importers and `weksa corpus add`, under `corpus-cast-plus-public-domain`; the source enum has no variant for Discord or any training-forbidden source (`corpus-clean`) | | Split assignment | Derived: `split(scheme, item.group)` | Fixed for a scheme; a new scheme is a new constant, never a reassignment. A model-written item whose group derives test is in no split; a superseding item keeps its predecessor's group | The hash function. Nobody assigns by hand, so there is no writer to forbid | | Loop round | Key `run: