diff --git a/.voidbot/state/weksa.cc b/.voidbot/state/weksa.cc index 7960180..be51ad6 100644 Binary files a/.voidbot/state/weksa.cc and b/.voidbot/state/weksa.cc differ diff --git a/AGENTS.md b/AGENTS.md index 55d488a..ce540a6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -51,13 +51,3 @@ The live state architecture is ported from EpiphanyAgent in reduced form: Project truth does not belong in agent personality memory. If Weksa later grows role dossiers or heartbeat scheduling, those surfaces must remain separate from language-project truth. - -## Building And Verifying - -- `crates/weksa` depends on CultLib's `cultcache-rs` as a git dependency pinned by - `rev`, fetched from GitHub. With the default `CARGO_HOME` that fetch fails on a - Mind's host. Use the pre-fetched home: - `CARGO_HOME=/var/lib/mind-weksa-self/cargo-home cargo test --workspace`. -- `weksa packet proof ` admits the nibu proof packet under the key - `packet:6e6962752d303031` (hex of `nibu-001`), not `nibu-proof-001`. Use that key - with `weksa lower `. diff --git a/crates/weksa/src/corpus.rs b/crates/weksa/src/corpus.rs index d67cc60..403cdb8 100644 --- a/crates/weksa/src/corpus.rs +++ b/crates/weksa/src/corpus.rs @@ -17,13 +17,10 @@ pub struct Turn { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub enum Source { - /// A line from a cast member's hand-authored fixture, with the human who - /// wrote it. A fixture that records no author is `ModelWritten` with family - /// `Unrecorded`, never this (`corpus-labels-tell-truth`). + /// A line from a cast member's hand-authored fixture. CastAuthored { persona: String, fixture: String, - author: String, }, OperatorWritten, OperatorApproved { diff --git a/crates/weksa/src/entity.rs b/crates/weksa/src/entity.rs index 33d22e1..4c51e84 100644 --- a/crates/weksa/src/entity.rs +++ b/crates/weksa/src/entity.rs @@ -13,21 +13,19 @@ use crate::packet::EntityId; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub struct LanguageForms { pub language: String, - /// 1 to 64 bytes each, no control or invisible format character, no edge - /// whitespace, none repeated. + /// 1 to 64 bytes each, no control character, no edge whitespace, none repeated. pub forms: Vec, } /// Where an entity was minted. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub enum EntityOrigin { - /// From a cast member's fixture: the persona and the fixture path the - /// entity came from. - Cast { persona: String, fixture: String }, + /// From a cast member's fixture. + Cast { persona: String }, /// From a public-domain work. PublicDomain { work: String }, - /// Minted live by the named persona before it names the entity. - Live { persona: String }, + /// Minted live by a persona before it names the entity. + Live, } /// Key: the entity id. Forms only grow: see `EntityStore::put`. diff --git a/crates/weksa/src/form.rs b/crates/weksa/src/form.rs index 9de8eb8..d8336fb 100644 --- a/crates/weksa/src/form.rs +++ b/crates/weksa/src/form.rs @@ -86,114 +86,3 @@ pub(crate) fn is_slug(s: &str) -> bool { && b[0] != b'-' && b[b.len() - 1] != b'-' } - -/// A General_Category Cf code point, or the line and paragraph separators -/// U+2028 and U+2029, other than the joiners ZWNJ U+200C and ZWJ U+200D. The -/// ranges are Unicode 15.1 Cf; `tests/format_refused.txt` is the full list from -/// Python's `unicodedata` that the table is swept against. -pub(crate) fn is_format(c: char) -> bool { - matches!( - c, - '\u{ad}' - | '\u{600}'..='\u{605}' - | '\u{61c}' - | '\u{6dd}' - | '\u{70f}' - | '\u{890}'..='\u{891}' - | '\u{8e2}' - | '\u{180e}' - | '\u{200b}' - | '\u{200e}'..='\u{200f}' - | '\u{2028}'..='\u{202e}' - | '\u{2060}'..='\u{2064}' - | '\u{2066}'..='\u{206f}' - | '\u{feff}' - | '\u{fff9}'..='\u{fffb}' - | '\u{110bd}' - | '\u{110cd}' - | '\u{13430}'..='\u{1343f}' - | '\u{1bca0}'..='\u{1bca3}' - | '\u{1d173}'..='\u{1d17a}' - | '\u{e0001}' - | '\u{e0020}'..='\u{e007f}' - ) -} - -/// ZWNJ and ZWJ: Cf, but scripts and emoji sequences need them inside a form. -pub(crate) fn is_joiner(c: char) -> bool { - matches!(c, '\u{200c}' | '\u{200d}') -} - -/// A character a name or form may not hold: every `is_format` code point, and -/// the invisible fillers that are Default_Ignorable without being Cf (Hangul -/// U+115F, U+1160, U+3164, U+FFA0 and Khmer U+17B4, U+17B5). Joiners, -/// variation selectors, CGJ and the Mongolian selectors are not refused. -pub(crate) fn is_refused_format(c: char) -> bool { - is_format(c) - || matches!( - c, - '\u{115f}' | '\u{1160}' | '\u{17b4}' | '\u{17b5}' | '\u{3164}' | '\u{ffa0}' - ) -} - -/// A joiner, a variation selector (U+FE00..U+FE0F, U+E0100..U+E01EF), CGJ -/// U+034F or a Mongolian free variation selector (U+180B..U+180D, U+180F): it -/// shapes a neighbour and shows nothing alone. -pub(crate) fn is_joining_only(c: char) -> bool { - is_joiner(c) - || matches!( - c, - '\u{34f}' - | '\u{180b}'..='\u{180d}' - | '\u{180f}' - | '\u{fe00}'..='\u{fe0f}' - | '\u{e0100}'..='\u{e01ef}' - ) -} - -/// An attestation string (author, persona, fixture, note, work, url, generator, -/// digest): at most 200 bytes, at least one alphanumeric, no control, format or -/// joiner character. Whitespace between words is fine; it attests something. -pub(crate) fn is_attestation(s: &str) -> bool { - s.len() <= 200 - && s.chars().any(char::is_alphanumeric) - && !s - .chars() - .any(|c| c.is_control() || is_refused_format(c) || is_joiner(c)) -} - -#[cfg(test)] -mod tests { - use super::*; - - /// The rule against every code point: `is_format` is exactly the listed - /// Cf-minus-joiners-plus-U+2028/9 set, so each range has both of its - /// edges and the gaps between ranges pinned. - #[test] - fn format_rule_matches_the_unicode_list_at_every_code_point() { - let listed: std::collections::BTreeSet = include_str!("../tests/format_refused.txt") - .lines() - .filter(|l| !l.starts_with('#')) - .map(|l| u32::from_str_radix(l, 16).unwrap()) - .collect(); - for code in 0..=0x10ffff_u32 { - if let Some(c) = char::from_u32(code) { - assert_eq!(is_format(c), listed.contains(&code), "U+{code:X}"); - } - } - assert!(!is_format('\u{200c}') && !is_format('\u{200d}')); - } - - #[test] - fn an_attestation_is_bounded_readable_text() { - assert!(is_attestation("a")); - assert!(is_attestation(&"x".repeat(200))); - assert!(!is_attestation(&"x".repeat(201))); - assert!(is_attestation("two words, 2 more")); - for bad in [ - "", " ", "\t", "\u{200b}", "\u{200d}", "\0", "--", "a\nb", "a\u{ad}b", - ] { - assert!(!is_attestation(bad), "{bad:?}"); - } - } -} diff --git a/crates/weksa/src/inventory.rs b/crates/weksa/src/inventory.rs index 22538dd..792cad4 100644 --- a/crates/weksa/src/inventory.rs +++ b/crates/weksa/src/inventory.rs @@ -473,38 +473,3 @@ pub fn seed() -> (InventoryRevision, Vec) { entries, ) } - -/// Revision 2: the concepts and frames the `nibu-proof-001` packet names, which -/// the seed leaves open. Hand-seeded like [`seed`] until the golf loop's -/// minting round owns concepts and frames. -pub fn proof_revision() -> (InventoryRevision, Vec) { - let mut entries: Vec = Vec::new(); - for (family, glosses) in [ - ( - Family::Concept, - ["endanger", "prevent", "player", "control", "ship_mind"].as_slice(), - ), - (Family::Frame, ["hazard_warning", "intervention"].as_slice()), - ] { - for (i, gloss) in glosses.iter().enumerate() { - entries.push(InventoryEntry { - id: family.id(NonZeroU32::MIN.saturating_add(i as u32)), - gloss: gloss.to_string(), - value_kind: None, - applies_to: vec![], - introduced_rev: 2, - retired_rev: None, - }); - } - } - let changes = entries.iter().map(|e| Change::Add(e.id)).collect(); - ( - InventoryRevision { - n: 2, - parent: Some(1), - changes, - frozen: false, - }, - entries, - ) -} diff --git a/crates/weksa/src/lib.rs b/crates/weksa/src/lib.rs index 0bfdb8c..479366b 100644 --- a/crates/weksa/src/lib.rs +++ b/crates/weksa/src/lib.rs @@ -7,7 +7,6 @@ pub mod corpus; pub mod entity; pub mod form; pub mod inventory; -pub mod lower; pub mod packet; pub mod schema; pub mod score; @@ -22,7 +21,6 @@ pub use inventory::{ Change, Family, InvId, Inventory, InventoryEntry, InventoryRevision, OptionLabel, OptionSet, Slot, ValueKind, }; -pub use lower::{LowerError, lower}; pub use packet::{ EncoderId, EntityId, Grade, NodeId, Packet, PacketId, PacketKind, SCHEMA_VERSION, Stance, }; diff --git a/crates/weksa/src/lower.rs b/crates/weksa/src/lower.rs deleted file mode 100644 index d6fcdfb..0000000 --- a/crates/weksa/src/lower.rs +++ /dev/null @@ -1,205 +0,0 @@ -//! The English lowerer: the validated lowering view of a packet to text, by a -//! fixed table keyed by inventory ids. It renders predications, referents and the -//! speech act, and nothing else: no style, no model, no corpus text. It takes a -//! [`LoweringInput`], so it cannot see the trace or the corpus item. A packet -//! carrying a feature, relation, extension, role fill or referent it does not -//! render is refused, because rendering part of a meaning would assert a different -//! one (`polarity: negative` dropped says the opposite). Context, kind and -//! constraints steer delivery and are not rendered; the concept of a predication -//! must resolve but the frame owns the verb. -//! -//! The table is a temporary owner. Each row speaks for an inventory entry and -//! names the gloss it speaks for, so a row cannot outlive a change of what its id -//! means. `language-ontology` moves the rows into ontology documents. - -use std::fmt; - -use crate::inventory::{Family, InvId, Inventory}; -use crate::packet::{NodeId, Predication}; -use crate::validate::LoweringInput; - -/// `(inventory id, the gloss the row speaks for, surface)`. -type Row = (&'static str, &'static str, &'static str); - -/// Referent concepts, as the noun phrase after "the". -const CONCEPTS: &[Row] = &[ - ("concept:3", "player", "player"), - ("concept:4", "control", "control"), - ("concept:5", "ship_mind", "ship mind"), -]; - -/// Frames: the clause a predication of that frame says. `{gloss}` is the filler -/// of the role whose inventory gloss it is. -const FRAMES: &[Row] = &[ - ( - "frame:1", - "hazard_warning", - "{agent} may endanger {patient} by touching {instrument}", - ), - ( - "frame:2", - "intervention", - "{agent} will prevent {patient} to protect {beneficiary}", - ), -]; - -/// Speech acts: the opener of the first sentence. -const SPEECH_ACTS: &[Row] = &[ - ("speech_act:1", "assert", ""), - ("speech_act:5", "warn", "Warning: "), -]; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum LowerError { - /// The packet carries a slot the lowerer does not render. - Unrendered { what: &'static str }, - /// No table row for an id the packet uses. - NoRow { id: InvId }, - /// The row's gloss is not the gloss the inventory gives its id at the - /// packet's revision, or the id does not resolve there. - GlossMismatch { id: InvId, row: &'static str }, - /// A frame template names a role the inventory has no live entry for. - UnknownRole { frame: InvId, role: String }, - /// The predication fills no role its frame's template needs. - MissingRole { predication: NodeId, role: String }, -} - -impl fmt::Display for LowerError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{self:?}") - } -} - -impl std::error::Error for LowerError {} - -fn row( - table: &'static [Row], - id: InvId, - inv: &Inventory, - rev: u32, -) -> Result<&'static str, LowerError> { - let (_, gloss, surface) = table - .iter() - .find(|(key, _, _)| *key == id.to_string()) - .ok_or(LowerError::NoRow { id })?; - match inv.resolve(&id, rev) { - Some(e) if e.gloss == *gloss => Ok(surface), - _ => Err(LowerError::GlossMismatch { id, row: gloss }), - } -} - -pub fn lower(input: &LoweringInput, inv: &Inventory) -> Result { - let rev = input.inventory_rev(); - let unrendered = |what| Err(LowerError::Unrendered { what }); - if input.predications().is_empty() { - return unrendered("no predication"); - } - if !input.discourse().features.is_empty() - || input.referents().iter().any(|r| !r.features.is_empty()) - || input.predications().iter().any(|p| !p.features.is_empty()) - { - return unrendered("features"); - } - if !input.relations().is_empty() { - return unrendered("relations"); - } - if !input.extensions().is_empty() { - return unrendered("extensions"); - } - let opener = match input.discourse().speech_act { - Some(act) => row(SPEECH_ACTS, act, inv, rev)?, - None => "", - }; - let mut sentences = Vec::new(); - for p in input.predications() { - sentences.push(clause(input, inv, p)?); - } - let mentioned = |r: &NodeId| { - input - .predications() - .iter() - .any(|p| p.roles.iter().any(|f| &f.filler == r)) - }; - if input.referents().iter().any(|r| !mentioned(&r.id)) { - return unrendered("a referent no role mentions"); - } - let mut out = String::new(); - for (i, clause) in sentences.iter().enumerate() { - if i > 0 { - out.push(' '); - } - if i == 0 { - out.push_str(opener); - } - let mut chars = clause.chars(); - if let Some(first) = chars.next() { - if i == 0 && !opener.is_empty() { - out.push(first); - } else { - out.extend(first.to_uppercase()); - } - out.push_str(chars.as_str()); - } - out.push('.'); - } - Ok(out) -} - -fn clause(input: &LoweringInput, inv: &Inventory, p: &Predication) -> Result { - let rev = input.inventory_rev(); - let template = row(FRAMES, p.frame, inv, rev)?; - let mut said: Vec<&NodeId> = Vec::new(); - let mut out = String::new(); - let mut rest = template; - while let Some(open) = rest.find('{') { - out.push_str(&rest[..open]); - let close = rest[open..].find('}').map_or(rest.len(), |c| open + c); - let name = &rest[open + 1..close]; - rest = &rest[(close + 1).min(rest.len())..]; - let role = inv - .entries() - .find(|e| e.family() == Family::Role && e.gloss == name && e.live_at(rev)) - .ok_or_else(|| LowerError::UnknownRole { - frame: p.frame, - role: name.to_string(), - })?; - let fill = - p.roles - .iter() - .find(|r| r.role == role.id) - .ok_or_else(|| LowerError::MissingRole { - predication: p.id.clone(), - role: name.to_string(), - })?; - out.push_str(&filler(input, inv, &fill.filler, &said)?); - said.push(&fill.filler); - } - out.push_str(rest); - if p.roles.len() != said.len() { - return Err(LowerError::Unrendered { - what: "a role the frame does not name", - }); - } - Ok(out) -} - -/// A role filler as a phrase: the addressee is "you", a predication is "that", -/// another referent is "the" and its concept word. The addressee said twice in -/// one predication is reflexive; no other referent is, because the packet has no -/// animacy to choose "themself" or "itself" by. -fn filler( - input: &LoweringInput, - inv: &Inventory, - node: &NodeId, - said: &[&NodeId], -) -> Result { - let again = said.contains(&node); - if let Some(r) = input.referents().iter().find(|r| &r.id == node) { - if input.discourse().addressee.as_ref() == Some(node) { - return Ok(if again { "yourself" } else { "you" }.to_string()); - } - let word = row(CONCEPTS, r.concept, inv, input.inventory_rev())?; - return Ok(format!("the {word}")); - } - Ok("that".to_string()) -} diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index f36ba27..48fce8f 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -8,12 +8,10 @@ use clap::{Args, Parser, Subcommand, ValueEnum}; use unicode_normalization::UnicodeNormalization; use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn}; use weksa::entity::{Entity, EntityOrigin, LanguageForms}; -use weksa::inventory::{proof_revision, seed}; -use weksa::packet::proof_packet; -use weksa::score::ScoreInput; +use weksa::inventory::seed; use weksa::{ - CorpusStore, EntityId, EntityStore, InventoryStore, OllamaEmbedder, PacketStore, Scorer, - SplitScheme, StoreError, lower, validate, + CorpusStore, EntityId, EntityStore, InventoryStore, PacketStore, SplitScheme, StoreError, + validate, }; #[derive(Parser)] @@ -45,12 +43,6 @@ enum Command { Inventory(InventoryCommand), #[command(subcommand)] Entity(EntityCommand), - #[command(subcommand)] - Packet(PacketCommand), - /// Lower a stored packet to English text. - Lower { store: PathBuf, key: String }, - /// Score output text (stdin, or --output) against corpus items, one embed call. - Score(ScoreArgs), /// Print the split of a corpus item. Split { item_key: String }, /// Write the published JSON Schemas. @@ -60,26 +52,6 @@ enum Command { }, } -#[derive(Subcommand)] -enum PacketCommand { - /// Admit the nibu-proof-001 packet through `PacketStore::admit`. - Proof { store: PathBuf }, -} - -#[derive(Args)] -struct ScoreArgs { - /// Corpus item keys, the references. - #[arg(required = true)] - items: Vec, - /// The text to score; stdin when absent. - #[arg(long)] - output: Option, - #[arg(long, default_value = "http://127.0.0.1:11434")] - ollama_url: String, - #[arg(long, default_value = "qwen3-embedding:0.6b")] - model: String, -} - #[derive(Subcommand)] enum CorpusCommand { Add(Box), @@ -96,8 +68,6 @@ enum InventoryCommand { Dump, /// Admit revision 1, the standard's enumerated values and the nuance features. Seed, - /// Admit revision 2, the concepts and frames the nibu-proof-001 packet names. - Proof, } #[derive(Subcommand)] @@ -118,12 +88,9 @@ struct PutArgs { forms: Vec, #[arg(long, value_enum)] origin: OriginKind, - /// The persona of a `cast` or `live` origin; refused with any other origin. + /// The persona of a `cast` origin; refused with any other origin. #[arg(long)] persona: Option, - /// The fixture path a `cast` entity came from; refused with any other origin. - #[arg(long)] - fixture: Option, /// The work of a `public-domain` origin; refused with any other origin. #[arg(long)] work: Option, @@ -140,7 +107,9 @@ impl PutArgs { fn entity(self) -> Result { let mut forms: Vec = Vec::new(); for f in &self.forms { - let (language, form) = f.split_once('=').context("--form is not language=form")?; + let (language, form) = f + .split_once('=') + .with_context(|| format!("--form {f:?} is not language=form"))?; match forms.iter_mut().find(|lf| lf.language == language) { Some(lf) => lf.forms.push(form.nfc().collect()), None => forms.push(LanguageForms { @@ -149,19 +118,16 @@ impl PutArgs { }), } } - let origin = match (self.origin, self.persona, self.fixture, self.work) { - (OriginKind::Cast, persona, fixture, None) => EntityOrigin::Cast { + let origin = match (self.origin, self.persona, self.work) { + (OriginKind::Cast, persona, None) => EntityOrigin::Cast { persona: need(persona, "persona")?, - fixture: need(fixture, "fixture")?, }, - (OriginKind::PublicDomain, None, None, work) => EntityOrigin::PublicDomain { + (OriginKind::PublicDomain, None, work) => EntityOrigin::PublicDomain { work: need(work, "work")?, }, - (OriginKind::Live, persona, None, None) => EntityOrigin::Live { - persona: need(persona, "persona")?, - }, + (OriginKind::Live, None, None) => EntityOrigin::Live, _ => bail!( - "--fixture is only for --origin cast, --persona only for --origin cast or live and --work only for --origin public-domain" + "--persona is only for --origin cast and --work only for --origin public-domain" ), }; Ok(Entity { @@ -236,7 +202,6 @@ struct AddArgs { note: Option, #[arg(long)] work: Option, - /// Who wrote the line: for `cast-authored` and `public-domain`. #[arg(long)] author: Option, #[arg(long)] @@ -261,37 +226,10 @@ fn need(value: Option, flag: &str) -> Result { impl AddArgs { fn source(&self) -> Result { let get = |v: &Option, flag: &str| need(v.clone(), flag); - let applies: &[&str] = match self.source { - SourceKind::CastAuthored => &["persona", "fixture", "author"], - SourceKind::OperatorWritten => &[], - SourceKind::OperatorApproved => &["note"], - SourceKind::PublicDomain => &["work", "author", "edition-url"], - SourceKind::ModelWritten => &["model-family", "generator", "prompt-digest"], - }; - let given = [ - ("persona", self.persona.is_some()), - ("fixture", self.fixture.is_some()), - ("author", self.author.is_some()), - ("note", self.note.is_some()), - ("work", self.work.is_some()), - ("edition-url", self.edition_url.is_some()), - ("model-family", self.family.is_some()), - ("generator", self.generator.is_some()), - ("prompt-digest", self.prompt_digest.is_some()), - ]; - let stray: Vec = given - .into_iter() - .filter(|(flag, given)| *given && !applies.contains(flag)) - .map(|(flag, _)| format!("--{flag}")) - .collect(); - if !stray.is_empty() { - bail!("{} does not apply to this --source", stray.join(", ")); - } Ok(match self.source { SourceKind::CastAuthored => Source::CastAuthored { persona: get(&self.persona, "persona")?, fixture: get(&self.fixture, "fixture")?, - author: get(&self.author, "author")?, }, SourceKind::OperatorWritten => Source::OperatorWritten, SourceKind::OperatorApproved => Source::OperatorApproved { @@ -320,11 +258,9 @@ impl AddArgs { .map(|t| { let (speaker, text) = t .split_once('=') - .context("--prior-turn is not speaker=text")?; + .with_context(|| format!("--prior-turn {t:?} is not speaker=text"))?; Ok(Turn { - speaker: speaker.parse().map_err(|_| { - anyhow::anyhow!("--prior-turn speaker is not entity:") - })?, + speaker: speaker.parse()?, text: text.nfc().collect(), }) }) @@ -333,13 +269,13 @@ impl AddArgs { .names .iter() .map(|n| { - let bad = || anyhow::anyhow!("--name is not start..end=entity:"); - let (range, entity) = n.split_once('=').ok_or_else(bad)?; - let (start, end) = range.split_once("..").ok_or_else(bad)?; + let bad = || format!("--name {n:?} is not start..end=entity:"); + let (range, entity) = n.split_once('=').with_context(bad)?; + let (start, end) = range.split_once("..").with_context(bad)?; Ok(NameSpan { - entity: entity.parse().map_err(|_| bad())?, - start: start.parse().map_err(|_| bad())?, - end: end.parse().map_err(|_| bad())?, + entity: entity.parse()?, + start: start.parse().with_context(bad)?, + end: end.parse().with_context(bad)?, }) }) .collect::>>()?; @@ -402,7 +338,7 @@ fn main() -> Result<()> { } Command::Corpus(CorpusCommand::Add(args)) => { let mut store = CorpusStore::open(&cli.corpus)?; - let key = store.next_key(&args.slug).map_err(refused)?; + let key = store.next_key(&args.slug)?; let entities = EntityStore::open(&cli.entities)?; let outcome = store .add((*args).item(key.clone())?, &entities) @@ -464,77 +400,6 @@ fn main() -> Result<()> { .map_err(refused)?; println!("seeded rev:1 with {count} entries"); } - Command::Inventory(InventoryCommand::Proof) => { - let (rev, entries) = proof_revision(); - let count = entries.len(); - InventoryStore::open(&cli.inventory)? - .admit_revision(rev, entries) - .map_err(refused)?; - println!("admitted rev:2 with {count} entries"); - } - Command::Packet(PacketCommand::Proof { store }) => { - let packet = proof_packet(); - let key = packet.id.to_string(); - PacketStore::open(&store)? - .admit( - packet, - &InventoryStore::open(&cli.inventory)?, - &CorpusStore::open(&cli.corpus)?, - &EntityStore::open(&cli.entities)?, - ) - .map_err(refused)?; - println!("admitted {key}"); - } - Command::Lower { store, key } => { - let inventory_store = InventoryStore::open(&cli.inventory)?; - let inventory = inventory_store.inventory()?; - let mut packets = PacketStore::open(&store)?; - let packet = packets - .get(&key)? - .with_context(|| format!("no packet {key}"))?; - // Admission is the only source of a LoweringInput; re-admitting the - // stored packet is a no-op write that re-runs every door check. - let view = packets - .admit( - packet, - &inventory_store, - &CorpusStore::open(&cli.corpus)?, - &EntityStore::open(&cli.entities)?, - ) - .map_err(refused)?; - println!("{}", lower(&view, &inventory)?); - } - Command::Score(args) => { - let output = match &args.output { - Some(path) => std::fs::read_to_string(path)?, - None => std::io::read_to_string(std::io::stdin())?, - }; - let corpus = CorpusStore::open(&cli.corpus)?; - let references = args - .items - .iter() - .map(|key| { - Ok(corpus - .get(key)? - .with_context(|| format!("no item {key}"))? - .text) - }) - .collect::>>()?; - let inputs: Vec = references - .iter() - .map(|reference| ScoreInput { - reference, - output: output.trim(), - }) - .collect(); - let scorer = Scorer::new(OllamaEmbedder::new(&args.ollama_url, &args.model)); - for (key, score) in args.items.iter().zip(scorer.score(&inputs)?) { - match score.meaning { - Some(m) => println!("{key} chrf={:.2} meaning={m:.4}", score.chrf), - None => println!("{key} chrf={:.2} meaning=none", score.chrf), - } - } - } Command::Split { item_key } => { let item = CorpusStore::open(&cli.corpus)? .get(&item_key)? diff --git a/crates/weksa/src/packet.rs b/crates/weksa/src/packet.rs index cae001b..73199a2 100644 --- a/crates/weksa/src/packet.rs +++ b/crates/weksa/src/packet.rs @@ -347,86 +347,3 @@ pub struct Trace { pub notes: Vec, pub rejected_readings: Vec, } - -/// The `nibu-proof-001` packet: the meaning of `examples/english-lowering/nibu-proof-001/interlingua.yaml` -/// (warn the player off the damaged control; Nibu prevents it for them) as typed -/// ids of inventory revision 2, for the corpus item `item:nibu:1`. No features: -/// nothing here is style. -pub fn proof_packet() -> Packet { - fn id(s: &str) -> InvId { - InvId::parse(s).expect("proof ids are well formed") - } - fn node(s: &str) -> NodeId { - NodeId::parse(s).expect("proof nodes are well formed") - } - let referent = |n: &str, kind: &str, concept: &str, entity: Option<&str>| Referent { - id: node(n), - kind: id(kind), - concept: id(concept), - entity: entity.map(|e| EntityId::parse(e).expect("proof entity is well formed")), - features: vec![], - }; - let fill = |role: &str, filler: &str| RoleFill { - role: id(role), - filler: node(filler), - }; - Packet { - id: PacketId::parse("packet:6e6962752d303031").expect("proof id"), - schema_version: SCHEMA_VERSION, - inventory_rev: 2, - kind: PacketKind::DialogueLine, - provenance: Provenance { - source: ProvenanceSource::Authored, - corpus_item: CorpusItemId::parse("item:nibu:1").expect("proof item"), - encoder: EncoderId::parse("encoder:6f70657261746f72").expect("proof encoder"), - confidence: Grade::new(900).expect("proof grade"), - }, - context: Context::default(), - discourse: Discourse { - speech_act: Some(id("speech_act:5")), - addressee: Some(node("n1")), - features: vec![], - }, - referents: vec![ - referent("n1", "referent_kind:1", "concept:3", None), - referent("n2", "referent_kind:5", "concept:4", None), - referent("n3", "referent_kind:1", "concept:5", Some("entity:nibu")), - ], - predications: vec![ - Predication { - id: node("n4"), - concept: id("concept:1"), - frame: id("frame:1"), - roles: vec![ - fill("role:1", "n1"), - fill("role:4", "n2"), - fill("role:2", "n1"), - ], - features: vec![], - }, - Predication { - id: node("n5"), - concept: id("concept:2"), - frame: id("frame:2"), - roles: vec![ - fill("role:1", "n3"), - fill("role:2", "n4"), - fill("role:8", "n1"), - ], - features: vec![], - }, - ], - relations: vec![], - constraints: Constraints { - preserve: vec![node("n1"), node("n2")], - deterministic: true, - max_latency_ms: None, - }, - extensions: vec![], - trace: Trace { - notes: vec!["First proof packet for Nibu flavored-English lowering.".to_string()], - rejected_readings: vec![], - }, - stance: Stance::default(), - } -} diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index 1caa6ac..abf66fc 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -6,14 +6,13 @@ use std::collections::BTreeSet; use std::fmt; use std::path::Path; -use caseless::default_case_fold_str; use cultcache_rs::{CultCache, SingleFileMessagePackBackingStore}; -use unicode_normalization::{UnicodeNormalization, char::is_combining_mark, is_nfc}; +use unicode_normalization::is_nfc; use unicode_segmentation::UnicodeSegmentation; -use crate::corpus::{CorpusItem, CorpusItemId, Source, item_key, key_slug}; -use crate::entity::{Entity, EntityOrigin}; -use crate::form::{is_attestation, is_joining_only, is_language_tag, is_refused_format}; +use crate::corpus::{CorpusItem, CorpusItemId, item_key, key_slug}; +use crate::entity::Entity; +use crate::form::is_language_tag; use crate::inventory::{Change, InvId, Inventory, InventoryEntry, InventoryRevision, revision_key}; use crate::packet::{EntityId, Packet}; use crate::validate::{LoweringInput, Refusal, admissible}; @@ -92,12 +91,15 @@ pub enum EntityBound { Forms, /// A form of more than 64 bytes. FormBytes, + /// A form of more than 4 words. + FormWords, } impl EntityBound { pub const MAX_LANGUAGES: usize = 8; pub const MAX_FORMS: usize = 8; pub const MAX_FORM_BYTES: usize = 64; + pub const MAX_FORM_WORDS: usize = 4; } #[derive(Debug)] @@ -186,7 +188,7 @@ impl CorpusStore { } /// The next free key for `slug`; withdrawn items keep their numbers. - pub fn next_key(&self, slug: &str) -> Result { + pub fn next_key(&self, slug: &str) -> anyhow::Result { let highest = self .all()? .iter() @@ -194,10 +196,7 @@ impl CorpusStore { .filter_map(|i| i.id.rsplit_once(':')?.1.parse::().ok()) .max() .unwrap_or(0); - let next = highest - .checked_add(1) - .ok_or_else(|| malformed(slug, "slug has no free key"))?; - Ok(item_key(slug, next)) + Ok(item_key(slug, highest + 1)) } /// The item `id` names, or `UnknownCorpusItem` when it holds nothing. @@ -240,14 +239,8 @@ impl CorpusStore { if !is_nfc(&item.text) || item.prior_turns.iter().any(|t| !is_nfc(&t.text)) { return Err(StoreRefusal::NotNfc { key: item.id }.into()); } - let text = normalized(&item.text); - if !text.chars().any(char::is_alphanumeric) { - return Err(malformed(&item.id, "text has no alphanumeric character")); - } - if let Some(why) = empty_attestation(&item.source) { - return Err(malformed(&item.id, why)); - } check_entities(&item, entities)?; + let text = normalized(&item.text); for held in self.all()? { if normalized(&held.text) == text && item.supersedes.as_deref() != Some(&held.id) { return Err(StoreRefusal::S6TextHeld { @@ -360,58 +353,15 @@ fn check_entities(item: &CorpusItem, entities: &EntityStore) -> Result<(), Store Ok(()) } -/// The S6 key of a text: Unicode compatibility caseless matching (D146: NFKC, -/// default case fold, NFKC) keeping letters, digits and combining marks -/// (General_Category L, N, M) so tone marks and viramas keep words apart. Composed and decomposed, -/// fullwidth, ligature and case-fold variants (straße, STRASSE) and spacing and -/// punctuation variants compare equal. Homoglyphs across scripts (Cyrillic a for -/// Latin a) stay apart: no key here reads what a character looks like. +/// Lowercased alphanumerics only, so case, spacing and punctuation variants of +/// a text compare equal. fn normalized(text: &str) -> String { - let folded = default_case_fold_str(&text.nfkc().collect::()); - folded - .nfkc() - .filter(|c| c.is_alphanumeric() || is_combining_mark(*c)) + text.chars() + .flat_map(char::to_lowercase) + .filter(|c| c.is_alphanumeric()) .collect() } -/// The field that holds an unattested provenance string, if any: a string with -/// no alphanumeric, a control or format character, or over 200 bytes attests -/// nothing. -fn empty_attestation(source: &Source) -> Option<&'static str> { - let empty = |s: &String| !is_attestation(s); - let empty_some = |s: &Option| s.as_ref().is_some_and(empty); - match source { - Source::CastAuthored { persona, .. } if empty(persona) => { - Some("a cast-authored source names its persona") - } - Source::CastAuthored { fixture, .. } if empty(fixture) => { - Some("a cast-authored source names its fixture") - } - Source::CastAuthored { author, .. } if empty(author) => { - Some("a cast-authored source names its author") - } - Source::OperatorApproved { note } if empty(note) => { - Some("an operator-approved source carries its note") - } - Source::PublicDomain { work, .. } if empty(work) => { - Some("a public-domain source names its work") - } - Source::PublicDomain { author, .. } if empty(author) => { - Some("a public-domain source names its author") - } - Source::PublicDomain { edition_url, .. } if empty(edition_url) => { - Some("a public-domain source names its edition url") - } - Source::ModelWritten { generator, .. } if empty_some(generator) => { - Some("a model-written source names a non-empty generator") - } - Source::ModelWritten { prompt_digest, .. } if empty_some(prompt_digest) => { - Some("a model-written source carries a non-empty prompt digest") - } - _ => None, - } -} - pub struct EntityStore { cache: CultCache, } @@ -437,9 +387,8 @@ impl EntityStore { /// every stored language and form in order and origin as it was, and only /// appends: forms grow, never shrink, reorder or change. An entity has 1 to /// 8 languages, each a BCP 47-form tag named once, with 1 to 8 NFC forms of - /// 1 to 64 bytes, no control or invisible format character (joiners and - /// variation selectors stay), no edge whitespace, none repeated; the origin - /// attests where it came from with strings that hold an alphanumeric, no control or format character and at most 200 bytes. + /// 1 to 64 bytes and at most 4 words, no control character, no edge + /// whitespace, none repeated. pub fn put(&mut self, entity: Entity) -> Result { let key = entity.id.to_string(); check_entity_shape(&entity)?; @@ -472,20 +421,6 @@ fn check_entity_shape(entity: &Entity) -> Result<(), StoreError> { } .into() }; - let (persona, fixture, work) = match &entity.origin { - EntityOrigin::Cast { persona, fixture } => (Some(persona), Some(fixture), None), - EntityOrigin::Live { persona } => (Some(persona), None, None), - EntityOrigin::PublicDomain { work } => (None, None, Some(work)), - }; - if persona.is_some_and(|s| !is_attestation(s)) { - return Err(bad("an origin names its persona")); - } - if fixture.is_some_and(|s| !is_attestation(s)) { - return Err(bad("a cast origin names its fixture")); - } - if work.is_some_and(|s| !is_attestation(s)) { - return Err(bad("a public-domain origin names its work")); - } if entity.forms.is_empty() { return Err(bad("an entity has forms in at least one language")); } @@ -511,6 +446,9 @@ fn check_entity_shape(entity: &Entity) -> Result<(), StoreError> { if f.len() > EntityBound::MAX_FORM_BYTES { return Err(bound(EntityBound::FormBytes)); } + if f.split_whitespace().count() > EntityBound::MAX_FORM_WORDS { + return Err(bound(EntityBound::FormWords)); + } if !is_nfc(f) { return Err(StoreRefusal::NotNfc { key: entity.id.to_string(), @@ -522,12 +460,6 @@ fn check_entity_shape(entity: &Entity) -> Result<(), StoreError> { "a form is not empty and has no control character or edge whitespace", )); } - if f.chars().any(is_refused_format) { - return Err(bad("a form holds an invisible format character")); - } - if f.chars().all(is_joining_only) { - return Err(bad("a form is more than joiners and selectors")); - } if !seen.insert(f) { return Err(bad("a form is not repeated within its language")); } diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index e05b24a..68229fa 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -1,6 +1,6 @@ mod common; use std::process::{Command, Output}; -use weksa::{CorpusStore, EntityStore, InventoryStore, PacketStore}; +use weksa::{InventoryStore, PacketStore}; fn weksa(dir: &std::path::Path, args: &[&str]) -> Output { Command::new(env!("CARGO_BIN_EXE_weksa")) @@ -35,8 +35,6 @@ fn cast_entities(dir: &std::path::Path, slugs: &[&str]) { "cast", "--persona", slug, - "--fixture", - "f.yaml", ], ); assert!(put.status.success(), "{}", out(&put)); @@ -83,7 +81,7 @@ fn seed_add_dump_split_withdraw() { "{dump}" ); - // A cast-authored item needs its persona, fixture and author. + // A cast-authored item needs its persona and fixture. let missing = weksa(d.path(), &ADD); assert!( !missing.status.success() && out(&missing).contains("--persona"), @@ -91,14 +89,7 @@ fn seed_add_dump_split_withdraw() { out(&missing) ); let mut add = ADD.to_vec(); - add.extend([ - "--persona", - "nibu", - "--fixture", - "f.yaml", - "--author", - "Ada Writer", - ]); + add.extend(["--persona", "nibu", "--fixture", "f.yaml"]); let added = weksa(d.path(), &add); assert!( added.status.success() && out(&added).contains("Added item:nibu:1"), @@ -303,15 +294,7 @@ fn schema_writes_the_published_files() { fn entity_put_and_name_spans() { let d = tempfile::tempdir().unwrap(); let put = |args: &[&str]| { - let mut all = vec![ - "entity", - "put", - "entity:kay", - "--origin", - "cast", - "--fixture", - "f.yaml", - ]; + let mut all = vec!["entity", "put", "entity:kay", "--origin", "cast"]; all.extend_from_slice(args); weksa(d.path(), &all) }; @@ -403,59 +386,24 @@ fn entity_origin_flags_must_match_origin() { weksa(d.path(), &all) }; for mismatched in [ - &["--origin", "live"][..], - &["--origin", "live", "--persona", "kay", "--work", "edda"], - &[ - "--origin", - "live", - "--persona", - "kay", - "--fixture", - "f.yaml", - ], - &["--origin", "cast", "--persona", "kay"], - &["--origin", "cast", "--fixture", "f.yaml"], - &[ - "--origin", - "cast", - "--persona", - "kay", - "--fixture", - "f.yaml", - "--work", - "edda", - ], - &["--origin", "public-domain"], + &["--origin", "live", "--persona", "kay"][..], + &["--origin", "live", "--work", "edda"], + &["--origin", "cast", "--persona", "kay", "--work", "edda"], &[ "--origin", "public-domain", - "--work", - "edda", "--persona", "kay", - ], - &[ - "--origin", - "public-domain", "--work", "edda", - "--fixture", - "f.yaml", ], - &["--origin", "cast", "--persona", "", "--fixture", "f.yaml"], - &["--origin", "cast", "--persona", "kay", "--fixture", ""], - &["--origin", "live", "--persona", ""], - &["--origin", "public-domain", "--work", ""], + &["--origin", "public-domain", "--persona", "kay"], ] { let o = put(mismatched); assert!(!o.status.success(), "{mismatched:?}: {}", out(&o)); } assert!(!out(&weksa(d.path(), &["entity", "dump"])).contains("entity:kay")); - assert!( - put(&["--origin", "live", "--persona", "kay"]) - .status - .success() - ); + assert!(put(&["--origin", "live"]).status.success()); let work = weksa( d.path(), &[ @@ -488,8 +436,6 @@ fn intake_normalizes_to_nfc() { "fr=Rene\u{301}", "--origin", "live", - "--persona", - "rene", ], ); assert!(put.status.success(), "{}", out(&put)); @@ -521,349 +467,3 @@ fn intake_normalizes_to_nfc() { let dump = out(&weksa(d.path(), &["corpus", "dump"])); assert!(dump.contains("Ren\u{e9} part."), "{dump}"); } - -/// `corpus add` of "Mind the hatch." by `entity:nibu` under `source`, with -/// `extra` flags. -fn add_with(dir: &std::path::Path, source: &str, extra: &[&str]) -> Output { - let mut args = vec![ - "corpus", - "add", - "--slug", - "nibu", - "--text", - "Mind the hatch.", - "--speaker", - "entity:nibu", - "--group", - "scene-1", - "--admitted-on", - "2026-10-29", - "--source", - source, - ]; - args.extend_from_slice(extra); - weksa(dir, &args) -} - -#[test] -fn source_flags_must_match_the_source() { - let d = tempfile::tempdir().unwrap(); - cast_entities(d.path(), &["nibu"]); - // A provenance flag that does not apply to --source refuses, naming the flag. - for (source, extra, flag) in [ - ("operator-written", &["--persona", "p"][..], "--persona"), - ( - "operator-approved", - &["--note", "n", "--work", "w"], - "--work", - ), - ( - "public-domain", - &[ - "--work", - "w", - "--author", - "a", - "--edition-url", - "u", - "--note", - "n", - ], - "--note", - ), - ( - "cast-authored", - &[ - "--persona", - "p", - "--fixture", - "f", - "--author", - "a", - "--generator", - "g", - ], - "--generator", - ), - ( - "model-written", - &["--model-family", "unrecorded", "--edition-url", "u"], - "--edition-url", - ), - ( - "operator-written", - &["--model-family", "unrecorded"], - "--model-family", - ), - ( - "model-written", - &[ - "--model-family", - "unrecorded", - "--prompt-digest", - "d", - "--fixture", - "f", - ], - "--fixture", - ), - // A source missing a flag it needs. - ( - "cast-authored", - &["--persona", "p", "--fixture", "f"], - "--author", - ), - ( - "public-domain", - &["--work", "w", "--edition-url", "u"], - "--author", - ), - // Empty provenance strings. - ( - "model-written", - &["--model-family", "unrecorded", "--generator", ""], - "", - ), - ( - "model-written", - &["--model-family", "unrecorded", "--prompt-digest", ""], - "", - ), - ( - "cast-authored", - &["--persona", "", "--fixture", "f", "--author", "a"], - "", - ), - ("operator-approved", &["--note", ""], ""), - ] { - let o = add_with(d.path(), source, extra); - assert!(!o.status.success(), "{source} {extra:?}: {}", out(&o)); - assert!(out(&o).contains(flag), "{source} {extra:?}: {}", out(&o)); - } - assert!(!out(&weksa(d.path(), &["corpus", "dump"])).contains("item:")); - // A source whose flags all apply is admitted; --author reaches a - // cast-authored item. - let cast = add_with( - d.path(), - "cast-authored", - &["--persona", "p", "--fixture", "f", "--author", "Ada Writer"], - ); - assert!(cast.status.success(), "{}", out(&cast)); - let dump = out(&weksa(d.path(), &["corpus", "dump"])); - assert!(dump.contains("author: \"Ada Writer\""), "{dump}"); -} - -#[test] -fn malformed_intake_never_echoes_its_values() { - const CANARY: &str = "CANARY-7f3a"; - let d = tempfile::tempdir().unwrap(); - cast_entities(d.path(), &["nibu"]); - let shape = |value: &str| format!("{CANARY}{value}"); - let puts = [ - (vec![shape("")], "--form"), - (vec![shape("_x=Kay")], ""), - (vec![format!("en-US={CANARY}\u{7}")], ""), - (vec![format!("{CANARY}_x=Kay")], ""), - ]; - for (forms, flag) in puts { - let mut args = vec![ - "entity", - "put", - "entity:kay", - "--origin", - "live", - "--persona", - "kay", - ]; - for f in &forms { - args.extend(["--form", f]); - } - let o = weksa(d.path(), &args); - assert!(!o.status.success(), "{forms:?}: {}", out(&o)); - assert!(!out(&o).contains(CANARY), "{forms:?}: {}", out(&o)); - assert!(out(&o).contains(flag), "{forms:?}: {}", out(&o)); - } - let bad_turn = [shape(""), format!("{CANARY}=hello")]; - let bad_name = [ - shape(""), - format!("0..1={CANARY}"), - format!("{CANARY}..1=entity:nibu"), - format!("0..{CANARY}=entity:nibu"), - ]; - let cases = bad_turn - .iter() - .map(|t| ("--prior-turn", t)) - .chain(bad_name.iter().map(|n| ("--name", n))); - for (flag, value) in cases { - let o = add_with(d.path(), "operator-written", &[flag, value]); - assert!(!o.status.success(), "{flag} {value}: {}", out(&o)); - assert!(!out(&o).contains(CANARY), "{flag} {value}: {}", out(&o)); - assert!(out(&o).contains(flag), "{flag} {value}: {}", out(&o)); - } -} - -#[test] -fn every_decomposed_form_and_turn_is_stored_composed() { - let d = tempfile::tempdir().unwrap(); - cast_entities(d.path(), &["nibu"]); - let put = weksa( - d.path(), - &[ - "entity", - "put", - "entity:rene", - "--form", - "fr=Rene\u{301}", - "--form", - "fr=Zoe\u{308}", - "--origin", - "live", - "--persona", - "rene", - ], - ); - assert!(put.status.success(), "{}", out(&put)); - let rene = EntityStore::open(&d.path().join("e.cc")) - .unwrap() - .get(&"entity:rene".parse().unwrap()) - .unwrap() - .unwrap(); - assert_eq!(rene.forms[0].forms, ["Ren\u{e9}", "Zo\u{eb}"]); - let add = add_with( - d.path(), - "operator-written", - &["--prior-turn", "entity:rene=Cafe\u{301} now"], - ); - assert!(add.status.success(), "{}", out(&add)); - let item = CorpusStore::open(&d.path().join("c.cc")) - .unwrap() - .get("item:nibu:1") - .unwrap() - .unwrap(); - assert_eq!(item.prior_turns[0].text, "Caf\u{e9} now"); -} - -/// An Ollama stand-in: every text embeds to `[1, its length]`, so identical -/// texts have cosine 1. -fn fake_embedder() -> String { - use std::io::{Read, Write}; - let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); - let url = format!("http://{}", listener.local_addr().unwrap()); - std::thread::spawn(move || { - for mut stream in listener.incoming().flatten() { - let mut buf = vec![0u8; 65536]; - let mut n = 0; - let body = loop { - n += stream.read(&mut buf[n..]).unwrap(); - let text = String::from_utf8_lossy(&buf[..n]).into_owned(); - if let Some((head, body)) = text.split_once("\r\n\r\n") { - let want: usize = head - .to_lowercase() - .split("content-length: ") - .nth(1) - .unwrap() - .split("\r\n") - .next() - .unwrap() - .parse() - .unwrap(); - if body.len() >= want { - break body.to_string(); - } - } - }; - let body: serde_json::Value = serde_json::from_str(&body).unwrap(); - let vectors: Vec> = body["input"] - .as_array() - .unwrap() - .iter() - .map(|t| vec![1.0, t.as_str().unwrap().len() as f64]) - .collect(); - let reply = serde_json::json!({ "embeddings": vectors }).to_string(); - write!( - stream, - "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{reply}", - reply.len() - ) - .unwrap(); - } - }); - url -} - -#[test] -fn proof_packet_lowers_and_the_lowering_scores_against_items() { - let d = tempfile::tempdir().unwrap(); - cast_entities(d.path(), &["nibu"]); - for args in [&["inventory", "seed"][..], &["inventory", "proof"]] { - let run = weksa(d.path(), args); - assert!(run.status.success(), "{}", out(&run)); - } - let item = weksa( - d.path(), - &[ - "corpus", - "add", - "--slug", - "nibu", - "--text", - "Warning: you may endanger yourself by touching the control. The ship mind will prevent that to protect you.", - "--speaker", - "entity:nibu", - "--group", - "scene-1", - "--admitted-on", - "2026-11-01", - "--source", - "operator-written", - ], - ); - assert!(item.status.success(), "{}", out(&item)); - let admitted = weksa(d.path(), &["packet", "proof", "./p.cc"]); - assert!( - admitted.status.success() && out(&admitted).contains("admitted packet:6e6962752d303031"), - "{}", - out(&admitted) - ); - let lowered = weksa(d.path(), &["lower", "./p.cc", "packet:6e6962752d303031"]); - assert!(lowered.status.success(), "{}", out(&lowered)); - let text = String::from_utf8(lowered.stdout).unwrap(); - assert!( - text.starts_with("Warning: you may endanger yourself"), - "{text}" - ); - - let missing = weksa(d.path(), &["lower", "./p.cc", "packet:0000000000000000"]); - assert!(!missing.status.success() && out(&missing).contains("no packet")); - - std::fs::write(d.path().join("out.txt"), &text).unwrap(); - let url = fake_embedder(); - let scored = weksa( - d.path(), - &[ - "score", - "item:nibu:1", - "--output", - "out.txt", - "--ollama-url", - &url, - ], - ); - assert!(scored.status.success(), "{}", out(&scored)); - assert_eq!( - out(&scored).trim(), - "item:nibu:1 chrf=100.00 meaning=1.0000" - ); - let unknown = weksa( - d.path(), - &[ - "score", - "item:nibu:9", - "--output", - "out.txt", - "--ollama-url", - &url, - ], - ); - assert!(!unknown.status.success() && out(&unknown).contains("no item item:nibu:9")); -} diff --git a/crates/weksa/tests/common/mod.rs b/crates/weksa/tests/common/mod.rs index 07e95ee..aecf5d0 100644 --- a/crates/weksa/tests/common/mod.rs +++ b/crates/weksa/tests/common/mod.rs @@ -67,7 +67,6 @@ pub fn cast() -> Source { Source::CastAuthored { persona: "nibu".into(), fixture: "f.yaml".into(), - author: "Ada Writer".into(), } } @@ -96,7 +95,6 @@ pub fn ent(slug: &str, language: &str, forms: &[&str]) -> Entity { }], origin: EntityOrigin::Cast { persona: slug.into(), - fixture: "f.yaml".into(), }, } } diff --git a/crates/weksa/tests/entities.rs b/crates/weksa/tests/entities.rs index 46aeb71..4f8f2f2 100644 --- a/crates/weksa/tests/entities.rs +++ b/crates/weksa/tests/entities.rs @@ -105,11 +105,7 @@ fn entity_forms_only_grow() { }, |e| e.forms.reverse(), |e| e.forms[1].language = "fi".into(), - |e| { - e.origin = EntityOrigin::Live { - persona: "kay".into(), - } - }, + |e| e.origin = EntityOrigin::Live, ]; assert_eq!( refusal(store.put(ent("kay", "en-US", &["Kay"]))), diff --git a/crates/weksa/tests/entity_door.rs b/crates/weksa/tests/entity_door.rs index bcd53e8..a0b3e9e 100644 --- a/crates/weksa/tests/entity_door.rs +++ b/crates/weksa/tests/entity_door.rs @@ -163,33 +163,28 @@ fn entity_capacity_is_bounded() { let make = |forms: Vec| Entity { id: eid("big"), forms, - origin: EntityOrigin::Live { - persona: "p".into(), - }, + origin: EntityOrigin::Live, }; - // A form is bounded by bytes, not words: a spaced name of any word count and - // names in scripts that write no spaces are admitted (the Thai name counts - // five UAX #29 words, a Han name one per character). - for (i, name) in [ - "I will never leave you, not ever.", - "Mary of the Isles too", - "de la Cruz y Garcia", - "\u{e2a}\u{e21}\u{e0a}\u{e32}\u{e22} \u{e43}\u{e08}\u{e14}\u{e35}", - "\u{5c71}\u{7530}\u{592a}\u{90ce}", - ] - .into_iter() - .enumerate() - { - w.entities - .put(Entity { - id: eid(&format!("name-{i}")), - ..make(vec![lf("en-US", one(name))]) - }) - .unwrap(); - } - // 64 bytes is the edge; 65 is over, counted in bytes (21 Han characters are - // 63 bytes, 22 are 66). + // A whole line is not a name: 7 words. + let line = "I will never leave you, not ever."; + let e = make(vec![lf("en-US", one(line))]); + assert_eq!( + refusal(w.entities.put(e.clone())), + bounds(&e, EntityBound::FormWords) + ); + // Four words is the edge; five is over. 64 bytes is the edge; 65 is over. + w.entities + .put(Entity { + id: eid("four"), + ..make(vec![lf("en-US", one("Mary of the Isles"))]) + }) + .unwrap(); + let five = make(vec![lf("en-US", one("Mary of the Isles too"))]); + assert_eq!( + refusal(w.entities.put(five.clone())), + bounds(&five, EntityBound::FormWords) + ); let wide = make(vec![lf("en-US", one(&"a".repeat(65)))]); assert_eq!( refusal(w.entities.put(wide.clone())), @@ -201,17 +196,6 @@ fn entity_capacity_is_bounded() { ..make(vec![lf("en-US", one(&"a".repeat(64)))]) }) .unwrap(); - w.entities - .put(Entity { - id: eid("han-edge"), - ..make(vec![lf("ja", one(&"\u{5c71}".repeat(21)))]) - }) - .unwrap(); - let han = make(vec![lf("ja", one(&"\u{5c71}".repeat(22)))]); - assert_eq!( - refusal(w.entities.put(han.clone())), - bounds(&han, EntityBound::FormBytes) - ); // 8 forms per language is the edge. let forms = |n: usize| (0..n).map(|i| format!("form{i}")).collect::>(); @@ -337,7 +321,6 @@ fn entities_are_append_only_and_replay_needs_them() { let mut persona = base.clone(); persona.origin = EntityOrigin::Cast { persona: "someone-else".into(), - fixture: "f.yaml".into(), }; assert_eq!(refusal(w.entities.put(persona)), not_append); let mut duplicate = base.clone(); @@ -363,276 +346,3 @@ fn entities_are_append_only_and_replay_needs_them() { } ); } - -/// One entity holding `form` as its only en-US form. -fn with_form(slug: &str, form: &str) -> Entity { - Entity { - id: eid(slug), - forms: vec![LanguageForms { - language: "en-US".into(), - forms: vec![form.into()], - }], - origin: EntityOrigin::Live { - persona: "p".into(), - }, - } -} - -#[test] -fn invisible_format_characters_are_refused_in_forms() { - let mut w = world(); - let hidden = |slug: &str| StoreRefusal::Malformed { - key: format!("entity:{slug}"), - why: "a form holds an invisible format character", - }; - let at = |code: u32| char::from_u32(code).unwrap(); - // Every Cf code point except the joiners, plus U+2028 and U+2029, listed - // from Python's unicodedata. The code-point sweep in form.rs pins the edges - // of the rule; this checks the door applies it, in each position. - let listed: Vec = include_str!("format_refused.txt") - .lines() - .filter(|l| !l.starts_with('#')) - .map(|l| u32::from_str_radix(l, 16).unwrap()) - .collect(); - assert_eq!(listed.len(), 170); - for (n, code) in listed.iter().enumerate() { - let c = at(*code); - let mut forms = vec![format!("Ka{c}y")]; - if !c.is_whitespace() { - forms.extend([format!("{c}Kay"), format!("Kay{c}")]); - } - for form in forms { - assert_eq!( - refusal(w.entities.put(with_form("fmt", &form))), - hidden("fmt"), - "U+{code:X} (#{n}) in {form:?}" - ); - } - } - // Default_Ignorable fillers that are not Cf: Hangul and Khmer. - for code in [0x115f, 0x1160, 0x17b4, 0x17b5, 0x3164, 0xffa0] { - assert_eq!( - refusal( - w.entities - .put(with_form("fmt", &format!("Ka{}y", at(code)))) - ), - hidden("fmt"), - "U+{code:X}" - ); - } - assert!(w.entities.get(&eid("fmt")).unwrap().is_none()); - - // The neighbours of the refused ranges, ZWNJ, ZWJ, CGJ, variation - // selectors and the Mongolian selectors (all Mn) are admitted in a form. - let admitted: [u32; 33] = [ - 0xac, - 0xae, - 0x34f, - 0x5ff, - 0x606, - 0x61b, - 0x61d, - 0x6dc, - 0x6de, - 0x70e, - 0x710, - 0x88f, - 0x892, - 0x8e1, - 0x8e3, - 0x180b, - 0x180c, - 0x180d, - 0x180f, - 0x200a, - 0x200c, - 0x200d, - 0x2010, - 0x202f, - 0x2065, - 0x2070, - 0xfe00, - 0xfe0f, - 0xfeff - 1, - 0xfffc, - 0x110be, - 0x1d172, - 0xe0100, - ]; - for (n, code) in admitted.into_iter().enumerate() { - let form = format!("Ka{}y", at(code)); - w.entities - .put(with_form(&format!("ok-{n}"), &form)) - .unwrap_or_else(|e| panic!("U+{code:X}: {e}")); - } - // ZWNJ in a Persian form, ZWJ in an emoji sequence, VS16 and a Mongolian - // free variation selector beside a letter. - for (n, form) in [ - "\u{645}\u{6cc}\u{200c}\u{62e}\u{648}\u{627}\u{647}\u{645}", - "\u{1f469}\u{200d}\u{1f4bb}", - "\u{2764}\u{fe0f}", - "\u{1820}\u{180b}", - "\u{1f600}\u{e0100}", - ] - .into_iter() - .enumerate() - { - w.entities - .put(with_form(&format!("script-{n}"), form)) - .unwrap(); - } - // A form of only joiners, variation selectors, CGJ or Mongolian selectors - // shows nothing and is refused. - let empty_looking = |slug: &str| StoreRefusal::Malformed { - key: format!("entity:{slug}"), - why: "a form is more than joiners and selectors", - }; - for form in [ - "\u{200d}", - "\u{200c}", - "\u{200c}\u{200d}", - "\u{fe0f}", - "\u{fe00}\u{fe0f}", - "\u{34f}", - "\u{180b}\u{180c}\u{180d}\u{180f}", - "\u{e0100}", - "\u{e01ef}\u{34f}\u{200d}", - ] { - assert_eq!( - refusal(w.entities.put(with_form("blank", form))), - empty_looking("blank"), - "{form:?}" - ); - } - assert!(w.entities.get(&eid("blank")).unwrap().is_none()); -} - -#[test] -fn an_origin_attests_where_the_entity_came_from() { - let mut w = world(); - let bad = |slug: &str, why: &'static str| StoreRefusal::Malformed { - key: format!("entity:{slug}"), - why, - }; - let with = |slug: &str, origin: EntityOrigin| Entity { - origin, - ..with_form(slug, "Kay") - }; - let cast = |persona: &str, fixture: &str| EntityOrigin::Cast { - persona: persona.into(), - fixture: fixture.into(), - }; - assert_eq!( - refusal(w.entities.put(with("a", cast("", "f.yaml")))), - bad("a", "an origin names its persona") - ); - assert_eq!( - refusal(w.entities.put(with("b", cast("kay", "")))), - bad("b", "a cast origin names its fixture") - ); - assert_eq!( - refusal(w.entities.put(with( - "c", - EntityOrigin::Live { - persona: String::new() - } - ))), - bad("c", "an origin names its persona") - ); - assert_eq!( - refusal(w.entities.put(with( - "d", - EntityOrigin::PublicDomain { - work: String::new() - } - ))), - bad("d", "a public-domain origin names its work") - ); - for slug in ["a", "b", "c", "d"] { - assert!(w.entities.get(&eid(slug)).unwrap().is_none()); - } - w.entities.put(with("e", cast("kay", "f.yaml"))).unwrap(); - w.entities - .put(with( - "f", - EntityOrigin::PublicDomain { - work: "edda".into(), - }, - )) - .unwrap(); -} - -#[test] -fn an_origin_attestation_holds_something_readable() { - let mut w = world(); - let with = |slug: &str, origin: EntityOrigin| Entity { - origin, - ..with_form(slug, "Kay") - }; - type Build = fn(String) -> EntityOrigin; - let origins: [(Build, &str); 4] = [ - ( - |v| EntityOrigin::Cast { - persona: v, - fixture: "f.yaml".into(), - }, - "an origin names its persona", - ), - ( - |v| EntityOrigin::Cast { - persona: "kay".into(), - fixture: v, - }, - "a cast origin names its fixture", - ), - ( - |v| EntityOrigin::Live { persona: v }, - "an origin names its persona", - ), - ( - |v| EntityOrigin::PublicDomain { work: v }, - "a public-domain origin names its work", - ), - ]; - let unreadable = [ - " ".to_string(), - "\t".into(), - "\n".into(), - "\u{200b}".into(), - "\u{200d}".into(), - "\0".into(), - "--".into(), - "ok\u{200b}ok".into(), - "ok\u{2028}ok".into(), - "ok\0ok".into(), - "x".repeat(201), - "x".repeat(2 * 1024 * 1024), - ]; - let mut n = 0; - for (build, why) in &origins { - for bad in &unreadable { - n += 1; - let slug = format!("o-{n}"); - assert_eq!( - refusal(w.entities.put(with(&slug, build(bad.clone())))), - StoreRefusal::Malformed { - key: format!("entity:{slug}"), - why - }, - "{why}: {:?}", - bad.chars().take(12).collect::() - ); - assert!(w.entities.get(&eid(&slug)).unwrap().is_none()); - } - } - for (m, (build, _)) in origins.iter().enumerate() { - for (k, good) in ["x".repeat(200), "two words, 2 more".to_string()] - .into_iter() - .enumerate() - { - w.entities - .put(with(&format!("g-{}", 2 * m + k), build(good))) - .unwrap(); - } - } -} diff --git a/crates/weksa/tests/format_refused.txt b/crates/weksa/tests/format_refused.txt deleted file mode 100644 index b994f95..0000000 --- a/crates/weksa/tests/format_refused.txt +++ /dev/null @@ -1,171 +0,0 @@ -# Cf minus ZWNJ/ZWJ, plus U+2028 and U+2029; Python 3.13 unicodedata 15.1.0 over 0..10FFFF -AD -600 -601 -602 -603 -604 -605 -61C -6DD -70F -890 -891 -8E2 -180E -200B -200E -200F -2028 -2029 -202A -202B -202C -202D -202E -2060 -2061 -2062 -2063 -2064 -2066 -2067 -2068 -2069 -206A -206B -206C -206D -206E -206F -FEFF -FFF9 -FFFA -FFFB -110BD -110CD -13430 -13431 -13432 -13433 -13434 -13435 -13436 -13437 -13438 -13439 -1343A -1343B -1343C -1343D -1343E -1343F -1BCA0 -1BCA1 -1BCA2 -1BCA3 -1D173 -1D174 -1D175 -1D176 -1D177 -1D178 -1D179 -1D17A -E0001 -E0020 -E0021 -E0022 -E0023 -E0024 -E0025 -E0026 -E0027 -E0028 -E0029 -E002A -E002B -E002C -E002D -E002E -E002F -E0030 -E0031 -E0032 -E0033 -E0034 -E0035 -E0036 -E0037 -E0038 -E0039 -E003A -E003B -E003C -E003D -E003E -E003F -E0040 -E0041 -E0042 -E0043 -E0044 -E0045 -E0046 -E0047 -E0048 -E0049 -E004A -E004B -E004C -E004D -E004E -E004F -E0050 -E0051 -E0052 -E0053 -E0054 -E0055 -E0056 -E0057 -E0058 -E0059 -E005A -E005B -E005C -E005D -E005E -E005F -E0060 -E0061 -E0062 -E0063 -E0064 -E0065 -E0066 -E0067 -E0068 -E0069 -E006A -E006B -E006C -E006D -E006E -E006F -E0070 -E0071 -E0072 -E0073 -E0074 -E0075 -E0076 -E0077 -E0078 -E0079 -E007A -E007B -E007C -E007D -E007E -E007F diff --git a/crates/weksa/tests/lower.rs b/crates/weksa/tests/lower.rs deleted file mode 100644 index f50f096..0000000 --- a/crates/weksa/tests/lower.rs +++ /dev/null @@ -1,212 +0,0 @@ -//! The English lowerer against the committed inventory (revision 2) and the -//! `nibu-proof-001` packet, and the rules that keep a table row from speaking -//! for a meaning it does not carry. - -mod common; -use std::path::Path; - -use common::{iv, nd}; -use weksa::inventory::proof_revision; -use weksa::packet::{FeatValue, FeatureValue, Packet, proof_packet}; -use weksa::{ - CorpusStore, EntityStore, Inventory, InventoryStore, LowerError, LoweringInput, PacketStore, - lower, validate, -}; - -const PROOF: &str = "Warning: you may endanger yourself by touching the control. \ - The ship mind will prevent that to protect you."; - -fn committed() -> (Inventory, tempfile::TempDir) { - let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); - let inv = InventoryStore::open(&root.join("data/interlingua/inventory.cc")).unwrap(); - (inv.inventory().unwrap(), tempfile::tempdir().unwrap()) -} - -/// A `LoweringInput` has one source, `PacketStore::admit`, so every packet -/// under test goes through a throwaway store backed by the committed stores. -fn view(packet: &Packet, _inv: &Inventory) -> LoweringInput { - let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); - let inventory = InventoryStore::open(&root.join("data/interlingua/inventory.cc")).unwrap(); - admitted(packet, &inventory) -} - -/// The same door against an inventory store the test built. -fn admitted(packet: &Packet, inventory: &InventoryStore) -> LoweringInput { - let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); - let corpus = CorpusStore::open(&root.join("data/corpus/corpus.cc")).unwrap(); - let entities = EntityStore::open(&root.join("data/entity/entity.cc")).unwrap(); - let dir = tempfile::tempdir().unwrap(); - PacketStore::open(&dir.path().join("p.cc")) - .unwrap() - .admit(packet.clone(), inventory, &corpus, &entities) - .unwrap() -} - -fn lowered(packet: &Packet, inv: &Inventory) -> Result { - lower(&view(packet, inv), inv) -} - -#[test] -fn the_proof_packet_is_admitted_by_the_store_and_lowers_to_its_sentences() { - let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); - let (inv, dir) = committed(); - let inventory = InventoryStore::open(&root.join("data/interlingua/inventory.cc")).unwrap(); - let corpus = CorpusStore::open(&root.join("data/corpus/corpus.cc")).unwrap(); - let entities = EntityStore::open(&root.join("data/entity/entity.cc")).unwrap(); - let mut packets = PacketStore::open(&dir.path().join("p.cc")).unwrap(); - packets - .admit(proof_packet(), &inventory, &corpus, &entities) - .unwrap(); - let stored = packets.get("packet:6e6962752d303031").unwrap().unwrap(); - assert_eq!(lowered(&stored, &inv).unwrap(), PROOF); -} - -#[test] -fn only_the_addressee_is_you_and_is_reflexive() { - let (inv, _dir) = committed(); - let mut p = proof_packet(); - p.discourse.addressee = None; - assert_eq!( - lowered(&p, &inv).unwrap(), - "Warning: the player may endanger the player by touching the control. \ - The ship mind will prevent that to protect the player." - ); -} - -#[test] -fn no_speech_act_means_no_opener_and_a_capital() { - let (inv, _dir) = committed(); - let mut p = proof_packet(); - p.discourse.speech_act = None; - assert!(lowered(&p, &inv).unwrap().starts_with("You may endanger")); - p.discourse.speech_act = Some(iv("speech_act:1")); - assert!(lowered(&p, &inv).unwrap().starts_with("You may endanger")); -} - -#[test] -fn a_packet_carrying_what_the_lowerer_does_not_render_is_refused() { - let dir = tempfile::tempdir().unwrap(); - let mut store = InventoryStore::open(&dir.path().join("i.cc")).unwrap(); - let (rev1, entries) = weksa::inventory::seed(); - store.admit_revision(rev1, entries).unwrap(); - let (rev2, entries) = proof_revision(); - store.admit_revision(rev2, entries).unwrap(); - let mut ext = common::entry("extension:1", 3); - ext.value_kind = Some(weksa::ValueKind::Grade); - common::admit(&mut store, 3, vec![ext]); - let inv = store.inventory().unwrap(); - let discourse = |p: &mut Packet| { - p.discourse.features.push(FeatureValue { - feature: iv("feature:6"), - value: FeatValue::Grade(common::grade(300)), - }) - }; - let predication = |p: &mut Packet| { - p.predications[0].features.push(FeatureValue { - feature: iv("feature:1"), - value: FeatValue::Option(common::lab("negative")), - }) - }; - let relation = |p: &mut Packet| { - p.relations.push(weksa::packet::Relation { - relation: iv("relation:8"), - from: nd("n4"), - to: nd("n5"), - features: vec![], - }) - }; - let extension = |p: &mut Packet| { - p.inventory_rev = 3; - p.extensions.push(weksa::packet::Extension { - key: iv("extension:1"), - value: FeatValue::Grade(common::grade(3)), - }) - }; - let referent_feature = |p: &mut Packet| { - p.referents[0].features.push(FeatureValue { - feature: iv("feature:6"), - value: FeatValue::Grade(common::grade(1)), - }) - }; - let no_predication = |p: &mut Packet| p.predications.clear(); - let extra_role = |p: &mut Packet| { - let filler = p.predications[0].roles[0].filler.clone(); - p.predications[0].roles.push(weksa::packet::RoleFill { - role: iv("role:3"), - filler, - }) - }; - let lone_referent = |p: &mut Packet| { - let mut r = p.referents[0].clone(); - r.id = nd("n99"); - p.referents.push(r) - }; - for (what, edit) in [ - ("features", &discourse as &dyn Fn(&mut Packet)), - ("features", &predication), - ("relations", &relation), - ("extensions", &extension), - ("no predication", &no_predication), - ("a role the frame does not name", &extra_role), - ("a referent no role mentions", &lone_referent), - ] { - let mut p = proof_packet(); - edit(&mut p); - assert_eq!( - lower(&admitted(&p, &store), &inv), - Err(LowerError::Unrendered { what }), - "{what}" - ); - } - // The seed lets no feature sit on a referent, so that arm is refused by validate. - let mut p = proof_packet(); - referent_feature(&mut p); - assert!(validate(&p, &inv).is_err()); -} - -#[test] -fn an_unknown_row_and_an_unfilled_role_are_refused_by_name() { - let (inv, _dir) = committed(); - let mut p = proof_packet(); - p.discourse.speech_act = Some(iv("speech_act:3")); - assert_eq!( - lowered(&p, &inv), - Err(LowerError::NoRow { - id: iv("speech_act:3") - }) - ); - let mut p = proof_packet(); - p.predications[0].roles.pop(); - assert_eq!( - lowered(&p, &inv), - Err(LowerError::MissingRole { - predication: nd("n4"), - role: "patient".into() - }) - ); -} - -#[test] -fn a_row_cannot_speak_for_an_id_whose_gloss_differs() { - // Revision 2 with concept:4 glossed "hatch" in place of "control": the row - // for concept:4 says "control" and must not render it. - let dir = tempfile::tempdir().unwrap(); - let mut store = InventoryStore::open(&dir.path().join("i.cc")).unwrap(); - let (rev1, entries) = weksa::inventory::seed(); - store.admit_revision(rev1, entries).unwrap(); - let (rev2, mut entries) = proof_revision(); - for e in &mut entries { - if e.id == iv("concept:4") { - e.gloss = "hatch".into(); - } - } - store.admit_revision(rev2, entries).unwrap(); - let inv = store.inventory().unwrap(); - assert_eq!( - lower(&admitted(&proof_packet(), &store), &inv), - Err(LowerError::GlossMismatch { - id: iv("concept:4"), - row: "control" - }) - ); -} diff --git a/crates/weksa/tests/split_schema.rs b/crates/weksa/tests/split_schema.rs index d1e638a..d69c5cb 100644 --- a/crates/weksa/tests/split_schema.rs +++ b/crates/weksa/tests/split_schema.rs @@ -22,7 +22,6 @@ fn cast() -> Source { Source::CastAuthored { persona: "p".into(), fixture: "f".into(), - author: "a".into(), } } diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs index 07cbe0c..c21e49a 100644 --- a/crates/weksa/tests/stores.rs +++ b/crates/weksa/tests/stores.rs @@ -144,17 +144,6 @@ fn corpus_keys_are_canonical() { } // An alias of a stored key is not a corpus item id at all. assert!(weksa::CorpusItemId::parse("item:nibu:01").is_err()); - // The last free number is handed out; past u32::MAX the slug has no free - // key, and asking neither wraps nor panics. - store - .add(item("item:b:4294967294", "one before the last"), &ents(&d)) - .unwrap(); - assert_eq!(store.next_key("b").unwrap(), "item:b:4294967295"); - let full = "a".repeat(48); - assert_eq!( - refusal(store.next_key(&full)), - malformed(&full, "slug has no free key") - ); } #[test] @@ -275,322 +264,6 @@ fn corpus_admission_rules() { ); } -#[test] -fn s6_reads_text_under_compatibility_case_folding() { - let d = dir(); - let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); - // Each pair is one text to the key: fullwidth, ligature, superscript and - // circled-digit compatibility forms and full case folds (straße, STRASSE). - let pairs = [ - ("Brokkr", "\u{ff22}\u{ff52}\u{ff4f}\u{ff4b}\u{ff4b}\u{ff52}"), - ("fine", "\u{fb01}ne"), - ("room 2", "room \u{b2}"), - ("gate 1", "gate \u{2460}"), - ("stra\u{df}e", "STRASSE"), - ( - "\u{3a3}\u{3af}\u{3c3}\u{3c5}\u{3c6}\u{3bf}\u{3c2}", - "\u{3c3}\u{3af}\u{3c3}\u{3c5}\u{3c6}\u{3bf}\u{3c3}", - ), - // The first NFKC step: a compatibility letter folds only after it is - // mapped (math-bold S has no case fold of its own; black-letter H is - // not a case variant of h until NFKC reads it as H). - ( - "sigrun", - "\u{1d412}\u{1d422}\u{1d420}\u{1d42b}\u{1d42e}\u{1d427}", - ), - ("h", "\u{210c}"), - // The second NFKC step: a full case fold can leave a decomposed or - // reordered sequence that only a second NFKC composes to the same key - // (dialytika tonos, caron under line, macron below). - ("\u{390}", "\u{3aa}\u{301}"), - ("\u{3b0}", "\u{3ab}\u{301}"), - ("\u{1e96}\u{30c}", "\u{21e}\u{331}"), - ]; - for (n, (a, b)) in pairs.into_iter().enumerate() { - let (first, second) = ( - format!("item:s6:{}", 2 * n + 1), - format!("item:s6:{}", 2 * n + 2), - ); - store.add(item(&first, a), &ents(&d)).unwrap(); - assert_eq!( - refusal(store.add(item(&second, b), &ents(&d))), - StoreRefusal::S6TextHeld { - key: second.clone(), - held_by: first.clone() - }, - "{a:?} vs {b:?}" - ); - assert!(store.get(&second).unwrap().is_none()); - } - // A decomposed text never reaches the key: the door refuses it as not NFC. - assert_eq!( - refusal(store.add(item("item:s6:90", "Bro\u{308}kkr"), &ents(&d))), - StoreRefusal::NotNfc { - key: "item:s6:90".into() - } - ); - // Cyrillic a is not Latin a: homoglyphs across scripts stay apart. - store.add(item("item:s6:91", "cat"), &ents(&d)).unwrap(); - store - .add(item("item:s6:92", "c\u{430}t"), &ents(&d)) - .unwrap(); -} - -#[test] -fn s6_keeps_tone_marks_and_viramas_apart() { - let d = dir(); - let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); - // Minimal pairs that differ only by a combining mark: Thai mai tho and mai - // ek on one word, Thai tone on a short vowel, a Devanagari virama. - let pairs = [ - ("\u{e19}\u{e49}\u{e33}", "\u{e19}\u{e33}"), - ("\u{e44}\u{e21}\u{e49}", "\u{e44}\u{e21}\u{e48}"), - ("\u{915}\u{94d}\u{92f}", "\u{915}\u{92f}"), - ]; - for (n, (a, b)) in pairs.into_iter().enumerate() { - store - .add(item(&format!("item:mk:{}", 2 * n + 1), a), &ents(&d)) - .unwrap_or_else(|e| panic!("{a:?}: {e}")); - store - .add(item(&format!("item:mk:{}", 2 * n + 2), b), &ents(&d)) - .unwrap_or_else(|e| panic!("{b:?} held by {a:?}: {e}")); - } - // A mark alone is still no text. - for (n, text) in ["\u{e49}", "\u{94d}"].into_iter().enumerate() { - let key = format!("item:mk:{}", 90 + n); - assert_eq!( - refusal(store.add(item(&key, text), &ents(&d))), - malformed(&key, "text has no alphanumeric character") - ); - } -} - -#[test] -fn an_attestation_holds_something_readable() { - let d = dir(); - let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); - let s = |x: &str| x.to_string(); - type Build = fn(String) -> Source; - let builders: [(Build, &'static str); 9] = [ - ( - |v| Source::CastAuthored { - persona: v, - fixture: "f".into(), - author: "a".into(), - }, - "a cast-authored source names its persona", - ), - ( - |v| Source::CastAuthored { - persona: "p".into(), - fixture: v, - author: "a".into(), - }, - "a cast-authored source names its fixture", - ), - ( - |v| Source::CastAuthored { - persona: "p".into(), - fixture: "f".into(), - author: v, - }, - "a cast-authored source names its author", - ), - ( - |v| Source::OperatorApproved { note: v }, - "an operator-approved source carries its note", - ), - ( - |v| Source::PublicDomain { - work: v, - author: "a".into(), - edition_url: "u".into(), - }, - "a public-domain source names its work", - ), - ( - |v| Source::PublicDomain { - work: "w".into(), - author: v, - edition_url: "u".into(), - }, - "a public-domain source names its author", - ), - ( - |v| Source::PublicDomain { - work: "w".into(), - author: "a".into(), - edition_url: v, - }, - "a public-domain source names its edition url", - ), - ( - |v| Source::ModelWritten { - family: ModelFamily::Unrecorded, - generator: Some(v), - prompt_digest: None, - }, - "a model-written source names a non-empty generator", - ), - ( - |v| Source::ModelWritten { - family: ModelFamily::Unrecorded, - generator: None, - prompt_digest: Some(v), - }, - "a model-written source carries a non-empty prompt digest", - ), - ]; - let unreadable = [ - s(" "), - s("\t"), - s("\n"), - s("\u{200b}"), - s("\u{200d}"), - s("\0"), - s("--"), - s("\u{e49}"), - s("ok\u{200b}ok"), - s("ok\u{200d}ok"), - s("ok\u{2028}ok"), - s("ok\nok"), - s("ok\0ok"), - "x".repeat(201), - "x".repeat(2 * 1024 * 1024), - ]; - let mut n = 0; - for (build, why) in &builders { - for bad in &unreadable { - n += 1; - let key = format!("item:att:{n}"); - let refused = item_in(&key, &format!("line {n}"), "scene-1", build(bad.clone())); - assert_eq!( - refusal(store.add(refused, &ents(&d))), - malformed(&key, why), - "{why}: {:?}", - bad.chars().take(12).collect::() - ); - } - } - assert!(store.all().unwrap().is_empty()); - // 200 bytes and plain words with spaces attest; 201 does not. - for (m, (build, _)) in builders.iter().enumerate() { - for (k, good) in ["x".repeat(200), s("two words, 2 more")] - .into_iter() - .enumerate() - { - let key = format!("item:att-ok:{}", 2 * m + k + 1); - store - .add( - item_in(&key, &format!("fine {m} {k}"), "scene-1", build(good)), - &ents(&d), - ) - .unwrap_or_else(|e| panic!("{key}: {e}")); - } - } -} - -#[test] -fn a_text_with_no_alphanumeric_character_is_refused() { - let d = dir(); - let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); - for (n, text) in ["...", "?!", " ", "\u{2014}\u{2014}", "\u{ff01}"] - .into_iter() - .enumerate() - { - let key = format!("item:p:{}", n + 1); - assert_eq!( - refusal(store.add(item(&key, text), &ents(&d))), - malformed(&key, "text has no alphanumeric character"), - "{text:?}" - ); - } - assert!(store.all().unwrap().is_empty()); - store.add(item("item:p:1", "a..."), &ents(&d)).unwrap(); -} - -#[test] -fn an_item_source_attests_with_non_empty_strings() { - let d = dir(); - let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); - let s = |x: &str| x.to_string(); - let cast = |persona: &str, fixture: &str, author: &str| Source::CastAuthored { - persona: s(persona), - fixture: s(fixture), - author: s(author), - }; - let model = |generator: Option<&str>, digest: Option<&str>| Source::ModelWritten { - family: ModelFamily::Unrecorded, - generator: generator.map(s), - prompt_digest: digest.map(s), - }; - let public = |work: &str, author: &str, url: &str| Source::PublicDomain { - work: s(work), - author: s(author), - edition_url: s(url), - }; - let cases: [(Source, &'static str); 9] = [ - ( - cast("", "f", "a"), - "a cast-authored source names its persona", - ), - ( - cast("p", "", "a"), - "a cast-authored source names its fixture", - ), - ( - cast("p", "f", ""), - "a cast-authored source names its author", - ), - ( - Source::OperatorApproved { note: s("") }, - "an operator-approved source carries its note", - ), - ( - public("", "a", "u"), - "a public-domain source names its work", - ), - ( - public("w", "", "u"), - "a public-domain source names its author", - ), - ( - public("w", "a", ""), - "a public-domain source names its edition url", - ), - ( - model(Some(""), None), - "a model-written source names a non-empty generator", - ), - ( - model(None, Some("")), - "a model-written source carries a non-empty prompt digest", - ), - ]; - for (n, (source, why)) in cases.into_iter().enumerate() { - let key = format!("item:prov:{}", n + 1); - let refused = item_in(&key, &format!("line {n}"), "scene-1", source); - assert_eq!(refusal(store.add(refused, &ents(&d))), malformed(&key, why)); - } - assert!(store.all().unwrap().is_empty()); - // The same sources with every string present are admitted. - for (n, source) in [ - cast("p", "f", "a"), - Source::OperatorApproved { note: s("n") }, - Source::OperatorWritten, - public("w", "a", "u"), - model(Some("g"), Some("d")), - model(None, None), - ] - .into_iter() - .enumerate() - { - let key = format!("item:prov:{}", n + 1); - let item = item_in(&key, &format!("line {n}"), "scene-1", source); - assert_eq!(store.add(item, &ents(&d)).unwrap(), AddOutcome::Added); - } -} - #[test] fn s5_supersede_keeps_the_group() { let d = dir(); @@ -1233,23 +906,6 @@ fn the_committed_stores_load_and_hold_clean_data() { "no item claims a recorded human author" ); assert_eq!(items.len(), 3); - // Every committed entity attests, with non-empty strings, where it came - // from, and a cast entity's fixture is a file of this repository. - let entities = EntityStore::open(&root.join("data/entity/entity.cc")) - .unwrap() - .all() - .unwrap(); - assert!(!entities.is_empty()); - for e in &entities { - match &e.origin { - weksa::entity::EntityOrigin::Cast { persona, fixture } => { - assert!(!persona.is_empty(), "{}", e.id); - assert!(root.join(fixture).is_file(), "{}: {fixture}", e.id); - } - weksa::entity::EntityOrigin::Live { persona } => assert!(!persona.is_empty()), - weksa::entity::EntityOrigin::PublicDomain { work } => assert!(!work.is_empty()), - } - } } #[test] @@ -1270,16 +926,14 @@ fn seed_reseeded() { .all() .unwrap(); assert_eq!(items.len(), 3); - // The committed inventory is the seed, reseeded (stance features in, the - // displayed-affect features out), then the nibu-proof revision. + // The committed inventory is the seed, reseeded: stance features in, the + // displayed-affect features out. let committed = InventoryStore::open(&root.join("data/interlingua/inventory.cc")) .unwrap() .inventory() .unwrap(); - let (rev, mut entries) = seed(); - let (proof, proof_entries) = weksa::inventory::proof_revision(); - assert!(committed.has_revision(rev.n) && committed.has_revision(proof.n)); - entries.extend(proof_entries); + let (rev, entries) = seed(); + assert!(committed.has_revision(rev.n)); let mut have: Vec<_> = committed.entries().cloned().collect(); let mut want = entries; have.sort_by_key(|e| e.id); diff --git a/data/entity/entity.cc b/data/entity/entity.cc index 8f9a198..f561201 100644 --- a/data/entity/entity.cc +++ b/data/entity/entity.cc @@ -1 +1 @@ -cultcache.store.v1weksa.entityweksa.entityweksa.entity.v1weksa.entityL{"schemaName":"weksa.entity","schemaVersion":"weksa.entity.v1","members":[]}weksa.entityentity:brokkrweksa.entity2026-10-29T16:43:28Znentity:brokkren-USBrokkrCastbrokkr?examples/persona-lowering/brokkr-edda-voice/lowered-output.yamlentity:nibuweksa.entity2026-10-29T16:43:28Zeentity:nibuen-USNibuCastnibu`, an opaque digest of item, inventory revision and encoder (the item key lives in `provenance.corpus_item`, which the lowerer never sees); CultCache type `weksa.interlingua_packet`, carrying `schema_version` and `inventory_rev`. Every id inside is an opaque form type (`n` nodes, `encoder:<16 hex>`, canonical `:`). From cut `speaker-state` (schema v4) it also carries the utterance **stance**: the speaker's felt affect and restraint, as inventory features of slot `Stance` (see Speaker state below) | Minted by an encoder for one corpus item under one spec version; immutable; a new spec version means a new packet, never an edit. Admitted only through `PacketStore::admit`, which validates against the inventory store at the packet's revision | The encoder writes; the form types refuse malformed ids at construction and load, the validator refuses broken references and over-capacity packets: a feature at most once per slot, at most 64 nodes (`typed-packet`); nothing else writes packets | +| Packet | Key `packet:<16 hex>`, an opaque digest of item, inventory revision and encoder (the item key lives in `provenance.corpus_item`, which the lowerer never sees); CultCache type `weksa.interlingua_packet`, carrying `schema_version` and `inventory_rev`. Every id inside is an opaque form type (`n` nodes, `encoder:<16 hex>`, canonical `:`). From cut `speaker-state` (schema v2) it also carries the utterance **stance**: the speaker's felt affect and restraint, as inventory features of slot `Stance` (see Speaker state below) | Minted by an encoder for one corpus item under one spec version; immutable; a new spec version means a new packet, never an edit. Admitted only through `PacketStore::admit`, which validates against the inventory store at the packet's revision | The encoder writes; the form types refuse malformed ids at construction and load, the validator refuses broken references and over-capacity packets: a feature at most once per slot, at most 64 nodes (`typed-packet`); nothing else writes packets | | Schema version | `weksa.interlingua_packet.v` constant in `crates/weksa`; published JSON Schema at `schemas/cultnet/` | Bumped by a commit that changes the Rust types; the published schema is regenerated in the same commit and a test fails on drift | A cut (Hands, under a spec); never the loop | -| Inventory entry | Key `:`, `n` canonical decimal >= 1 (families: concept, frame, role, relation, speech_act, feature, referent_kind, extension); type `weksa.inventory_entry`, with a bounded gloss, value kind (option labels bounded), the slots a feature may sit in (`Discourse`, `Referent`, `Predication`, `Relation`; `Stance` from cut `speaker-state`, `Speaker` and `Scene` from cut `speaker-scene`), `introduced_rev` and optional `retired_rev`. Speaker axes, scene relations and affects are features of those slots, so the loop grows them like any closed-class family and convergence covers them | Added or retired by an inventory revision; never edited in place (a changed meaning is a retire plus an add); a retired id still resolves for packets of earlier revisions | The loop's accepted rounds, whose minting call the context builder assembles from Train items only; Self on merge to main. Admission checks form, not text | +| Inventory entry | Key `:`, `n` canonical decimal >= 1 (families: concept, frame, role, relation, speech_act, feature, referent_kind, extension); type `weksa.inventory_entry`, with a bounded gloss, value kind (option labels bounded), the slots a feature may sit in (`Discourse`, `Referent`, `Predication`, `Relation`; from cut `speaker-state` also `Stance`, `Speaker`, `Scene`), `introduced_rev` and optional `retired_rev`. Speaker axes, scene relations and affects are features of those slots, so the loop grows them like any closed-class family and convergence covers them | Added or retired by an inventory revision; never edited in place (a changed meaning is a retire plus an add); a retired id still resolves for packets of earlier revisions | The loop's accepted rounds, whose minting call the context builder assembles from Train items only; Self on merge to main. Admission checks form, not text | | Inventory revision | Key `rev:`, type `weksa.inventory_revision`, with parent, change list and `frozen` | Appended per accepted round; exactly one revision is ever marked frozen, by the convergence test | The loop's accept rule appends; the convergence test freezes; Self records the freeze as a ruling | | Target profile | Locale or language-project id (`en-US`, `ja-JP`, a conlang's slug) plus a revision `profile::` | Authored, revised by commit or by an accepted round that changed the lowering; each lowering names the revision it used | Profile authors and accepted rounds; never the lowerer's defaults (`persona-drives-register`) | | Language ontology entry (cut `language-ontology`) | Key `onto::`; type `weksa.ontology_entry`: conditions over feature values (stance, speaker, scene, packet features) and community ids, and the surface choice they select (layer: lexicon, grammar, discourse, phonology, orthography; a bounded descriptor and at most three bounded examples); introduced and retired by a profile revision | Added or retired by a profile revision, never edited; converted from `data/target-language-ontology/*.yaml`, which is deleted in the same cut so the YAML is not a second owner | The language's owner (the operator for her settings' languages) and accepted rounds; an ontology-minting call is assembled from Train items only, like an inventory-minting call, and its text is in the scorer's leak diagnostic. Selection is deterministic code (`select(profile, stance, speaker, scene)`); the lowerer realizes the selected choices and the trace lists their ids (`lowering-traced`) | | Community (cut `language-ontology`) | Key `community:`; type `weksa.community`, converted from `data/cultural-ontology/cultures/` and `referents/` (YAML deleted in the same cut) | Revised by commit; a speaker names the communities it belongs to by id | The cultural ontology's author (owner-attested evidence only; Discord-only content stays deleted, `discord-only-content-deleted`) | -| Speaker (cut `speaker-scene`) | Key `speaker:@`; type `weksa.speaker`: stable axes as feature values of slot `Speaker` (origin, socioeconomic standing, age cohort, education and whatever the loop adds), at an inventory revision, and an `Annotator` | A new revision appends, never edits; a corpus item and every prior turn name a speaker slug, and a round records the speaker revisions it read | The annotator it names: `PersonaSnapshot` (a projection of a `gamecult.persona_state.v0` document, naming its source document and `updatedAt`), `Operator`, `WorkMetadata` (a public-domain work's own cast list and narration), or `Model { family, generator }` (the 2b `ModelFamily`) | -| Scene (cut `speaker-scene`) | Key `scene:@`, the item's split group; type `weksa.scene`: participants (speaker slugs) and, per ordered pair, relation features of slot `Scene` (relative status, intimacy, in-group, audience and whatever the loop adds), at an inventory revision, and an `Annotator` | As Speaker. Keyed by the group, so a scene never straddles the split | As Speaker | +| Speaker (cut `speaker-state`) | Key `speaker:@`; type `weksa.speaker`: stable axes as feature values of slot `Speaker` (origin, socioeconomic standing, age cohort, education and whatever the loop adds), at an inventory revision, and an `Annotator` | A new revision appends, never edits; a corpus item and every prior turn name a speaker slug, and a round records the speaker revisions it read | The annotator it names: `PersonaSnapshot` (a projection of a `gamecult.persona_state.v0` document, naming its source document and `updatedAt`), `Operator`, `WorkMetadata` (a public-domain work's own cast list and narration), or `Model { family, generator }` (the 2b `ModelFamily`) | +| Scene (cut `speaker-state`) | Key `scene:@`, the item's split group; type `weksa.scene`: participants (speaker slugs) and, per ordered pair, relation features of slot `Scene` (relative status, intimacy, in-group, audience and whatever the loop adds), at an inventory revision, and an `Annotator` | As Speaker. Keyed by the group, so a scene never straddles the split | As Speaker | | Corpus item | Key `item::`, `n` canonical decimal >= 1 (no sign, no leading zero, from cut 2b); type `weksa.corpus_item` with text, language, speaker, split group, prior turns, source. Source `ModelWritten` carries a `ModelFamily` enum, the generator string as the fixture wrote it, and a prompt digest (cut 2b). The item carries no speaker state of its own: its speaker slug and group name the Speaker and Scene documents | Admitted once; the whole item is immutable (re-adding an identical item is a no-op, any difference is refused; a correction is a new item that `supersedes` the old); text already held under another key is refused unless the new item supersedes that key (cut 2b); withdrawn, never deleted, so old scores stay explainable | Importers and `weksa corpus add`, under `corpus-cast-plus-public-domain`; the source enum has no variant for Discord or any training-forbidden source (`corpus-clean`) | | Split assignment | Derived: `split(scheme, item.group)` | Fixed for a scheme; a new scheme is a new constant, never a reassignment. A model-written item whose group derives test is in no split; a superseding item keeps its predecessor's group | The hash function. Nobody assigns by hand, so there is no writer to forbid | | Loop round | Key `run: