From 20865cac596931677404ba6681680822fe3226c2 Mon Sep 17 00:00:00 2001 From: Eureka Date: Thu, 29 Oct 2026 17:14:06 +0100 Subject: [PATCH] entity-store (2c): referents cite entity ids; typed entity store with bounded NFC forms per language; corpus name spans on grapheme boundaries; required CorpusItemId; packet schema v3 (squash of weksa-lowering/entity-store 29d671e..ea2c96c; Soul s1, s2) Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 23 + README.md | 8 +- crates/weksa/Cargo.toml | 2 + crates/weksa/src/corpus.rs | 55 ++- crates/weksa/src/entity.rs | 51 +++ crates/weksa/src/form.rs | 26 ++ crates/weksa/src/lib.rs | 11 +- crates/weksa/src/main.rs | 132 +++++- crates/weksa/src/packet.rs | 104 ++--- crates/weksa/src/schema.rs | 2 + crates/weksa/src/store.rs | 292 +++++++++++- crates/weksa/src/validate.rs | 10 +- crates/weksa/tests/cli.rs | 221 ++++++++- crates/weksa/tests/common/mod.rs | 62 ++- crates/weksa/tests/entities.rs | 419 ++++++++++++++++++ crates/weksa/tests/entity_door.rs | 348 +++++++++++++++ crates/weksa/tests/forms.rs | 98 +--- crates/weksa/tests/seeds.rs | 38 ++ crates/weksa/tests/split_schema.rs | 5 +- crates/weksa/tests/stores.rs | 197 +++++--- crates/weksa/tests/validate.rs | 4 +- data/corpus/corpus.cc | 2 +- data/entity/entity.cc | 1 + docs/interlingua-standard.md | 28 +- docs/targets/interlingua-lowering-map.md | 6 +- schemas/cultnet/weksa.corpus_item.schema.json | 42 +- schemas/cultnet/weksa.entity.schema.json | 103 +++++ .../weksa.interlingua_packet.schema.json | 44 +- schemas/interlingua-packet.md | 5 +- 29 files changed, 1999 insertions(+), 340 deletions(-) create mode 100644 crates/weksa/src/entity.rs create mode 100644 crates/weksa/tests/entities.rs create mode 100644 crates/weksa/tests/entity_door.rs create mode 100644 crates/weksa/tests/seeds.rs create mode 100644 data/entity/entity.cc create mode 100644 schemas/cultnet/weksa.entity.schema.json diff --git a/Cargo.lock b/Cargo.lock index 4f5d537..3aa8a3e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -659,12 +659,33 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + [[package]] name = "unicode-ident" version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" +[[package]] +name = "unicode-normalization" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75cd3fb03d5288b3ba9a1ee103ac7e9cbb59f0b03d64f8ddfcc04dd42758a85d" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + [[package]] name = "utf8parse" version = "0.2.2" @@ -738,6 +759,8 @@ dependencies = [ "serde", "serde_json", "tempfile", + "unicode-normalization", + "unicode-segmentation", ] [[package]] diff --git a/README.md b/README.md index eae6668..b599515 100644 --- a/README.md +++ b/README.md @@ -18,11 +18,11 @@ measured. - [Interlingua Standard](./docs/interlingua-standard.md): the packet contract. - `crates/weksa`: the packet, inventory and corpus types, `validate`, the - derived split and the `weksa` CLI (`weksa validate`, `corpus`, `inventory`, + derived split and the `weksa` CLI (`weksa validate`, `corpus`, `entity`, `inventory`, `split`, `schema`); published schemas in `schemas/cultnet/`. -- `data/corpus/corpus.cc` and `data/interlingua/inventory.cc`: the English - corpus and the inventory, read with `weksa corpus dump` and - `weksa inventory dump`. +- `data/corpus/corpus.cc`, `data/entity/entity.cc` and `data/interlingua/inventory.cc`: the English + corpus, its entities and the inventory, read with `weksa corpus dump`, + `weksa entity dump` and `weksa inventory dump`. - [Lowering target](./docs/targets/interlingua-lowering.md) and [its map](./docs/targets/interlingua-lowering-map.md): the design, scoring and cut sequence. diff --git a/crates/weksa/Cargo.toml b/crates/weksa/Cargo.toml index f3ae47d..75b5ac6 100644 --- a/crates/weksa/Cargo.toml +++ b/crates/weksa/Cargo.toml @@ -19,6 +19,8 @@ cultcache-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "64b86e0 schemars = "1" serde = { version = "1", features = ["derive"] } serde_json = "1" +unicode-normalization = "0.1" +unicode-segmentation = "1" [dev-dependencies] tempfile = "3" diff --git a/crates/weksa/src/corpus.rs b/crates/weksa/src/corpus.rs index 2b04c11..403cdb8 100644 --- a/crates/weksa/src/corpus.rs +++ b/crates/weksa/src/corpus.rs @@ -6,9 +6,12 @@ use cultcache_rs::DatabaseEntry; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use crate::form::{FormError, is_slug, string_form}; +use crate::packet::EntityId; + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub struct Turn { - pub speaker: String, + pub speaker: EntityId, pub text: String, } @@ -62,7 +65,7 @@ pub struct CorpusItem { #[cultcache(key = 2)] pub language: String, #[cultcache(key = 3)] - pub speaker: String, + pub speaker: EntityId, /// The unit the split hashes: a scene, a fixture, a work. #[cultcache(key = 4)] pub group: String, @@ -77,8 +80,48 @@ pub struct CorpusItem { pub supersedes: Option, #[cultcache(key = 9)] pub withdrawn: bool, + /// Which entities the text names, and where. Fixed with the text at intake. + #[cultcache(key = 10)] + pub names: Vec, } +/// A byte range `[start, end)` of an item's text that names an entity. The +/// store admits it only when both ends sit on extended grapheme cluster +/// boundaries (UAX #29) of the NFC text and the range's bytes equal one of the +/// entity's forms in the item's language. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct NameSpan { + pub entity: EntityId, + pub start: u32, + pub end: u32, +} + +/// The key of a corpus item, `item::` as `key_slug` reads it. A packet +/// names the item it was minted for by this type, so the key cannot be free text. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct CorpusItemId(String); + +impl CorpusItemId { + pub fn parse(s: &str) -> Result { + key_slug(s) + .map(|_| Self(s.to_string())) + .ok_or_else(|| FormError::new("CorpusItemId", s)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Display for CorpusItemId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +string_form!(CorpusItemId, { "type": "string", "pattern": "^item:[a-z0-9]([a-z0-9-]{0,46}[a-z0-9])?:[1-9][0-9]*$" }); + /// `item::`. pub fn item_key(slug: &str, n: u32) -> String { format!("item:{slug}:{n}") @@ -92,11 +135,5 @@ pub fn key_slug(key: &str) -> Option<&str> { let (slug, n) = rest.rsplit_once(':')?; let canonical_n = n.bytes().all(|b| b.is_ascii_digit()) && !n.starts_with('0') && n.parse::().is_ok(); - let b = slug.as_bytes(); - let slug_ok = (1..=48).contains(&b.len()) - && b.iter() - .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || *c == b'-') - && b[0] != b'-' - && b[b.len() - 1] != b'-'; - (canonical_n && slug_ok).then_some(slug) + (canonical_n && is_slug(slug)).then_some(slug) } diff --git a/crates/weksa/src/entity.rs b/crates/weksa/src/entity.rs new file mode 100644 index 0000000..4c51e84 --- /dev/null +++ b/crates/weksa/src/entity.rs @@ -0,0 +1,51 @@ +//! The entity store's documents. An entity is a named thing a packet may cite by +//! id. Its forms per language are data, citation form first, then inflected or +//! particle-bound forms; no rule here reads what a form means, only its shape. + +use cultcache_rs::DatabaseEntry; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::packet::EntityId; + +/// An entity's forms in one language, as the corpus item language tag spells it. +/// The citation form comes first. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct LanguageForms { + pub language: String, + /// 1 to 64 bytes each, no control character, no edge whitespace, none repeated. + pub forms: Vec, +} + +/// Where an entity was minted. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub enum EntityOrigin { + /// From a cast member's fixture. + Cast { persona: String }, + /// From a public-domain work. + PublicDomain { work: String }, + /// Minted live by a persona before it names the entity. + Live, +} + +/// Key: the entity id. Forms only grow: see `EntityStore::put`. +#[derive(Clone, Debug, PartialEq, Eq, JsonSchema, DatabaseEntry)] +#[cultcache(type = "weksa.entity")] +pub struct Entity { + #[cultcache(key = 0)] + pub id: EntityId, + #[cultcache(key = 1)] + pub forms: Vec, + #[cultcache(key = 2)] + pub origin: EntityOrigin, +} + +impl Entity { + /// The forms stored for `language`, if any. + pub fn forms_in(&self, language: &str) -> Option<&[String]> { + self.forms + .iter() + .find(|f| f.language == language) + .map(|f| f.forms.as_slice()) + } +} diff --git a/crates/weksa/src/form.rs b/crates/weksa/src/form.rs index bbde59e..d8336fb 100644 --- a/crates/weksa/src/form.rs +++ b/crates/weksa/src/form.rs @@ -60,3 +60,29 @@ macro_rules! string_form { }; } pub(crate) use string_form; + +/// A BCP 47-form language tag: 1 to 35 bytes of ASCII alphanumerics in +/// hyphen-separated subtags of 1 to 8 bytes; the first is 2 to 8 letters, or the +/// single letter `x` of a private-use tag. Form only: no registry is consulted. +pub(crate) fn is_language_tag(s: &str) -> bool { + if s.len() > 35 { + return false; + } + let mut subtags = s.split('-'); + let first = subtags.next().unwrap_or(""); + let first_ok = first.bytes().all(|b| b.is_ascii_alphabetic()) + && ((2..=8).contains(&first.len()) || first == "x"); + first_ok + && subtags + .all(|t| (1..=8).contains(&t.len()) && t.bytes().all(|b| b.is_ascii_alphanumeric())) +} + +/// A slug: `[a-z0-9]` and inner `-`, 1 to 48 bytes, no edge `-`. +pub(crate) fn is_slug(s: &str) -> bool { + let b = s.as_bytes(); + (1..=48).contains(&b.len()) + && b.iter() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || *c == b'-') + && b[0] != b'-' + && b[b.len() - 1] != b'-' +} diff --git a/crates/weksa/src/lib.rs b/crates/weksa/src/lib.rs index 0be0bcc..8aa7384 100644 --- a/crates/weksa/src/lib.rs +++ b/crates/weksa/src/lib.rs @@ -3,6 +3,7 @@ //! train/validation/test split, and the stores that admit them. pub mod corpus; +pub mod entity; pub mod form; pub mod inventory; pub mod packet; @@ -11,15 +12,19 @@ pub mod split; pub mod store; pub mod validate; -pub use corpus::{CorpusItem, Source, Turn}; +pub use corpus::{CorpusItem, CorpusItemId, NameSpan, Source, Turn}; +pub use entity::{Entity, EntityOrigin, LanguageForms}; pub use form::FormError; pub use inventory::{ Change, Family, InvId, Inventory, InventoryEntry, InventoryRevision, OptionLabel, OptionSet, Slot, ValueKind, }; pub use packet::{ - EncoderId, Grade, NodeId, Packet, PacketId, PacketKind, ProperName, SCHEMA_VERSION, + EncoderId, EntityId, Grade, NodeId, Packet, PacketId, PacketKind, SCHEMA_VERSION, }; pub use split::{Split, SplitScheme}; -pub use store::{AddOutcome, CorpusStore, InventoryStore, PacketStore, StoreError, StoreRefusal}; +pub use store::{ + AddOutcome, CorpusStore, EntityBound, EntityStore, InventoryStore, PacketStore, StoreError, + StoreRefusal, +}; pub use validate::{LoweringInput, Refusal, validate}; diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index e86c2f1..48fce8f 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -5,9 +5,14 @@ use std::path::PathBuf; use anyhow::{Context, Result, bail}; use clap::{Args, Parser, Subcommand, ValueEnum}; -use weksa::corpus::{CorpusItem, ModelFamily, Source, Turn}; +use unicode_normalization::UnicodeNormalization; +use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn}; +use weksa::entity::{Entity, EntityOrigin, LanguageForms}; use weksa::inventory::seed; -use weksa::{CorpusStore, InventoryStore, PacketStore, SplitScheme, StoreError, validate}; +use weksa::{ + CorpusStore, EntityId, EntityStore, InventoryStore, PacketStore, SplitScheme, StoreError, + validate, +}; #[derive(Parser)] #[command( @@ -21,6 +26,9 @@ struct Cli { /// The inventory store. #[arg(long, global = true, default_value = "data/interlingua/inventory.cc")] inventory: PathBuf, + /// The entity store. + #[arg(long, global = true, default_value = "data/entity/entity.cc")] + entities: PathBuf, #[command(subcommand)] command: Command, } @@ -33,6 +41,8 @@ enum Command { Corpus(CorpusCommand), #[command(subcommand)] Inventory(InventoryCommand), + #[command(subcommand)] + Entity(EntityCommand), /// Print the split of a corpus item. Split { item_key: String }, /// Write the published JSON Schemas. @@ -60,6 +70,74 @@ enum InventoryCommand { Seed, } +#[derive(Subcommand)] +enum EntityCommand { + /// Admit an entity, or append forms to a stored one. + Put(PutArgs), + /// Text projection of the entity store for review; not a store. + Dump, +} + +#[derive(Args)] +struct PutArgs { + /// `entity:`. + id: String, + /// `language=form`, repeated; a language's forms keep the order given, + /// citation form first. Forms are NFC-normalized here. + #[arg(long = "form", required = true)] + forms: Vec, + #[arg(long, value_enum)] + origin: OriginKind, + /// The persona of a `cast` origin; refused with any other origin. + #[arg(long)] + persona: Option, + /// The work of a `public-domain` origin; refused with any other origin. + #[arg(long)] + work: Option, +} + +#[derive(Clone, Copy, ValueEnum)] +enum OriginKind { + Cast, + PublicDomain, + Live, +} + +impl PutArgs { + fn entity(self) -> Result { + let mut forms: Vec = Vec::new(); + for f in &self.forms { + let (language, form) = f + .split_once('=') + .with_context(|| format!("--form {f:?} is not language=form"))?; + match forms.iter_mut().find(|lf| lf.language == language) { + Some(lf) => lf.forms.push(form.nfc().collect()), + None => forms.push(LanguageForms { + language: language.to_string(), + forms: vec![form.nfc().collect()], + }), + } + } + let origin = match (self.origin, self.persona, self.work) { + (OriginKind::Cast, persona, None) => EntityOrigin::Cast { + persona: need(persona, "persona")?, + }, + (OriginKind::PublicDomain, None, work) => EntityOrigin::PublicDomain { + work: need(work, "work")?, + }, + (OriginKind::Live, None, None) => EntityOrigin::Live, + _ => bail!( + "--persona is only for --origin cast and --work only for --origin public-domain" + ), + }; + Ok(Entity { + id: self.id.parse()?, + forms, + origin, + }) + } +} + #[derive(Clone, Copy, ValueEnum)] enum SourceKind { CastAuthored, @@ -98,12 +176,17 @@ impl From for ModelFamily { struct AddArgs { #[arg(long)] slug: String, + /// NFC-normalized here; `--name` byte offsets index the normalized text. #[arg(long)] text: String, #[arg(long, default_value = "en-US")] language: String, + /// An entity id. #[arg(long)] - speaker: String, + speaker: EntityId, + /// `start..end=entity:`, repeated: a byte range of the text naming an entity. + #[arg(long = "name")] + names: Vec, #[arg(long)] group: String, /// `speaker=text`, repeated, oldest first. @@ -177,15 +260,29 @@ impl AddArgs { .split_once('=') .with_context(|| format!("--prior-turn {t:?} is not speaker=text"))?; Ok(Turn { - speaker: speaker.to_string(), - text: text.to_string(), + speaker: speaker.parse()?, + text: text.nfc().collect(), + }) + }) + .collect::>>()?; + let names = self + .names + .iter() + .map(|n| { + let bad = || format!("--name {n:?} is not start..end=entity:"); + let (range, entity) = n.split_once('=').with_context(bad)?; + let (start, end) = range.split_once("..").with_context(bad)?; + Ok(NameSpan { + entity: entity.parse()?, + start: start.parse().with_context(bad)?, + end: end.parse().with_context(bad)?, }) }) .collect::>>()?; Ok(CorpusItem { id, source: self.source()?, - text: self.text, + text: self.text.nfc().collect(), language: self.language, speaker: self.speaker, group: self.group, @@ -193,6 +290,7 @@ impl AddArgs { admitted_on: self.admitted_on, supersedes: self.supersedes, withdrawn: false, + names, }) } } @@ -241,7 +339,10 @@ fn main() -> Result<()> { Command::Corpus(CorpusCommand::Add(args)) => { let mut store = CorpusStore::open(&cli.corpus)?; let key = store.next_key(&args.slug)?; - let outcome = store.add((*args).item(key.clone())?).map_err(refused)?; + let entities = EntityStore::open(&cli.entities)?; + let outcome = store + .add((*args).item(key.clone())?, &entities) + .map_err(refused)?; println!("{outcome:?} {key}"); } Command::Corpus(CorpusCommand::Dump) => { @@ -249,8 +350,8 @@ fn main() -> Result<()> { let split = split_text(&i); let withdrawn = if i.withdrawn { " withdrawn" } else { "" }; println!( - "{} [{} {} group={} {split}{withdrawn}]\n {:?}\n source: {:?}", - i.id, i.language, i.speaker, i.group, i.text, i.source + "{} [{} {} group={} {split}{withdrawn}]\n {:?}\n names: {:?}\n source: {:?}", + i.id, i.language, i.speaker, i.group, i.text, i.names, i.source ); } } @@ -278,6 +379,19 @@ fn main() -> Result<()> { ); } } + Command::Entity(EntityCommand::Put(args)) => { + let entity = args.entity()?; + let id = entity.id.clone(); + let outcome = EntityStore::open(&cli.entities)? + .put(entity) + .map_err(refused)?; + println!("{outcome:?} {id}"); + } + Command::Entity(EntityCommand::Dump) => { + for e in EntityStore::open(&cli.entities)?.all()? { + println!("{} {:?}\n forms: {:?}", e.id, e.origin, e.forms); + } + } Command::Inventory(InventoryCommand::Seed) => { let (rev, entries) = seed(); let count = entries.len(); diff --git a/crates/weksa/src/packet.rs b/crates/weksa/src/packet.rs index cb56672..1708707 100644 --- a/crates/weksa/src/packet.rs +++ b/crates/weksa/src/packet.rs @@ -1,9 +1,8 @@ //! The interlingua packet. Every id is an opaque form type: a node id `n`, a //! packet id `packet:`, an encoder id `encoder:`, an inventory id -//! `:`. The free text is a bounded [`ProperName`] and the trace, which -//! is stripped before lowering. A malformed id or name cannot be built, and -//! cannot be read back from a store, so a packet cannot carry target-language -//! wording through an id slot. +//! `:`. The only free text is the trace, which is stripped before +//! lowering. A malformed id cannot be built, and cannot be read back from a +//! store, so a packet cannot carry target-language wording through an id slot. use std::fmt; @@ -11,12 +10,13 @@ use cultcache_rs::DatabaseEntry; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use crate::form::{FormError, string_form}; +use crate::corpus::CorpusItemId; +use crate::form::{FormError, is_slug, string_form}; pub use crate::inventory::InvId; use crate::inventory::OptionLabel; /// Bumped by a commit that changes these types; the published schema moves with it. -pub const SCHEMA_VERSION: u32 = 1; +pub const SCHEMA_VERSION: u32 = 3; /// Id of a referent or predication within one packet: `n`, `k` a canonical /// decimal 1..=9999. @@ -109,6 +109,33 @@ impl fmt::Display for EncoderId { string_form!(EncoderId, { "type": "string", "pattern": "^encoder:[0-9a-f]{16}$" }); +/// Id of an entity: `entity:`, a slug as an item key's (`[a-z0-9]` and +/// inner `-`, 1 to 48 bytes, no edge `-`). +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct EntityId(String); + +impl EntityId { + pub fn parse(s: &str) -> Result { + match s.strip_prefix("entity:") { + Some(slug) if is_slug(slug) => Ok(Self(s.to_string())), + _ => Err(FormError::new("EntityId", s)), + } + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for EntityId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +string_form!(EntityId, { "type": "string", "pattern": "^entity:[a-z0-9]([a-z0-9-]{0,46}[a-z0-9])?$" }); + /// Permille, 0..=1000. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] #[serde(try_from = "u16", into = "u16")] @@ -152,60 +179,6 @@ impl JsonSchema for Grade { } } -/// A proper name: 1 to 4 tokens joined by single spaces, 64 bytes at most. A -/// token is 1 to 24 letters, apostrophes (`'` or U+2019) and hyphens, beginning -/// and ending with a letter. The form bounds capacity only: names are text the -/// scorer masks, and no door judges them for leakage. -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(try_from = "String", into = "String")] -pub struct ProperName(String); - -impl ProperName { - pub const MAX_BYTES: usize = 64; - pub const MAX_TOKENS: usize = 4; - pub const MAX_TOKEN_CHARS: usize = 24; - - pub fn parse(s: &str) -> Result { - let tokens: Vec<&str> = s.split(' ').collect(); - if s.len() <= Self::MAX_BYTES - && tokens.len() <= Self::MAX_TOKENS - && tokens.iter().all(|t| Self::token_ok(t)) - { - Ok(Self(s.to_string())) - } else { - Err(FormError::new("ProperName", s)) - } - } - - fn token_ok(t: &str) -> bool { - let (Some(first), Some(last)) = (t.chars().next(), t.chars().next_back()) else { - return false; - }; - t.chars().count() <= Self::MAX_TOKEN_CHARS - && first.is_alphabetic() - && last.is_alphabetic() - && t.chars() - .all(|c| c.is_alphabetic() || matches!(c, '\'' | '\u{2019}' | '-')) - } - - pub fn as_str(&self) -> &str { - &self.0 - } -} - -impl fmt::Display for ProperName { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&self.0) - } -} - -string_form!(ProperName, { - "type": "string", - "maxLength": 64, - "pattern": "^\\p{L}([\\p{L}'\u{2019}-]{0,22}\\p{L})?( \\p{L}([\\p{L}'\u{2019}-]{0,22}\\p{L})?){0,3}$", - "description": "1 to 4 space-joined tokens of letters, apostrophes and hyphens." -}); - #[derive(Clone, Debug, PartialEq, Eq, JsonSchema, DatabaseEntry)] #[cultcache(type = "weksa.interlingua_packet")] pub struct Packet { @@ -265,8 +238,9 @@ pub enum ProvenanceSource { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub struct Provenance { pub source: ProvenanceSource, - /// Key of the corpus item this packet was minted for. - pub corpus_item: Option, + /// The corpus item this packet was minted for. Required: the packet store + /// admits only packets that name their item. + pub corpus_item: CorpusItemId, pub encoder: EncoderId, pub confidence: Grade, } @@ -310,9 +284,9 @@ pub struct Referent { pub kind: InvId, /// Concept id. pub concept: InvId, - /// A proper name, the one place a referent carries wording. It is - /// form-bounded text; the scorer masks it. - pub name: Option, + /// The entity this referent names, when it has one. The entity store holds + /// its wording; the packet holds only the id. + pub entity: Option, pub features: Vec, } diff --git a/crates/weksa/src/schema.rs b/crates/weksa/src/schema.rs index f3efc73..72c27e4 100644 --- a/crates/weksa/src/schema.rs +++ b/crates/weksa/src/schema.rs @@ -2,6 +2,7 @@ //! documents; nothing in this crate reads JSON. use crate::corpus::CorpusItem; +use crate::entity::Entity; use crate::inventory::{InventoryEntry, InventoryRevision}; use crate::packet::Packet; use cultcache_rs::DatabaseEntry; @@ -19,5 +20,6 @@ pub fn published() -> Vec<(String, String)> { published_for::(), published_for::(), published_for::(), + published_for::(), ] } diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index 3b55f4f..6b0c858 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -7,10 +7,14 @@ use std::fmt; use std::path::Path; use cultcache_rs::{CultCache, SingleFileMessagePackBackingStore}; +use unicode_normalization::is_nfc; +use unicode_segmentation::UnicodeSegmentation; -use crate::corpus::{CorpusItem, item_key, key_slug}; +use crate::corpus::{CorpusItem, CorpusItemId, item_key, key_slug}; +use crate::entity::Entity; +use crate::form::is_language_tag; use crate::inventory::{Change, InvId, Inventory, InventoryEntry, InventoryRevision, revision_key}; -use crate::packet::Packet; +use crate::packet::{EntityId, Packet}; use crate::validate::{Refusal, validate}; /// A broken store rule, named. @@ -40,10 +44,61 @@ pub enum StoreRefusal { UnknownCorpusItem { key: String }, /// A packet key already holds a different packet. PacketImmutable { key: String }, + /// An entity id the entity store does not hold: a speaker, a prior-turn + /// speaker, a name span's entity or a referent's entity. + UnknownEntity { entity: EntityId }, + /// A name span with `start >= end`, `end` past the text, or an end off a + /// character boundary. + NameSpanRange { key: String, start: u32, end: u32 }, + /// A name span that starts or ends inside an extended grapheme cluster + /// (UAX #29) of the item's text. + NameSpanSplitsCluster { key: String, start: u32, end: u32 }, + /// An item text or prior-turn text, or an entity form, that is not NFC. + /// Callers normalize at intake; the stores refuse what they were handed. + NotNfc { key: String }, + /// An entity beyond a capacity bound. + EntityBounds { id: EntityId, bound: EntityBound }, + /// Two name spans of one item that overlap. + NameSpanOverlap { key: String }, + /// A name span whose bytes equal none of its entity's forms in the item's + /// language (an entity with no forms in that language included). + NameSpanNotAForm { + key: String, + entity: EntityId, + start: u32, + end: u32, + }, + /// A `put` that drops, reorders or edits a stored language, form or origin. + EntityNotAppendOnly { id: EntityId }, + /// A corpus packet's referent cites an entity that is neither named on its + /// item nor the item's speaker or a prior-turn speaker. + EntityNotOnItem { entity: EntityId, item: String }, /// Any other malformed document or change. Malformed { key: String, why: &'static str }, } +/// The capacity bound an entity broke. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum EntityBound { + /// A language that is not a BCP 47-form tag (1 to 35 bytes). + LanguageTag, + /// More than 8 languages. + Languages, + /// More than 8 forms in one language. + Forms, + /// A form of more than 64 bytes. + FormBytes, + /// A form of more than 4 words. + FormWords, +} + +impl EntityBound { + pub const MAX_LANGUAGES: usize = 8; + pub const MAX_FORMS: usize = 8; + pub const MAX_FORM_BYTES: usize = 64; + pub const MAX_FORM_WORDS: usize = 4; +} + #[derive(Debug)] pub enum StoreError { Refused(StoreRefusal), @@ -141,23 +196,25 @@ impl CorpusStore { Ok(item_key(slug, highest + 1)) } - /// The item `key` names, or `UnknownCorpusItem` when the key is not - /// `item::` or holds nothing. - fn resolve(&self, key: &str) -> Result { - let unknown = || StoreRefusal::UnknownCorpusItem { - key: key.to_string(), - }; - if key_slug(key).is_none() { - return Err(unknown().into()); - } - self.get(key)?.ok_or_else(|| unknown().into()) + /// The item `id` names, or `UnknownCorpusItem` when it holds nothing. + fn resolve(&self, id: &CorpusItemId) -> Result { + self.get(id.as_str())?.ok_or_else(|| { + StoreRefusal::UnknownCorpusItem { + key: id.to_string(), + } + .into() + }) } /// S1: an item key holds its whole item for good. S6: a text held under /// one key (live or withdrawn) is not re-added under another unless the new /// item supersedes that key. S5: a correction stays in its predecessor's /// group, so it cannot move across the split. - pub fn add(&mut self, item: CorpusItem) -> Result { + pub fn add( + &mut self, + item: CorpusItem, + entities: &EntityStore, + ) -> Result { if key_slug(&item.id).is_none() { return Err(malformed(&item.id, "item key is not item::")); } @@ -173,6 +230,13 @@ impl CorpusStore { } return Err(StoreRefusal::S1ItemImmutable { key: item.id }.into()); } + if !is_language_tag(&item.language) { + return Err(malformed(&item.id, "language is not a BCP 47-form tag")); + } + if !is_nfc(&item.text) || item.prior_turns.iter().any(|t| !is_nfc(&t.text)) { + return Err(StoreRefusal::NotNfc { key: item.id }.into()); + } + check_entities(&item, entities)?; let text = normalized(&item.text); for held in self.all()? { if normalized(&held.text) == text && item.supersedes.as_deref() != Some(&held.id) { @@ -216,6 +280,76 @@ impl CorpusStore { } } +/// Every entity an item cites exists, and every name span is a well-formed, +/// non-overlapping byte range of the text, starting and ending on extended +/// grapheme cluster boundaries, equal to one of its entity's forms in the +/// item's language. Id lookup, UAX #29 boundaries and byte equality: no reading +/// of meaning, so every script is held to the same check. +fn check_entities(item: &CorpusItem, entities: &EntityStore) -> Result<(), StoreError> { + let known = |id: &EntityId| -> Result { + entities + .get(id)? + .ok_or_else(|| StoreRefusal::UnknownEntity { entity: id.clone() }.into()) + }; + known(&item.speaker)?; + for turn in &item.prior_turns { + known(&turn.speaker)?; + } + let mut ranges = Vec::with_capacity(item.names.len()); + let cuts: BTreeSet = item + .text + .grapheme_indices(true) + .map(|(i, _)| i) + .chain([item.text.len()]) + .collect(); + for span in &item.names { + let entity = known(&span.entity)?; + let (start, end) = (span.start as usize, span.end as usize); + let Some(bytes) = item + .text + .get(start..end) + .filter(|_| start < end) + .map(str::as_bytes) + else { + return Err(StoreRefusal::NameSpanRange { + key: item.id.clone(), + start: span.start, + end: span.end, + } + .into()); + }; + if !cuts.contains(&start) || !cuts.contains(&end) { + return Err(StoreRefusal::NameSpanSplitsCluster { + key: item.id.clone(), + start: span.start, + end: span.end, + } + .into()); + } + let is_form = entity + .forms_in(&item.language) + .is_some_and(|forms| forms.iter().any(|f| f.as_bytes() == bytes)); + if !is_form { + return Err(StoreRefusal::NameSpanNotAForm { + key: item.id.clone(), + entity: span.entity.clone(), + start: span.start, + end: span.end, + } + .into()); + } + ranges.push((span.start, span.end)); + } + ranges.sort_unstable(); + if ranges.windows(2).any(|w| w[1].0 < w[0].1) { + return Err(StoreRefusal::NameSpanOverlap { + key: item.id.clone(), + } + .into()); + } + Ok(()) +} + /// Lowercased alphanumerics only, so case, spacing and punctuation variants of /// a text compare equal. fn normalized(text: &str) -> String { @@ -225,6 +359,112 @@ fn normalized(text: &str) -> String { .collect() } +pub struct EntityStore { + cache: CultCache, +} + +impl EntityStore { + pub fn open(path: &Path) -> anyhow::Result { + Ok(Self { + cache: open_cache(path, |c| c.register_entry_type::())?, + }) + } + + pub fn all(&self) -> anyhow::Result> { + let mut entities = self.cache.get_all::()?; + entities.sort_by(|a, b| a.id.cmp(&b.id)); + Ok(entities) + } + + pub fn get(&self, id: &EntityId) -> anyhow::Result> { + self.cache.get::(id.as_str()) + } + + /// Admits a new entity, an identical replay, or a replacement that keeps + /// every stored language and form in order and origin as it was, and only + /// appends: forms grow, never shrink, reorder or change. An entity has 1 to + /// 8 languages, each a BCP 47-form tag named once, with 1 to 8 NFC forms of + /// 1 to 64 bytes and at most 4 words, no control character, no edge + /// whitespace, none repeated. + pub fn put(&mut self, entity: Entity) -> Result { + let key = entity.id.to_string(); + check_entity_shape(&entity)?; + if let Some(old) = self.get(&entity.id)? { + if old == entity { + return Ok(AddOutcome::Unchanged); + } + let grows = old.origin == entity.origin + && old.forms.len() <= entity.forms.len() + && old.forms.iter().zip(&entity.forms).all(|(o, n)| { + o.language == n.language + && o.forms.len() <= n.forms.len() + && n.forms.starts_with(&o.forms) + }); + if !grows { + return Err(StoreRefusal::EntityNotAppendOnly { id: entity.id }.into()); + } + } + self.cache.put(key, &entity)?; + Ok(AddOutcome::Added) + } +} + +fn check_entity_shape(entity: &Entity) -> Result<(), StoreError> { + let bad = |why| malformed(entity.id.as_str(), why); + let bound = |bound| -> StoreError { + StoreRefusal::EntityBounds { + id: entity.id.clone(), + bound, + } + .into() + }; + if entity.forms.is_empty() { + return Err(bad("an entity has forms in at least one language")); + } + if entity.forms.len() > EntityBound::MAX_LANGUAGES { + return Err(bound(EntityBound::Languages)); + } + let mut languages = BTreeSet::new(); + for lf in &entity.forms { + if !is_language_tag(&lf.language) { + return Err(bound(EntityBound::LanguageTag)); + } + if !languages.insert(&lf.language) { + return Err(bad("a language is named once")); + } + if lf.forms.is_empty() { + return Err(bad("a language has at least one form")); + } + if lf.forms.len() > EntityBound::MAX_FORMS { + return Err(bound(EntityBound::Forms)); + } + let mut seen = BTreeSet::new(); + for f in &lf.forms { + if f.len() > EntityBound::MAX_FORM_BYTES { + return Err(bound(EntityBound::FormBytes)); + } + if f.split_whitespace().count() > EntityBound::MAX_FORM_WORDS { + return Err(bound(EntityBound::FormWords)); + } + if !is_nfc(f) { + return Err(StoreRefusal::NotNfc { + key: entity.id.to_string(), + } + .into()); + } + if f.is_empty() || f.chars().any(char::is_control) || f.trim() != f.as_str() { + return Err(bad( + "a form is not empty and has no control character or edge whitespace", + )); + } + if !seen.insert(f) { + return Err(bad("a form is not repeated within its language")); + } + } + } + Ok(()) +} + pub struct InventoryStore { cache: CultCache, } @@ -360,16 +600,36 @@ impl PacketStore { /// The only door: a packet exists only if `validate` passes against the /// inventory the store holds at the packet's revision, its `corpus_item` - /// (when set) resolves, and a key holds its packet for good. + /// resolves, every referent's entity is in the entity store and is named on + /// that item or is its speaker or a prior-turn speaker (an id-set check). A + /// key holds its packet for good. A packet that names no corpus item has no + /// door here. pub fn admit( &mut self, packet: Packet, inventory: &InventoryStore, corpus: &CorpusStore, + entities: &EntityStore, ) -> Result<(), StoreError> { validate(&packet, &inventory.inventory()?).map_err(StoreError::Invalid)?; - if let Some(item_key) = &packet.provenance.corpus_item { - corpus.resolve(item_key)?; + let item = corpus.resolve(&packet.provenance.corpus_item)?; + for entity in packet.referents.iter().filter_map(|r| r.entity.as_ref()) { + if entities.get(entity)?.is_none() { + return Err(StoreRefusal::UnknownEntity { + entity: entity.clone(), + } + .into()); + } + let on_item = item.names.iter().any(|n| &n.entity == entity) + || &item.speaker == entity + || item.prior_turns.iter().any(|t| &t.speaker == entity); + if !on_item { + return Err(StoreRefusal::EntityNotOnItem { + entity: entity.clone(), + item: item.id.clone(), + } + .into()); + } } let key = packet.id.to_string(); if let Some(existing) = self.get(&key)? { diff --git a/crates/weksa/src/validate.rs b/crates/weksa/src/validate.rs index 854e099..a88c18e 100644 --- a/crates/weksa/src/validate.rs +++ b/crates/weksa/src/validate.rs @@ -4,9 +4,11 @@ //! or a grade is its type's parse, not a rule, so V9 is retired and not //! renumbered. //! -//! `validate` owns form and capacity and makes no leak decision. A packet's -//! free text is form-bounded and every list is bounded, so a packet has no -//! unbounded channel. What bounds each list (V11, V12): a feature list holds at +//! `validate` owns form and capacity and makes no leak decision. A packet's one +//! free text is its trace (`notes` and `rejected_readings`): `validate` does not +//! bound or read it, and [`LoweringInput`] drops it, so no lowerer sees it. +//! Every other slot is an id type, so the lists are the bounded part. What bounds +//! each list (V11, V12): a feature list holds at //! most one value per feature id and `extensions` at most one per extension id, //! so each is at most the inventory's ids of that kind; `participants`, //! `audience` and `preserve` hold each node id at most once and every id names @@ -14,7 +16,7 @@ //! hold each role id at most once, so at most the inventory's roles; and the //! nodes themselves are at most `Packet::MAX_NODES` (V12). Whether a score may //! rise because corpus text reached a packet is owned by the golf loop's -//! context builder and the scorer (ruling `leak-owned-by-flow-and-scorer`), +//! context builder and the scorer (ruling `leak-owned-by-flow-and-scorer-3`), //! never by a text rule here, in `admit_revision` or in `PacketStore::admit`. use std::collections::BTreeSet; diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 6b13b77..68229fa 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -5,12 +5,42 @@ use weksa::{InventoryStore, PacketStore}; fn weksa(dir: &std::path::Path, args: &[&str]) -> Output { Command::new(env!("CARGO_BIN_EXE_weksa")) .current_dir(dir) - .args(["--corpus", "./c.cc", "--inventory", "./i.cc"]) + .args([ + "--corpus", + "./c.cc", + "--inventory", + "./i.cc", + "--entities", + "./e.cc", + ]) .args(args) .output() .unwrap() } +/// Admits the cast entities the tests cite. +fn cast_entities(dir: &std::path::Path, slugs: &[&str]) { + for slug in slugs { + let id = format!("entity:{slug}"); + let form = format!("en-US={}", slug.to_uppercase()); + let put = weksa( + dir, + &[ + "entity", + "put", + &id, + "--form", + &form, + "--origin", + "cast", + "--persona", + slug, + ], + ); + assert!(put.status.success(), "{}", out(&put)); + } +} + fn out(o: &Output) -> String { String::from_utf8_lossy(&o.stdout).into_owned() + &String::from_utf8_lossy(&o.stderr) } @@ -23,7 +53,7 @@ const ADD: [&str; 14] = [ "--text", "Mind the hatch.", "--speaker", - "nibu", + "entity:nibu", "--group", "scene-1", "--admitted-on", @@ -35,6 +65,7 @@ const ADD: [&str; 14] = [ #[test] fn seed_add_dump_split_withdraw() { let d = tempfile::tempdir().unwrap(); + cast_entities(d.path(), &["nibu"]); let seeded = weksa(d.path(), &["inventory", "seed"]); assert!(seeded.status.success(), "{}", out(&seeded)); assert!(out(&seeded).contains("seeded rev:1")); @@ -105,13 +136,14 @@ fn seed_add_dump_split_withdraw() { #[test] fn model_written_and_public_domain_sources_name_their_provenance() { let d = tempfile::tempdir().unwrap(); + cast_entities(d.path(), &["s", "player"]); let base = [ "corpus", "add", "--slug", "x", "--speaker", - "s", + "entity:s", "--group", "g", "--admitted-on", @@ -186,7 +218,7 @@ fn model_written_and_public_domain_sources_name_their_provenance() { "--source", "operator-written", "--prior-turn", - "player=hello", + "entity:player=hello", ]); assert!(turn.status.success(), "{}", out(&turn)); } @@ -200,7 +232,7 @@ fn validate_reports_each_packet() { let packet = common::good(); PacketStore::open(&packets) .unwrap() - .admit(packet.clone(), &w.inventory, &w.corpus) + .admit(packet.clone(), &w.inventory, &w.corpus, &w.entities) .unwrap(); let validate = |inventory: &std::path::Path, extra: &[&str]| { @@ -250,9 +282,188 @@ fn schema_writes_the_published_files() { names, [ "weksa.corpus_item.schema.json", + "weksa.entity.schema.json", "weksa.interlingua_packet.schema.json", "weksa.inventory_entry.schema.json", "weksa.inventory_revision.schema.json" ] ); } + +#[test] +fn entity_put_and_name_spans() { + let d = tempfile::tempdir().unwrap(); + let put = |args: &[&str]| { + let mut all = vec!["entity", "put", "entity:kay", "--origin", "cast"]; + all.extend_from_slice(args); + weksa(d.path(), &all) + }; + let no_persona = put(&["--form", "en-US=Kay"]); + assert!( + !no_persona.status.success() && out(&no_persona).contains("--persona"), + "{}", + out(&no_persona) + ); + let first = put(&["--form", "en-US=Kay", "--persona", "kay"]); + assert!( + first.status.success() && out(&first).contains("Added entity:kay"), + "{}", + out(&first) + ); + // Forms append; a language's forms keep the order given. + let grown = put(&[ + "--form", + "en-US=Kay", + "--form", + "en-US=Kay's", + "--persona", + "kay", + ]); + assert!(grown.status.success(), "{}", out(&grown)); + let shrunk = put(&["--form", "en-US=Kay", "--persona", "kay"]); + assert!( + !shrunk.status.success() && out(&shrunk).contains("EntityNotAppendOnly"), + "{}", + out(&shrunk) + ); + let dump = out(&weksa(d.path(), &["entity", "dump"])); + assert!( + dump.contains("entity:kay") && dump.contains("Kay's"), + "{dump}" + ); + + cast_entities(d.path(), &["nibu"]); + let add = |range: &str, text: &str| { + weksa( + d.path(), + &[ + "corpus", + "add", + "--slug", + "kay", + "--text", + text, + "--speaker", + "entity:nibu", + "--group", + "g", + "--admitted-on", + "2026-10-29", + "--source", + "operator-written", + "--name", + range, + ], + ) + }; + let named = add("0..3=entity:kay", "Kay left."); + assert!(named.status.success(), "{}", out(&named)); + let unequal = add("0..3=entity:kay", "Kai left."); + assert!( + !unequal.status.success() && out(&unequal).contains("NameSpanNotAForm"), + "{}", + out(&unequal) + ); + let malformed = add("0-3", "Kay again."); + assert!( + !malformed.status.success() && out(&malformed).contains("start..end=entity"), + "{}", + out(&malformed) + ); + let dump = out(&weksa(d.path(), &["corpus", "dump"])); + assert!( + dump.contains("names:") && dump.contains("entity:kay"), + "{dump}" + ); +} + +#[test] +fn entity_origin_flags_must_match_origin() { + let d = tempfile::tempdir().unwrap(); + let put = |args: &[&str]| { + let mut all = vec!["entity", "put", "entity:kay", "--form", "en-US=Kay"]; + all.extend_from_slice(args); + weksa(d.path(), &all) + }; + for mismatched in [ + &["--origin", "live", "--persona", "kay"][..], + &["--origin", "live", "--work", "edda"], + &["--origin", "cast", "--persona", "kay", "--work", "edda"], + &[ + "--origin", + "public-domain", + "--persona", + "kay", + "--work", + "edda", + ], + &["--origin", "public-domain", "--persona", "kay"], + ] { + let o = put(mismatched); + assert!(!o.status.success(), "{mismatched:?}: {}", out(&o)); + } + assert!(!out(&weksa(d.path(), &["entity", "dump"])).contains("entity:kay")); + assert!(put(&["--origin", "live"]).status.success()); + let work = weksa( + d.path(), + &[ + "entity", + "put", + "entity:brokkr", + "--form", + "en-US=Brokkr", + "--origin", + "public-domain", + "--work", + "edda", + ], + ); + assert!(work.status.success(), "{}", out(&work)); + assert!(out(&weksa(d.path(), &["entity", "dump"])).contains("PublicDomain")); +} + +#[test] +fn intake_normalizes_to_nfc() { + let d = tempfile::tempdir().unwrap(); + // A decomposed form and decomposed text meet as the same composed bytes. + let put = weksa( + d.path(), + &[ + "entity", + "put", + "entity:rene", + "--form", + "fr=Rene\u{301}", + "--origin", + "live", + ], + ); + assert!(put.status.success(), "{}", out(&put)); + cast_entities(d.path(), &["nibu"]); + let add = weksa( + d.path(), + &[ + "corpus", + "add", + "--slug", + "rene", + "--text", + "Rene\u{301} part.", + "--language", + "fr", + "--speaker", + "entity:nibu", + "--group", + "g", + "--admitted-on", + "2026-10-29", + "--source", + "operator-written", + "--name", + "0..5=entity:rene", + ], + ); + assert!(add.status.success(), "{}", out(&add)); + let dump = out(&weksa(d.path(), &["corpus", "dump"])); + assert!(dump.contains("Ren\u{e9} part."), "{dump}"); +} diff --git a/crates/weksa/tests/common/mod.rs b/crates/weksa/tests/common/mod.rs index dfd8106..a26c462 100644 --- a/crates/weksa/tests/common/mod.rs +++ b/crates/weksa/tests/common/mod.rs @@ -7,8 +7,9 @@ use weksa::corpus::ModelFamily; use weksa::inventory::{Change, seed}; use weksa::packet::*; use weksa::{ - CorpusItem, CorpusStore, InvId, Inventory, InventoryEntry, InventoryRevision, InventoryStore, - OptionLabel, Slot, Source, SplitScheme, ValueKind, + CorpusItem, CorpusItemId, CorpusStore, Entity, EntityOrigin, EntityStore, InvId, Inventory, + InventoryEntry, InventoryRevision, InventoryStore, LanguageForms, NameSpan, OptionLabel, Slot, + Source, SplitScheme, ValueKind, }; pub const PERSON: &str = "referent_kind:1"; @@ -73,12 +74,45 @@ pub fn model() -> Source { } } +/// Every speaker the tests use, as entities with one en-US form. +const SPEAKERS: [&str; 6] = ["nibu", "brokkr", "player", "s", "someone", "hero"]; + +pub fn eid(slug: &str) -> EntityId { + EntityId::parse(&format!("entity:{slug}")).unwrap() +} + +/// A cast entity with `forms` in `language`, citation form first. +pub fn ent(slug: &str, language: &str, forms: &[&str]) -> Entity { + Entity { + id: eid(slug), + forms: vec![LanguageForms { + language: language.into(), + forms: forms.iter().map(|f| f.to_string()).collect(), + }], + origin: EntityOrigin::Cast { + persona: slug.into(), + }, + } +} + +/// The entity store at `entities.cc` under `dir`, holding every test speaker +/// (en-US form: the slug capitalized). Reopening replays identically. +pub fn entities_in(dir: &std::path::Path) -> EntityStore { + let mut store = EntityStore::open(&dir.join("entities.cc")).unwrap(); + for slug in SPEAKERS { + let form = slug[..1].to_uppercase() + &slug[1..]; + store.put(ent(slug, "en-US", &[&form])).unwrap(); + } + store +} + pub fn item_in(key: &str, text: &str, group: &str, source: Source) -> CorpusItem { CorpusItem { id: key.into(), text: text.into(), language: "en-US".into(), - speaker: "nibu".into(), + speaker: eid("nibu"), + names: vec![], group: group.into(), prior_turns: vec![], source, @@ -132,6 +166,7 @@ pub struct World { pub dir: TempDir, pub inventory: InventoryStore, pub corpus: CorpusStore, + pub entities: EntityStore, } impl World { @@ -143,15 +178,15 @@ impl World { pub fn world() -> World { let dir = tempfile::tempdir().unwrap(); let mut corpus = CorpusStore::open(&dir.path().join("c.cc")).unwrap(); + let entities = entities_in(dir.path()); let train = group_in_bucket(10); - corpus - .add(item_in( - "item:nibu:1", - "Nibu says mind the hatch.", - &train, - cast(), - )) - .unwrap(); + let mut first = item_in("item:nibu:1", "Nibu says mind the hatch.", &train, cast()); + first.names = vec![NameSpan { + entity: eid("nibu"), + start: 0, + end: 4, + }]; + corpus.add(first, &entities).unwrap(); let mut inventory = InventoryStore::open(&dir.path().join("i.cc")).unwrap(); let (rev1, entries) = seed(); inventory.admit_revision(rev1, entries).unwrap(); @@ -192,6 +227,7 @@ pub fn world() -> World { dir, inventory, corpus, + entities, } } @@ -209,7 +245,7 @@ pub fn good() -> Packet { kind: PacketKind::DialogueLine, provenance: Provenance { source: ProvenanceSource::Authored, - corpus_item: Some("item:nibu:1".into()), + corpus_item: CorpusItemId::parse("item:nibu:1").unwrap(), encoder: EncoderId::parse(ENCODER_ID).unwrap(), confidence: grade(900), }, @@ -231,7 +267,7 @@ pub fn good() -> Packet { id: nd("n1"), kind: iv(PERSON), concept: iv(CONCEPT_A), - name: Some(ProperName::parse("Nibu").unwrap()), + entity: Some(eid("nibu")), features: vec![], }], predications: vec![Predication { diff --git a/crates/weksa/tests/entities.rs b/crates/weksa/tests/entities.rs new file mode 100644 index 0000000..30d43e3 --- /dev/null +++ b/crates/weksa/tests/entities.rs @@ -0,0 +1,419 @@ +mod common; +use common::*; +use weksa::corpus::{CorpusItem, NameSpan, Turn}; +use weksa::packet::{Packet, Referent}; +use weksa::{ + AddOutcome, CorpusItemId, CorpusStore, Entity, EntityId, EntityOrigin, EntityStore, PacketId, + PacketStore, Refusal, SCHEMA_VERSION, StoreError, StoreRefusal, validate, +}; + +fn refusal(r: Result) -> StoreRefusal { + match r { + Err(StoreError::Refused(r)) => r, + other => panic!("expected a store refusal, got {other:?}"), + } +} + +fn malformed(r: Result) -> &'static str { + match refusal(r) { + StoreRefusal::Malformed { why, .. } => why, + other => panic!("expected Malformed, got {other:?}"), + } +} + +fn span(slug: &str, start: u32, end: u32) -> NameSpan { + NameSpan { + entity: eid(slug), + start, + end, + } +} + +/// An item in `language` whose text and spans the test chooses. +fn named(n: u32, language: &str, text: &str, names: Vec) -> CorpusItem { + CorpusItem { + language: language.into(), + names, + ..item_in(&format!("item:t:{n}"), text, "g", cast()) + } +} + +#[test] +fn entity_id_form() { + for ok in ["entity:brokkr", "entity:a-1", "entity:7", "entity:a-b-c"] { + assert_eq!(EntityId::parse(ok).unwrap().as_str(), ok); + } + let long_ok = format!("entity:{}", "a".repeat(48)); + assert!(EntityId::parse(&long_ok).is_ok()); + let too_long = format!("entity:{}", "a".repeat(49)); + for bad in [ + "Entity:x", + "entity:-a", + "entity:a-", + "entity:", + "entity:a_b", + "entity:A", + "entity:a b", + "entity:a:b", + "brokkr", + "entity:\u{5c71}", + &too_long, + ] { + assert!(EntityId::parse(bad).is_err(), "{bad:?}"); + assert!( + serde_json::from_value::(bad.into()).is_err(), + "{bad:?} must not read back from a store" + ); + } + // A referent reads its entity through the same parse. + let mut referent = serde_json::to_value(good().referents[0].clone()).unwrap(); + assert!(serde_json::from_value::(referent.clone()).is_ok()); + referent["entity"] = "Nibu".into(); + assert!(serde_json::from_value::(referent).is_err()); +} + +#[test] +fn entity_forms_only_grow() { + let d = tempfile::tempdir().unwrap(); + let mut store = EntityStore::open(&d.path().join("e.cc")).unwrap(); + let kay = ent("kay", "en-US", &["Kay"]); + assert_eq!(store.put(kay.clone()).unwrap(), AddOutcome::Added); + assert_eq!(store.put(kay.clone()).unwrap(), AddOutcome::Unchanged); + + // A form appends; a language appends. + let mut grown = kay.clone(); + grown.forms[0].forms.push("Kay's".into()); + assert_eq!(store.put(grown.clone()).unwrap(), AddOutcome::Added); + let mut more = grown.clone(); + more.forms.push(weksa::LanguageForms { + language: "tr".into(), + forms: vec!["Kay".into()], + }); + assert_eq!(store.put(more.clone()).unwrap(), AddOutcome::Added); + // The store read back from disk holds the growth. + let mut store = EntityStore::open(&d.path().join("e.cc")).unwrap(); + assert_eq!(store.get(&eid("kay")).unwrap(), Some(more.clone())); + + // Dropping, reordering, editing and re-originating are refused, each leaving + // the stored entity as it was. + let not_append = StoreRefusal::EntityNotAppendOnly { id: eid("kay") }; + let edits: [fn(&mut Entity); 6] = [ + |e| e.forms[0].forms.reverse(), + |e| e.forms[0].forms[0] = "Kai".into(), + |e| { + e.forms.pop(); + }, + |e| e.forms.reverse(), + |e| e.forms[1].language = "fi".into(), + |e| e.origin = EntityOrigin::Live, + ]; + assert_eq!( + refusal(store.put(ent("kay", "en-US", &["Kay"]))), + not_append + ); + for edit in edits { + let mut v = more.clone(); + edit(&mut v); + assert_eq!(refusal(store.put(v.clone())), not_append, "{v:?}"); + } + assert_eq!(store.get(&eid("kay")).unwrap(), Some(more)); + + // Shape: no forms, a language twice, an empty language, an empty language + // entry, an empty, padded, control-bearing, oversize or repeated form. + let shape = |forms: Vec| weksa::Entity { + forms, + ..ent("shape", "en-US", &["X"]) + }; + let lf = |language: &str, forms: &[&str]| weksa::LanguageForms { + language: language.into(), + forms: forms.iter().map(|f| f.to_string()).collect(), + }; + for bad in [ + vec![], + vec![lf("en-US", &["A"]), lf("en-US", &["B"])], + vec![lf("en-US", &[])], + vec![lf("en-US", &[""])], + vec![lf("en-US", &[" A"])], + vec![lf("en-US", &["A\u{a0}"])], + vec![lf("en-US", &["A\nB"])], + vec![lf("en-US", &["A", "A"])], + ] { + malformed(store.put(shape(bad.clone()))); + assert!( + store.get(&eid("shape")).unwrap().is_none(), + "{bad:?} was stored" + ); + } + // An empty language is not a tag. + let e = shape(vec![lf("", &["A"])]); + assert!(matches!( + refusal(store.put(e)), + StoreRefusal::EntityBounds { + bound: weksa::store::EntityBound::LanguageTag, + .. + } + )); + // 64 bytes is the edge; a form may repeat across languages. + let edge = "a".repeat(64); + store + .put(shape(vec![lf("en-US", &[&edge]), lf("tr", &[&edge])])) + .unwrap(); +} + +#[test] +fn corpus_spans_checked() { + let d = tempfile::tempdir().unwrap(); + let mut ents = entities_in(d.path()); + ents.put(ent("kay", "en-US", &["Kay"])).unwrap(); + ents.put(ent("ab", "en-US", &["abab", "bab"])).unwrap(); + let mut corpus = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + + assert_eq!( + corpus + .add( + named(1, "en-US", "Kay left.", vec![span("kay", 0, 3)]), + &ents + ) + .unwrap(), + AddOutcome::Added + ); + + let range = |n: u32, start: u32, end: u32| StoreRefusal::NameSpanRange { + key: format!("item:t:{n}"), + start, + end, + }; + let not_a_form = |n: u32, slug: &str, start: u32, end: u32| StoreRefusal::NameSpanNotAForm { + key: format!("item:t:{n}"), + entity: eid(slug), + start, + end, + }; + let mut refused = |item: CorpusItem| refusal(corpus.add(item, &ents)); + // Empty, backwards, past the end: a range refusal. + for (start, end) in [(3, 3), (4, 3), (0, 99)] { + let item = named(2, "en-US", "Kay left.", vec![span("kay", start, end)]); + assert_eq!(refused(item), range(2, start, end)); + } + // Inside a multibyte character: start and end off the boundary. + let text = "\u{e9}Kay"; + assert_eq!(text.len(), 5); + let item = named(2, "en-US", text, vec![span("kay", 1, 4)]); + assert_eq!(refused(item), range(2, 1, 4)); + let item = named(2, "en-US", "Kay\u{e9}", vec![span("kay", 0, 4)]); + assert_eq!(refused(item), range(2, 0, 4)); + // An unknown entity. + let item = named(2, "en-US", "Kay left.", vec![span("ghost", 0, 3)]); + assert_eq!( + refused(item), + StoreRefusal::UnknownEntity { + entity: eid("ghost") + } + ); + // Not byte-equal: other text, other case, a missing language. + let item = named(2, "en-US", "Kai left.", vec![span("kay", 0, 3)]); + assert_eq!(refused(item), not_a_form(2, "kay", 0, 3)); + let item = named(2, "en-US", "kay left.", vec![span("kay", 0, 3)]); + assert_eq!(refused(item), not_a_form(2, "kay", 0, 3)); + let item = named(2, "tr", "Kay left.", vec![span("kay", 0, 3)]); + assert_eq!(refused(item), not_a_form(2, "kay", 0, 3)); + // Overlap: both spans are forms, they share bytes. + let item = named(2, "en-US", "abab", vec![span("ab", 0, 4), span("ab", 1, 4)]); + assert_eq!( + refused(item), + StoreRefusal::NameSpanOverlap { + key: "item:t:2".into() + } + ); + // Adjacent spans do not overlap, whatever order they are listed in. + let adjacent = named( + 90, + "en-US", + "KayKay", + vec![span("kay", 3, 6), span("kay", 0, 3)], + ); + assert_eq!(corpus.add(adjacent, &ents).unwrap(), AddOutcome::Added); + + // Speakers: unknown, and unknown in a prior turn. + let ghost = EntityId::parse("entity:ghost").unwrap(); + let unknown = StoreRefusal::UnknownEntity { + entity: ghost.clone(), + }; + let mut item = named(91, "en-US", "Hello there.", vec![]); + item.speaker = ghost.clone(); + assert_eq!(refusal(corpus.add(item, &ents)), unknown); + let mut item = named(92, "en-US", "Hello again.", vec![]); + item.prior_turns = vec![ + Turn { + speaker: eid("nibu"), + text: "hi".into(), + }, + Turn { + speaker: ghost, + text: "hi".into(), + }, + ]; + assert_eq!(refusal(corpus.add(item, &ents)), unknown); + let mut item = named(93, "en-US", "Hello at last.", vec![]); + item.prior_turns = vec![Turn { + speaker: eid("nibu"), + text: "hi".into(), + }]; + assert_eq!(corpus.add(item, &ents).unwrap(), AddOutcome::Added); + // Nothing refused was stored. + let keys: Vec = corpus.all().unwrap().into_iter().map(|i| i.id).collect(); + assert_eq!(keys, ["item:t:1", "item:t:90", "item:t:93"]); +} + +#[test] +fn corpus_spans_every_script() { + let d = tempfile::tempdir().unwrap(); + let mut ents = entities_in(d.path()); + ents.put(ent( + "minsu", + "ko", + &["\u{BBFC}\u{C218}", "\u{BBFC}\u{C218}\u{AC00}"], + )) + .unwrap(); + ents.put(ent("brokkr-tr", "tr", &["Brokkr", "Brokkr'\u{131}n"])) + .unwrap(); + ents.put(ent( + "somchai", + "th", + &["\u{E2A}\u{E21}\u{E0A}\u{E32}\u{E22}"], + )) + .unwrap(); + let mut corpus = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + + // The particle-bound Korean form: byte equality with the stored form. + let ko = "\u{BBFC}\u{C218}\u{AC00} \u{C654}\u{B2E4}"; + let with_particle = "\u{BBFC}\u{C218}\u{AC00}".len(); + assert_eq!(with_particle, 9); + let item = named(1, "ko", ko, vec![span("minsu", 0, with_particle as u32)]); + assert_eq!(corpus.add(item, &ents).unwrap(), AddOutcome::Added); + // The bare stem is a stored form too; a span cutting a syllable is not. + let item = named( + 2, + "ko", + "\u{BBFC}\u{C218} \u{C654}\u{B2E4}", + vec![span("minsu", 0, 6)], + ); + assert_eq!(corpus.add(item, &ents).unwrap(), AddOutcome::Added); + let item = named(3, "ko", ko, vec![span("minsu", 0, 7)]); + assert!(matches!( + refusal(corpus.add(item, &ents)), + StoreRefusal::NameSpanRange { .. } + )); + // Turkish: the apostrophe suffix. + let tr = "Brokkr'\u{131}n geldi."; + let len = "Brokkr'\u{131}n".len() as u32; + let item = named(4, "tr", tr, vec![span("brokkr-tr", 0, len)]); + assert_eq!(corpus.add(item, &ents).unwrap(), AddOutcome::Added); + // Thai: no spaces, no segmentation, a span is a byte range. + let th = "\u{E2A}\u{E21}\u{E0A}\u{E32}\u{E22}\u{E21}\u{E32}\u{E41}\u{E25}\u{E49}\u{E27}"; + let len = "\u{E2A}\u{E21}\u{E0A}\u{E32}\u{E22}".len() as u32; + let item = named(5, "th", th, vec![span("somchai", 0, len)]); + assert_eq!(corpus.add(item, &ents).unwrap(), AddOutcome::Added); + // The same bytes under another language tag are refused. + let item = named(6, "ko", th, vec![span("somchai", 0, len)]); + assert!(matches!( + refusal(corpus.add(item, &ents)), + StoreRefusal::NameSpanNotAForm { .. } + )); +} + +#[test] +fn packet_entities_resolve() { + let mut w = world(); + w.entities.put(ent("kay", "en-US", &["Kay"])).unwrap(); + let mut talk = named(2, "en-US", "Kay and Hero talk.", vec![span("kay", 0, 3)]); + talk.id = "item:nibu:2".into(); + talk.prior_turns = vec![Turn { + speaker: eid("hero"), + text: "hi".into(), + }]; + w.corpus.add(talk, &w.entities).unwrap(); + + let mut packets = PacketStore::open(&w.dir.path().join("p.cc")).unwrap(); + let packet = |n: u8, item: &str, entity: &str| { + let mut p = good(); + p.id = PacketId::parse(&format!("packet:{n:016x}")).unwrap(); + p.provenance.corpus_item = CorpusItemId::parse(item).unwrap(); + p.referents[0].entity = Some(eid(entity)); + p + }; + let mut admit = |p: Packet| packets.admit(p, &w.inventory, &w.corpus, &w.entities); + + // An entity the store lacks. + let unknown = StoreRefusal::UnknownEntity { + entity: eid("ghost"), + }; + assert_eq!(refusal(admit(packet(1, "item:nibu:1", "ghost"))), unknown); + assert_eq!(refusal(admit(packet(2, "item:nibu:2", "ghost"))), unknown); + // A stored entity that is not on the item. + let off_item = |item: &str, entity: &str| StoreRefusal::EntityNotOnItem { + entity: eid(entity), + item: item.into(), + }; + assert_eq!( + refusal(admit(packet(3, "item:nibu:2", "brokkr"))), + off_item("item:nibu:2", "brokkr") + ); + // nibu:1 names nibu (and is spoken by nibu); kay is only on nibu:2. + assert_eq!( + refusal(admit(packet(4, "item:nibu:1", "kay"))), + off_item("item:nibu:1", "kay") + ); + // On the item: a name entity, the speaker, a prior-turn speaker. + admit(packet(5, "item:nibu:2", "kay")).unwrap(); + admit(packet(6, "item:nibu:2", "nibu")).unwrap(); + admit(packet(7, "item:nibu:2", "hero")).unwrap(); + // No packet escapes the item check: a stored entity that no item names is + // refused on every item, and a packet cannot omit its item. + assert_eq!( + refusal(admit(packet(8, "item:nibu:1", "brokkr"))), + off_item("item:nibu:1", "brokkr") + ); + // A referent with no entity needs none; only a packet that cites checks. + let mut anonymous = packet(9, "item:nibu:2", "kay"); + anonymous.referents[0].entity = None; + admit(anonymous).unwrap(); + // Refused packets were not stored. + let stored: Vec = PacketStore::open(&w.dir.path().join("p.cc")) + .unwrap() + .all() + .unwrap() + .iter() + .map(|p| p.id.to_string()) + .collect(); + assert_eq!(stored.len(), 4, "{stored:?}"); +} + +#[test] +fn packet_schema_v2() { + assert_eq!(SCHEMA_VERSION, 3); + let mut p = good(); + p.schema_version = 2; + let Err(refused) = validate(&p, &inventory()) else { + panic!("a v1 packet must not validate") + }; + assert_eq!(refused, vec![Refusal::V1SchemaVersion { found: 2 }]); + + let published: std::collections::BTreeMap<_, _> = + weksa::schema::published().into_iter().collect(); + let packet: serde_json::Value = + serde_json::from_str(&published["weksa.interlingua_packet.schema.json"]).unwrap(); + let referent = &packet["$defs"]["Referent"]["properties"]; + assert!(referent.get("entity").is_some(), "{referent}"); + assert!(referent.get("name").is_none(), "{referent}"); + let defs = packet["$defs"].as_object().unwrap(); + assert!(!defs.keys().any(|k| k.ends_with("Name")), "{defs:?}"); + assert_eq!( + packet["$defs"]["EntityId"]["pattern"], + "^entity:[a-z0-9]([a-z0-9-]{0,46}[a-z0-9])?$" + ); + let item: serde_json::Value = + serde_json::from_str(&published["weksa.corpus_item.schema.json"]).unwrap(); + assert!(item["properties"].get("names").is_some()); + assert!(published.contains_key("weksa.entity.schema.json")); +} diff --git a/crates/weksa/tests/entity_door.rs b/crates/weksa/tests/entity_door.rs new file mode 100644 index 0000000..a0b3e9e --- /dev/null +++ b/crates/weksa/tests/entity_door.rs @@ -0,0 +1,348 @@ +//! Behavioural pins from Soul's entity-store s1 probes: the entity door does not +//! trust the writer (ruling entity-door-bounds). +mod common; +use common::*; +use unicode_normalization::UnicodeNormalization; +use weksa::corpus::{CorpusItem, NameSpan}; +use weksa::entity::{Entity, EntityOrigin, LanguageForms}; +use weksa::store::EntityBound; +use weksa::{EntityStore, StoreError, StoreRefusal}; + +fn refusal(r: Result) -> StoreRefusal { + match r { + Err(StoreError::Refused(r)) => r, + other => panic!("expected a store refusal, got {other:?}"), + } +} + +fn item( + n: impl std::fmt::Display, + lang: &str, + text: &str, + names: Vec, + group: &str, +) -> CorpusItem { + CorpusItem { + language: lang.into(), + names, + ..item_in(&format!("item:probe:{n}"), text, group, cast()) + } +} + +fn span(slug: &str, start: usize, end: usize) -> NameSpan { + NameSpan { + entity: eid(slug), + start: start as u32, + end: end as u32, + } +} + +fn bounds(entity: &Entity, bound: EntityBound) -> StoreRefusal { + StoreRefusal::EntityBounds { + id: entity.id.clone(), + bound, + } +} + +#[test] +fn span_never_splits_a_grapheme_cluster() { + let mut w = world(); + let g = group_in_bucket(10); + let splits = |n: u32, start: usize, end: usize| StoreRefusal::NameSpanSplitsCluster { + key: format!("item:probe:{n}"), + start: start as u32, + end: end as u32, + }; + + // Devanagari: the word is a base, a spacing vowel sign and a base; the form + // "र" ends inside the cluster of the first two. + w.entities.put(ent("ra", "hi", &["र"])).unwrap(); + let text = "राम आया"; + assert_eq!( + refusal( + w.corpus + .add(item(1, "hi", text, vec![span("ra", 0, 3)], &g), &w.entities) + ), + splits(1, 0, 3) + ); + // The whole words are legal names, the second starting after a space. + w.entities.put(ent("ram", "hi", &["राम", "आया"])).unwrap(); + let both = vec![span("ram", 0, 9), span("ram", 10, text.len())]; + w.corpus + .add(item(2, "hi", text, both, &g), &w.entities) + .unwrap(); + + // A start inside a cluster is refused too. + w.entities.put(ent("aa", "hi", &["ा"])).unwrap(); + assert_eq!( + refusal(w.corpus.add( + item(3, "hi", "राम x", vec![span("aa", 3, 6)], &g), + &w.entities + )), + splits(3, 3, 6) + ); + + // Latin base + combining mark with no precomposed form, so NFC keeps two + // code points: the form "q" leaves the accent outside. + let q = "q\u{301} arrive."; + w.entities.put(ent("q", "fr", &["q"])).unwrap(); + assert_eq!( + refusal( + w.corpus + .add(item(4, "fr", q, vec![span("q", 0, 1)], &g), &w.entities) + ), + splits(4, 0, 1) + ); + w.entities.put(ent("qa", "fr", &["q\u{301}"])).unwrap(); + w.corpus + .add(item(5, "fr", q, vec![span("qa", 0, 3)], &g), &w.entities) + .unwrap(); +} + +#[test] +fn text_and_forms_are_stored_nfc() { + let mut w = world(); + let g = group_in_bucket(10); + let not_nfc = |key: &str| StoreRefusal::NotNfc { key: key.into() }; + // Decomposed Hangul (conjoining jamo) text is refused, not segmented. + let nfd = "\u{1106}\u{1175}\u{11AB}\u{1109}\u{116E} \u{110B}\u{116A}\u{11BB}\u{1103}\u{1161}"; + assert!(!unicode_normalization::is_nfc(nfd)); + assert_eq!( + refusal(w.corpus.add(item(1, "ko", nfd, vec![], &g), &w.entities)), + not_nfc("item:probe:1") + ); + // The same words composed are admitted, and a span on a syllable pair is exact. + let nfc: String = nfd.nfc().collect(); + let name: String = "\u{1106}\u{1175}\u{11AB}\u{1109}\u{116E}".nfc().collect(); + w.entities.put(ent("minsu", "ko", &[&name])).unwrap(); + w.corpus + .add( + item(2, "ko", &nfc, vec![span("minsu", 0, name.len())], &g), + &w.entities, + ) + .unwrap(); + // Latin NFD text, NFD forms and NFD prior turns are refused. + assert_eq!( + refusal(w.corpus.add( + item(3, "fr", "Rene\u{301} arrive.", vec![], &g), + &w.entities + )), + not_nfc("item:probe:3") + ); + assert_eq!( + refusal(w.entities.put(ent("rene", "fr", &["Rene\u{301}"]))), + not_nfc("entity:rene") + ); + let mut turns = item(4, "fr", "Salut.", vec![], &g); + turns.prior_turns = vec![weksa::Turn { + speaker: eid("nibu"), + text: "Rene\u{301}?".into(), + }]; + assert_eq!( + refusal(w.corpus.add(turns, &w.entities)), + not_nfc("item:probe:4") + ); + // NFC text and an NFC form of the same name agree byte for byte. + w.entities.put(ent("rene", "fr", &["Ren\u{e9}"])).unwrap(); + w.corpus + .add( + item(5, "fr", "Ren\u{e9} part.", vec![span("rene", 0, 5)], &g), + &w.entities, + ) + .unwrap(); +} + +#[test] +fn entity_capacity_is_bounded() { + let mut w = world(); + let lf = |language: &str, forms: Vec| LanguageForms { + language: language.into(), + forms, + }; + let one = |f: &str| vec![f.to_string()]; + let make = |forms: Vec| Entity { + id: eid("big"), + forms, + origin: EntityOrigin::Live, + }; + + // A whole line is not a name: 7 words. + let line = "I will never leave you, not ever."; + let e = make(vec![lf("en-US", one(line))]); + assert_eq!( + refusal(w.entities.put(e.clone())), + bounds(&e, EntityBound::FormWords) + ); + // Four words is the edge; five is over. 64 bytes is the edge; 65 is over. + w.entities + .put(Entity { + id: eid("four"), + ..make(vec![lf("en-US", one("Mary of the Isles"))]) + }) + .unwrap(); + let five = make(vec![lf("en-US", one("Mary of the Isles too"))]); + assert_eq!( + refusal(w.entities.put(five.clone())), + bounds(&five, EntityBound::FormWords) + ); + let wide = make(vec![lf("en-US", one(&"a".repeat(65)))]); + assert_eq!( + refusal(w.entities.put(wide.clone())), + bounds(&wide, EntityBound::FormBytes) + ); + w.entities + .put(Entity { + id: eid("edge"), + ..make(vec![lf("en-US", one(&"a".repeat(64)))]) + }) + .unwrap(); + + // 8 forms per language is the edge. + let forms = |n: usize| (0..n).map(|i| format!("form{i}")).collect::>(); + w.entities + .put(Entity { + id: eid("eight"), + ..make(vec![lf("en-US", forms(8))]) + }) + .unwrap(); + let nine = make(vec![lf("en-US", forms(9))]); + assert_eq!( + refusal(w.entities.put(nine.clone())), + bounds(&nine, EntityBound::Forms) + ); + + // 8 languages is the edge. + let langs = |n: usize| { + (0..n) + .map(|i| lf(&format!("x-l{i}"), one("Form"))) + .collect::>() + }; + w.entities + .put(Entity { + id: eid("langs"), + ..make(langs(8)) + }) + .unwrap(); + let many = make(langs(9)); + assert_eq!( + refusal(w.entities.put(many.clone())), + bounds(&many, EntityBound::Languages) + ); + + // Soul's probe: 200 junk languages of 5000 long forms each. + let long: Vec = (0..5000) + .map(|i| format!("form number {i} of a long free text channel")) + .collect(); + let probe = make( + (0..200) + .map(|l| lf(&format!("x-{l}\n\u{7}"), long.clone())) + .collect(), + ); + assert_eq!( + refusal(w.entities.put(probe.clone())), + bounds(&probe, EntityBound::Languages) + ); + assert!(w.entities.get(&eid("big")).unwrap().is_none()); + + // Language tags are BCP 47 in form. + for bad in [ + "", + "e", + "en_US", + "x-0\n\u{7}", + "-en", + "en-", + "en--US", + "en-US-abcdefghi", + "1n", + "\u{65e5}\u{672c}", + "en-US-aaaaaaaa-bbbbbbbb-cccccccc-dddddddd-eeeeeeee", + "en-aaaaaaaa-aaaaaaaa-aaaaaaaa-bbbbbb", + ] { + let e = make(vec![lf(bad, one("Form"))]); + assert_eq!( + refusal(w.entities.put(e.clone())), + bounds(&e, EntityBound::LanguageTag), + "{bad:?}" + ); + } + for (i, ok) in [ + "en", + "en-US", + "zh-Hant-TW", + "x-klingon", + "de-CH-1996", + "en-aaaaaaaa-aaaaaaaa-aaaaaaaa-bbbbb", + ] + .into_iter() + .enumerate() + { + w.entities + .put(Entity { + id: eid(&format!("tag-{i}")), + ..make(vec![lf(ok, one("Form"))]) + }) + .unwrap(); + } + assert!(w.entities.get(&eid("big")).unwrap().is_none()); +} + +#[test] +fn item_language_is_a_tag() { + let mut w = world(); + let g = group_in_bucket(10); + for (n, bad) in ["", "en_US", "x-0\n\u{7}"].into_iter().enumerate() { + assert!(matches!( + refusal( + w.corpus + .add(item(n + 1, bad, "Nibu left.", vec![], &g), &w.entities) + ), + StoreRefusal::Malformed { .. } + )); + } +} + +#[test] +fn entities_are_append_only_and_replay_needs_them() { + let mut w = world(); + let base = ent("z", "en-US", &["Zed"]); + w.entities.put(base.clone()).unwrap(); + let not_append = StoreRefusal::EntityNotAppendOnly { id: eid("z") }; + + let mut prepend = base.clone(); + prepend.forms.insert( + 0, + LanguageForms { + language: "fr".into(), + forms: vec!["Z\u{e8}de".into()], + }, + ); + assert_eq!(refusal(w.entities.put(prepend)), not_append); + let mut persona = base.clone(); + persona.origin = EntityOrigin::Cast { + persona: "someone-else".into(), + }; + assert_eq!(refusal(w.entities.put(persona)), not_append); + let mut duplicate = base.clone(); + duplicate.forms.push(LanguageForms { + language: "en-US".into(), + forms: vec!["Zeddy".into()], + }); + assert!(matches!( + refusal(w.entities.put(duplicate)), + StoreRefusal::Malformed { .. } + )); + + // A corpus replay against an entity store that lacks the speaker is refused. + let empty = EntityStore::open(&w.dir.path().join("empty.cc")).unwrap(); + let again = CorpusItem { + id: "item:nibu:2".into(), + ..item(1, "en-US", "Nibu says hello.", vec![], &group_in_bucket(10)) + }; + assert_eq!( + refusal(w.corpus.add(again, &empty)), + StoreRefusal::UnknownEntity { + entity: eid("nibu") + } + ); +} diff --git a/crates/weksa/tests/forms.rs b/crates/weksa/tests/forms.rs index eac5dc0..61b62e1 100644 --- a/crates/weksa/tests/forms.rs +++ b/crates/weksa/tests/forms.rs @@ -1,6 +1,6 @@ mod common; use common::*; -use weksa::packet::{EncoderId, Grade, NodeId, PacketId, ProperName, Referent}; +use weksa::packet::{EncoderId, Grade, NodeId, PacketId, Referent}; use weksa::{Family, InvId, OptionLabel, OptionSet}; /// Soul's probe: a whole corpus line, and the snake_case form that would @@ -77,9 +77,6 @@ fn a_malformed_value_does_not_deserialize() { referent["id"] = SNAKE.into(); assert!(serde_json::from_value::(referent.clone()).is_err()); referent["id"] = "n1".into(); - referent["name"] = LINE.into(); - assert!(serde_json::from_value::(referent.clone()).is_err()); - referent["name"] = "Nibu".into(); referent["concept"] = "concept:01".into(); assert!(serde_json::from_value::(referent).is_err()); assert!(serde_json::from_str::("1001").is_err()); @@ -87,64 +84,6 @@ fn a_malformed_value_does_not_deserialize() { assert!(serde_json::from_str::("[]").is_err()); } -#[test] -fn proper_name_bounds() { - for ok in [ - "Nibu", - "Brokkr Ivaldi", - "O'Neil", - "O\u{2019}Neil", - "Jean-Luc", - "A B C D", - ] { - assert_eq!(ProperName::parse(ok).unwrap().as_str(), ok); - } - // A caseless script is a name; so is a capital beyond ASCII. - assert!(ProperName::parse("\u{5c71}\u{7530}").is_ok()); - assert!(ProperName::parse("\u{c9}owyn").is_ok()); - for bad in [ - LINE, - "A B C D E", - "Nibu.", - "Nibu, Ivaldi", - "Nibu Ivaldi", - " Nibu", - "Nibu ", - "", - "Nibu2", - "'Nibu", - "-Nibu", - "Nibu-", - "Nibu'", - "Nibu_Ivaldi", - ] { - assert!(ProperName::parse(bad).is_err(), "{bad:?}"); - } - // 24 chars a token, 4 tokens and 64 bytes are the bounds. - let token24 = format!("A{}", "b".repeat(23)); - assert!(ProperName::parse(&token24).is_ok()); - assert!(ProperName::parse(&format!("{token24}b")).is_err()); - let sized = |lens: [usize; 4]| { - lens.iter() - .map(|n| format!("A{}", "b".repeat(n - 1))) - .collect::>() - .join(" ") - }; - let at_limit = sized([15, 15, 15, 16]); - assert_eq!(at_limit.len(), 64); - assert!(ProperName::parse(&at_limit).is_ok()); - let over = sized([15, 15, 15, 17]); - assert_eq!(over.len(), 65); - assert!(ProperName::parse(&over).is_err()); - // The byte bound counts bytes, not characters. - let wide = format!( - "{0} {0} {0}", - "\u{e9}".repeat(11).replace('\u{e9}', "\u{c9}") - ); - assert!(wide.chars().count() < 64 && wide.len() > 64); - assert!(ProperName::parse(&wide).is_err()); -} - #[test] fn inv_id_canonical() { for bad in [ @@ -211,38 +150,3 @@ fn grade_bound_is_the_type() { assert_eq!(grade["minimum"], 0, "{grade}"); assert_eq!(schema["$defs"]["NodeId"]["pattern"], "^n[1-9][0-9]{0,3}$"); } - -#[test] -fn proper_name_bound_in_chars() { - // The token bound counts characters: 24 two-byte letters are 48 bytes and parse. - let cyrillic = |n: usize| "\u{411}".repeat(n); - assert_eq!(cyrillic(24).len(), 48); - assert!(ProperName::parse(&cyrillic(24)).is_ok()); - assert!(ProperName::parse(&cyrillic(25)).is_err()); - // The 64-byte bound still counts bytes: two 48-byte tokens are 97 bytes. - assert!(ProperName::parse(&format!("{0} {0}", cyrillic(24))).is_err()); -} - -#[test] -fn proper_name_is_form_only() { - // A name is bounded text: no case rule, so lowercase, hyphenated and - // caseless-script tokens parse. - for ok in [ - "nibu", - "Nibu ivaldi", - "take-your-hand-off-that", - "Take-your-hand-off-that", - "\u{3131}-control-unless-you", - "\u{5c71}\u{7530}", - ] { - assert_eq!(ProperName::parse(ok).unwrap().as_str(), ok); - } - // The bounds still hold: 5 tokens, 65 bytes, a sentence's punctuation. - assert!(ProperName::parse("a b c d e").is_err()); - assert!(ProperName::parse(&"a".repeat(25)).is_err()); - let token = |n: usize| "a".repeat(n); - let over = [token(15), token(15), token(15), token(17)].join(" "); - assert_eq!(over.len(), 65); - assert!(ProperName::parse(&over).is_err()); - assert!(ProperName::parse("take your hand, now.").is_err()); -} diff --git a/crates/weksa/tests/seeds.rs b/crates/weksa/tests/seeds.rs new file mode 100644 index 0000000..3cec02d --- /dev/null +++ b/crates/weksa/tests/seeds.rs @@ -0,0 +1,38 @@ +//! The committed seed stores replay through today's doors, so a rule that +//! tightens cannot leave a seed the stores would now refuse. +mod common; +use std::path::Path; +use weksa::{CorpusStore, EntityStore}; + +#[test] +fn committed_seeds_replay_through_the_doors() { + let data = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data"); + let seeded_entities = EntityStore::open(&data.join("entity/entity.cc")).unwrap(); + let seeded_corpus = CorpusStore::open(&data.join("corpus/corpus.cc")).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let mut entities = EntityStore::open(&dir.path().join("e.cc")).unwrap(); + let all = seeded_entities.all().unwrap(); + assert!(!all.is_empty()); + for e in all { + entities.put(e).unwrap(); + } + let mut corpus = CorpusStore::open(&dir.path().join("c.cc")).unwrap(); + let items = seeded_corpus.all().unwrap(); + assert!(!items.is_empty()); + for item in items { + let withdrawn = item.withdrawn; + let id = item.id.clone(); + corpus + .add( + weksa::CorpusItem { + withdrawn: false, + ..item + }, + &entities, + ) + .unwrap(); + if withdrawn { + corpus.withdraw(&id).unwrap(); + } + } +} diff --git a/crates/weksa/tests/split_schema.rs b/crates/weksa/tests/split_schema.rs index 81e9cb1..d69c5cb 100644 --- a/crates/weksa/tests/split_schema.rs +++ b/crates/weksa/tests/split_schema.rs @@ -7,7 +7,8 @@ fn item(group: &str, source: Source) -> CorpusItem { id: "item:t:1".into(), text: "t".into(), language: "en-US".into(), - speaker: "s".into(), + speaker: weksa::EntityId::parse("entity:s").unwrap(), + names: vec![], group: group.into(), prior_turns: vec![], source, @@ -54,7 +55,7 @@ fn p1_split_deterministic() { } // The split reads the group and nothing else: speaker, text and id do not move it. let mut other = item("scene-9", cast()); - other.speaker = "someone".into(); + other.speaker = weksa::EntityId::parse("entity:someone").unwrap(); other.text = "different".into(); other.id = "item:other:7".into(); assert_eq!(SplitScheme::V1.split(&other), a); diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs index c8aea2c..2c6ab7c 100644 --- a/crates/weksa/tests/stores.rs +++ b/crates/weksa/tests/stores.rs @@ -4,14 +4,14 @@ use tempfile::TempDir; use weksa::corpus::{CorpusItem, ModelFamily, Source, Turn}; use weksa::inventory::{Change, seed}; use weksa::{ - AddOutcome, CorpusStore, Family, InventoryEntry, InventoryRevision, InventoryStore, - PacketStore, Refusal, Slot, SplitScheme, StoreError, StoreRefusal, ValueKind, + AddOutcome, CorpusStore, EntityStore, Family, InventoryEntry, InventoryRevision, + InventoryStore, PacketStore, Refusal, Slot, SplitScheme, StoreError, StoreRefusal, ValueKind, }; fn item(key: &str, text: &str) -> CorpusItem { CorpusItem { prior_turns: vec![Turn { - speaker: "player".into(), + speaker: eid("player"), text: "hello".into(), }], ..item_in(key, text, "scene-1", cast()) @@ -25,6 +25,10 @@ fn refusal(r: Result) -> StoreRefusal { } } +fn ents(d: &TempDir) -> EntityStore { + entities_in(d.path()) +} + fn dir() -> TempDir { tempfile::tempdir().unwrap() } @@ -46,13 +50,17 @@ fn s1_item_immutable() { let path = d.path().join("c.cc"); let mut store = CorpusStore::open(&path).unwrap(); assert_eq!( - store.add(item("item:nibu:1", "Mind the hatch.")).unwrap(), + store + .add(item("item:nibu:1", "Mind the hatch."), &ents(&d)) + .unwrap(), AddOutcome::Added ); // An equal item is idempotent, even from a reopened store. let mut store = CorpusStore::open(&path).unwrap(); assert_eq!( - store.add(item("item:nibu:1", "Mind the hatch.")).unwrap(), + store + .add(item("item:nibu:1", "Mind the hatch."), &ents(&d)) + .unwrap(), AddOutcome::Unchanged ); let refused = || StoreRefusal::S1ItemImmutable { @@ -71,7 +79,7 @@ fn s1_item_immutable() { v.language = "en-GB".into(); variants.push(v); let mut v = base.clone(); - v.speaker = "brokkr".into(); + v.speaker = eid("brokkr"); variants.push(v); let mut v = base.clone(); v.group = "scene-2".into(); @@ -86,12 +94,12 @@ fn s1_item_immutable() { v.supersedes = Some("item:nibu:9".into()); variants.push(v); for v in variants { - assert_eq!(refusal(store.add(v.clone())), refused(), "{v:?}"); + assert_eq!(refusal(store.add(v.clone(), &ents(&d))), refused(), "{v:?}"); } assert_eq!(store.get("item:nibu:1").unwrap(), Some(base.clone())); // A withdrawn item is a different item: re-adding it live is refused too. store.withdraw("item:nibu:1").unwrap(); - assert_eq!(refusal(store.add(base)), refused()); + assert_eq!(refusal(store.add(base, &ents(&d))), refused()); } #[test] @@ -115,7 +123,7 @@ fn corpus_keys_are_canonical() { ]; for (i, key) in not_canonical.into_iter().enumerate() { assert_eq!( - refusal(store.add(item(key, &format!("line {i}")))), + refusal(store.add(item(key, &format!("line {i}")), &ents(&d))), malformed(key, "item key is not item::"), "{key}" ); @@ -126,29 +134,24 @@ fn corpus_keys_are_canonical() { let canonical = ["item:a:1", "item:a-b:10", "item:7:2", "item:ab:9", &longest]; for (i, key) in canonical.into_iter().enumerate() { assert_eq!( - store.add(item(key, &format!("fine {i}"))).unwrap(), + store + .add(item(key, &format!("fine {i}")), &ents(&d)) + .unwrap(), AddOutcome::Added, "{key}" ); } - // An alias of a stored key resolves to nothing. - let w = world(); - let mut packets = PacketStore::open(&w.dir.path().join("p.cc")).unwrap(); - let mut p = good(); - p.provenance.corpus_item = Some("item:nibu:01".into()); - assert_eq!( - refusal(packets.admit(p, &w.inventory, &w.corpus)), - StoreRefusal::UnknownCorpusItem { - key: "item:nibu:01".into() - } - ); + // An alias of a stored key is not a corpus item id at all. + assert!(weksa::CorpusItemId::parse("item:nibu:01").is_err()); } #[test] fn s6_text_held_by_another_key() { let d = dir(); let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); - store.add(item("item:nibu:1", "Mind the hatch.")).unwrap(); + store + .add(item("item:nibu:1", "Mind the hatch."), &ents(&d)) + .unwrap(); let elsewhere = |key: &str, text: &str| CorpusItem { group: "scene-2".into(), ..item(key, text) @@ -159,11 +162,11 @@ fn s6_text_held_by_another_key() { }; // The same text in another group, and in the same group, is refused. assert_eq!( - refusal(store.add(elsewhere("item:nibu:2", "Mind the hatch."))), + refusal(store.add(elsewhere("item:nibu:2", "Mind the hatch."), &ents(&d))), held("item:nibu:2", "item:nibu:1") ); assert_eq!( - refusal(store.add(item("item:nibu:2", "Mind the hatch."))), + refusal(store.add(item("item:nibu:2", "Mind the hatch."), &ents(&d))), held("item:nibu:2", "item:nibu:1") ); // Case, spacing and punctuation variants are the same text. @@ -174,38 +177,51 @@ fn s6_text_held_by_another_key() { "Mind the hatch.\n", ] { assert_eq!( - refusal(store.add(elsewhere("item:nibu:2", text))), + refusal(store.add(elsewhere("item:nibu:2", text), &ents(&d))), held("item:nibu:2", "item:nibu:1"), "{text:?}" ); } store - .add(item("item:nibu:3", "\u{c4}rger \u{fc}ber \u{d6}l")) + .add( + item("item:nibu:3", "\u{c4}rger \u{fc}ber \u{d6}l"), + &ents(&d), + ) .unwrap(); assert_eq!( - refusal(store.add(elsewhere("item:nibu:4", "\u{e4}rger \u{dc}BER \u{f6}l"))), + refusal(store.add( + elsewhere("item:nibu:4", "\u{e4}rger \u{dc}BER \u{f6}l"), + &ents(&d) + )), held("item:nibu:4", "item:nibu:3") ); // Text that differs in a letter or digit is a different text. - store.add(item("item:nibu:5", "Mind the hatch 2")).unwrap(); - store.add(item("item:nibu:6", "Mind the hatches.")).unwrap(); + store + .add(item("item:nibu:5", "Mind the hatch 2"), &ents(&d)) + .unwrap(); + store + .add(item("item:nibu:6", "Mind the hatches."), &ents(&d)) + .unwrap(); // A withdrawn holder still holds. store.withdraw("item:nibu:1").unwrap(); assert_eq!( - refusal(store.add(elsewhere("item:nibu:2", "mind the hatch"))), + refusal(store.add(elsewhere("item:nibu:2", "mind the hatch"), &ents(&d))), held("item:nibu:2", "item:nibu:1") ); // Superseding a different key does not excuse a copy of another item. let mut copy = item("item:nibu:8", "Mind the hatches!"); copy.supersedes = Some("item:nibu:5".into()); - assert_eq!(refusal(store.add(copy)), held("item:nibu:8", "item:nibu:6")); + assert_eq!( + refusal(store.add(copy, &ents(&d))), + held("item:nibu:8", "item:nibu:6") + ); // A superseder of the holder, in the holder's group, is Added. let mut fix = item("item:nibu:2", "mind the hatch"); fix.supersedes = Some("item:nibu:1".into()); - assert_eq!(store.add(fix).unwrap(), AddOutcome::Added); + assert_eq!(store.add(fix, &ents(&d)).unwrap(), AddOutcome::Added); // The superseded holder's text stays held against a third key. assert_eq!( - refusal(store.add(elsewhere("item:nibu:7", "Mind the hatch."))), + refusal(store.add(elsewhere("item:nibu:7", "Mind the hatch."), &ents(&d))), held("item:nibu:7", "item:nibu:1") ); } @@ -215,32 +231,32 @@ fn corpus_admission_rules() { let d = dir(); let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); assert_eq!( - refusal(store.add(item("nibu:1", "x"))), + refusal(store.add(item("nibu:1", "x"), &ents(&d))), malformed("nibu:1", "item key is not item::") ); assert_eq!( - refusal(store.add(item("item:nibu:x", "x"))), + refusal(store.add(item("item:nibu:x", "x"), &ents(&d))), malformed("item:nibu:x", "item key is not item::") ); let mut born_withdrawn = item("item:nibu:1", "x"); born_withdrawn.withdrawn = true; assert_eq!( - refusal(store.add(born_withdrawn)), + refusal(store.add(born_withdrawn, &ents(&d))), malformed("item:nibu:1", "an item is admitted live, then withdrawn") ); let mut orphan = item("item:nibu:1", "x"); orphan.supersedes = Some("item:nibu:9".into()); assert_eq!( - refusal(store.add(orphan)), + refusal(store.add(orphan, &ents(&d))), malformed("item:nibu:1", "supersedes an item that does not exist") ); // Nothing was written by the refusals. assert!(store.all().unwrap().is_empty()); // A correction is a new item that supersedes the old one. - store.add(item("item:nibu:1", "old")).unwrap(); + store.add(item("item:nibu:1", "old"), &ents(&d)).unwrap(); let mut fix = item("item:nibu:2", "new"); fix.supersedes = Some("item:nibu:1".into()); - assert_eq!(store.add(fix).unwrap(), AddOutcome::Added); + assert_eq!(store.add(fix, &ents(&d)).unwrap(), AddOutcome::Added); assert_eq!( refusal(store.withdraw("item:nibu:7")), malformed("item:nibu:7", "no such item") @@ -251,12 +267,12 @@ fn corpus_admission_rules() { fn s5_supersede_keeps_the_group() { let d = dir(); let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); - store.add(item("item:nibu:1", "old")).unwrap(); + store.add(item("item:nibu:1", "old"), &ents(&d)).unwrap(); let mut elsewhere = item("item:nibu:2", "new"); elsewhere.supersedes = Some("item:nibu:1".into()); elsewhere.group = "scene-2".into(); assert_eq!( - refusal(store.add(elsewhere)), + refusal(store.add(elsewhere, &ents(&d))), StoreRefusal::S5SupersedeGroup { key: "item:nibu:2".into(), expected: "scene-1".into(), @@ -266,7 +282,7 @@ fn s5_supersede_keeps_the_group() { assert!(store.get("item:nibu:2").unwrap().is_none()); let mut same = item("item:nibu:2", "new"); same.supersedes = Some("item:nibu:1".into()); - assert_eq!(store.add(same).unwrap(), AddOutcome::Added); + assert_eq!(store.add(same, &ents(&d)).unwrap(), AddOutcome::Added); } #[test] @@ -274,7 +290,9 @@ fn withdrawing_keeps_the_text_and_the_number() { let d = dir(); let path = d.path().join("c.cc"); let mut store = CorpusStore::open(&path).unwrap(); - store.add(item("item:nibu:1", "keep me")).unwrap(); + store + .add(item("item:nibu:1", "keep me"), &ents(&d)) + .unwrap(); assert_eq!(store.next_key("nibu").unwrap(), "item:nibu:2"); assert_eq!(store.next_key("other").unwrap(), "item:other:1"); store.withdraw("item:nibu:1").unwrap(); @@ -604,7 +622,7 @@ fn packet_admission_goes_through_the_validator() { let mut store = PacketStore::open(&w.dir.path().join("p.cc")).unwrap(); let mut bad = good(); bad.schema_version = 5; - match store.admit(bad, &w.inventory, &w.corpus) { + match store.admit(bad, &w.inventory, &w.corpus, &w.entities) { Err(e @ StoreError::Invalid(_)) => { assert!(e.to_string().contains("V1"), "{e}"); let StoreError::Invalid(r) = e else { @@ -615,13 +633,17 @@ fn packet_admission_goes_through_the_validator() { other => panic!("{other:?}"), } assert!(store.all().unwrap().is_empty()); - store.admit(good(), &w.inventory, &w.corpus).unwrap(); + store + .admit(good(), &w.inventory, &w.corpus, &w.entities) + .unwrap(); // The same packet is idempotent; a different one under the same key is refused. - store.admit(good(), &w.inventory, &w.corpus).unwrap(); + store + .admit(good(), &w.inventory, &w.corpus, &w.entities) + .unwrap(); let mut other = good(); other.provenance.confidence = grade(1); assert_eq!( - refusal(store.admit(other, &w.inventory, &w.corpus)), + refusal(store.admit(other, &w.inventory, &w.corpus, &w.entities)), StoreRefusal::PacketImmutable { key: PACKET_ID.into() } @@ -637,17 +659,15 @@ fn admission_reads_the_store_inventory() { let d = dir(); let mut corpus = empty_corpus(&d); corpus - .add(item_in( - "item:nibu:1", - "Nibu says mind the hatch.", - "g", - cast(), - )) + .add( + item_in("item:nibu:1", "Nibu says mind the hatch.", "g", cast()), + &ents(&d), + ) .unwrap(); let mut inventory = seeded(&d.path().join("i.cc")); let mut packets = PacketStore::open(&d.path().join("p.cc")).unwrap(); assert_eq!( - match packets.admit(good(), &inventory, &corpus) { + match packets.admit(good(), &inventory, &corpus, &ents(&d)) { Err(StoreError::Invalid(r)) => r, other => panic!("{other:?}"), }, @@ -667,26 +687,36 @@ fn admission_reads_the_store_inventory() { ext, ], ); - packets.admit(good(), &inventory, &corpus).unwrap(); + packets + .admit(good(), &inventory, &corpus, &ents(&d)) + .unwrap(); } #[test] fn corpus_item_must_resolve() { let w = world(); let mut packets = PacketStore::open(&w.dir.path().join("p.cc")).unwrap(); - for key in ["item:nibu:9", "banana"] { - let mut p = good(); - p.provenance.corpus_item = Some(key.into()); - assert_eq!( - refusal(packets.admit(p, &w.inventory, &w.corpus)), - StoreRefusal::UnknownCorpusItem { key: key.into() } - ); - } - assert!(packets.all().unwrap().is_empty()); - // A packet minted for no item needs none. let mut p = good(); - p.provenance.corpus_item = None; - packets.admit(p, &w.inventory, &w.corpus).unwrap(); + p.provenance.corpus_item = weksa::CorpusItemId::parse("item:nibu:9").unwrap(); + assert_eq!( + refusal(packets.admit(p, &w.inventory, &w.corpus, &w.entities)), + StoreRefusal::UnknownCorpusItem { + key: "item:nibu:9".into() + } + ); + assert!(packets.all().unwrap().is_empty()); + // Free text is not an item id, in Rust or on the wire; a packet with no item + // cannot be built or read back. + for bad in ["banana", "", "the reference sentence goes right here"] { + assert!(weksa::CorpusItemId::parse(bad).is_err(), "{bad:?}"); + } + let mut wire = serde_json::to_value(good().provenance).unwrap(); + for bad in [serde_json::Value::Null, "banana".into()] { + wire["corpus_item"] = bad; + assert!(serde_json::from_value::(wire.clone()).is_err()); + } + wire.as_object_mut().unwrap().remove("corpus_item"); + assert!(serde_json::from_value::(wire).is_err()); } #[test] @@ -698,7 +728,7 @@ fn store_roundtrip() { let p = good(); PacketStore::open(&packets) .unwrap() - .admit(p.clone(), &w.inventory, &w.corpus) + .admit(p.clone(), &w.inventory, &w.corpus, &w.entities) .unwrap(); assert_eq!( PacketStore::open(&packets).unwrap().get(PACKET_ID).unwrap(), @@ -732,7 +762,10 @@ fn store_roundtrip() { let corpus = d.path().join("c.cc"); let it = item("item:nibu:1", "round trip"); - CorpusStore::open(&corpus).unwrap().add(it.clone()).unwrap(); + CorpusStore::open(&corpus) + .unwrap() + .add(it.clone(), &ents(&d)) + .unwrap(); assert_eq!( CorpusStore::open(&corpus) .unwrap() @@ -810,3 +843,29 @@ fn the_committed_stores_load_and_hold_clean_data() { ); assert_eq!(items.len(), 3); } + +#[test] +fn seed_reseeded() { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let entities = EntityStore::open(&root.join("data/entity/entity.cc")).unwrap(); + let ids: Vec = entities + .all() + .unwrap() + .iter() + .map(|e| e.id.to_string()) + .collect(); + assert_eq!(ids, ["entity:brokkr", "entity:nibu"]); + // The committed corpus replays through the door with its entities: every + // speaker resolves and every span is a stored form. + let items = CorpusStore::open(&root.join("data/corpus/corpus.cc")) + .unwrap() + .all() + .unwrap(); + assert_eq!(items.len(), 3); + let d = dir(); + let mut replay = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + for i in items { + assert!(entities.get(&i.speaker).unwrap().is_some(), "{}", i.id); + replay.add(i, &entities).unwrap(); + } +} diff --git a/crates/weksa/tests/validate.rs b/crates/weksa/tests/validate.rs index 1d4eaa6..cdf7644 100644 --- a/crates/weksa/tests/validate.rs +++ b/crates/weksa/tests/validate.rs @@ -360,7 +360,6 @@ fn lowering_input_carries_no_trace_id_or_provenance() { p.trace.rejected_readings = vec!["SECRET-READING".into()]; let input = validate(&p, &inventory()).unwrap(); let seen = format!("{input:?}"); - assert!(seen.contains("Nibu"), "the input carries the name: {seen}"); for hidden in [PACKET_ID, ENCODER_ID, "item:nibu:1", "SECRET"] { assert!(!seen.contains(hidden), "{hidden} reached the input: {seen}"); } @@ -396,7 +395,7 @@ fn referent(id: &str) -> Referent { id: nd(id), kind: iv(PERSON), concept: iv(CONCEPT_A), - name: None, + entity: None, features: vec![], } } @@ -507,7 +506,6 @@ fn two_referents() -> Packet { let mut p = good(); let mut r = p.referents[0].clone(); r.id = nd("n3"); - r.name = None; p.referents.push(r); p } diff --git a/data/corpus/corpus.cc b/data/corpus/corpus.cc index dd24f7b..619a875 100644 --- a/data/corpus/corpus.cc +++ b/data/corpus/corpus.cc @@ -1 +1 @@ -cultcache.store.v1weksa.corpus_itemweksa.corpus_itemweksa.corpus_item.v1weksa.corpus_itemV{"schemaName":"weksa.corpus_item","schemaVersion":"weksa.corpus_item.v1","members":[]}weksa.corpus_itemitem:brokkr:1weksa.corpus_item2026-10-29T13:16:54ZĠitem:brokkr:1,The command has no receipt. It is not truth.en-USbrokkrbrokkr-edda-voiceModelWrittenOpenAicodex-local-source-aware-draft2026-10-29”item:nibu:1weksa.corpus_item2026-10-29T13:16:54Zitem:nibu:1nTake your hand off that control unless you want your nervous system to become my least useful circuit diagram.en-USnibunibu-proof-001ModelWrittenUnrecorded2026-10-29”item:nibu:2weksa.corpus_item2026-10-29T13:16:54Z䚫item:nibu:2ّDo not touch that control. It is damaged; one touch may put you directly in harm's way. I am preventing an avoidable injury, not asking politely.en-USnibunibu-proof-001ModelWrittenUnrecorded2026-10-29 \ No newline at end of file +cultcache.store.v1weksa.corpus_itemweksa.corpus_itemweksa.corpus_item.v1weksa.corpus_itemV{"schemaName":"weksa.corpus_item","schemaVersion":"weksa.corpus_item.v1","members":[]}weksa.corpus_itemitem:brokkr:1weksa.corpus_item2026-10-29T14:59:18ZĨitem:brokkr:1,The command has no receipt. It is not truth.en-USentity:brokkrbrokkr-edda-voiceModelWrittenOpenAicodex-local-source-aware-draft2026-10-29item:nibu:1weksa.corpus_item2026-10-29T14:59:18Zɛitem:nibu:1nTake your hand off that control unless you want your nervous system to become my least useful circuit diagram.en-USentity:nibunibu-proof-001ModelWrittenUnrecorded2026-10-29item:nibu:2weksa.corpus_item2026-10-29T14:59:18Z웫item:nibu:2ّDo not touch that control. It is damaged; one touch may put you directly in harm's way. I am preventing an avoidable injury, not asking politely.en-USentity:nibunibu-proof-001ModelWrittenUnrecorded2026-10-29 \ No newline at end of file diff --git a/data/entity/entity.cc b/data/entity/entity.cc new file mode 100644 index 0000000..f561201 --- /dev/null +++ b/data/entity/entity.cc @@ -0,0 +1 @@ +cultcache.store.v1weksa.entityweksa.entityweksa.entity.v1weksa.entityL{"schemaName":"weksa.entity","schemaVersion":"weksa.entity.v1","members":[]}weksa.entityentity:brokkrweksa.entity2026-10-29T14:59:18Z-entity:brokkren-USBrokkrCastbrokkrentity:nibuweksa.entity2026-10-29T14:59:18Z'entity:nibuen-USNibuCastnibu \ No newline at end of file diff --git a/docs/interlingua-standard.md b/docs/interlingua-standard.md index f06bcd3..a59a963 100644 --- a/docs/interlingua-standard.md +++ b/docs/interlingua-standard.md @@ -100,13 +100,19 @@ Every id is an opaque form type, parsed when it is built and when a store reads it: an inventory id is `:` with `n` a canonical decimal of at least 1 (`role:1`; `role:01` is not an id), a node id is `n` with `k` in 1..=9999, a packet id is `packet:<16 hex>` and an encoder id is -`encoder:<16 hex>`. The free text a packet may hold is a referent's `name`, -a `ProperName` of 1 to 4 space-joined tokens of letters, apostrophes and -hyphens within 64 bytes, and the `trace`. A name is bounded text, not a judged -one: no door compares packet text with the corpus. Leakage of scored text is -owned outside the validator, by the loop's context builder (what each model -call sees) and the scorer (names are masked before scoring); the validator -owns form and capacity. The trace is stripped before lowering. Concept ids are opaque and carry a gloss in the inventory; concepts +`encoder:<16 hex>`, an entity id is `entity:` (`[a-z0-9]` and inner +`-`, 1 to 48 bytes) and a corpus item id is `item::`. The only free +text a packet holds is the `trace` (`notes` and `rejected_readings`), which the +validator neither bounds nor reads and which is stripped before lowering: a +referent names its entity by id, and the entity store (`weksa.entity`) holds +the entity's forms per language. The packet store refuses a referent whose +entity the store lacks, a packet whose `corpus_item` the corpus lacks, and one +whose entity is neither named on that item nor its speaker or a prior-turn +speaker (a packet without a corpus item has no door in this store): +an id-set check, not a text rule. Leakage of scored text is owned outside the +validator, by the loop's context builder (what each model call sees) and the +scorer; the validator owns form and capacity. The trace is stripped before +lowering. Concept ids are opaque and carry a gloss in the inventory; concepts are open-class and grow, while the other families are closed-class. ```text @@ -115,7 +121,7 @@ Packet provenance { source, corpus_item, encoder, confidence } context { setting, medium, participants, audience } discourse { speech_act, addressee, features } - referents [ { id, kind, concept, name, features } ] + referents [ { id, kind, concept, entity, features } ] predications [ { id, concept, frame, roles: [ { role, filler } ], features } ] relations [ { relation, from, to, features } ] constraints { preserve, deterministic, max_latency_ms } @@ -165,8 +171,8 @@ reads at the packet's revision, never one the caller supplies. ### `provenance` `source` is `Authored`, `ModelDecomposition`, `GameTelemetry` or -`ImportedCorpus`. `corpus_item` is the key of the corpus item the packet was -minted for, `encoder` names what wrote it, and `confidence` is a grade. +`ImportedCorpus`. `corpus_item` is the required `item::` key of the +corpus item the packet was minted for, `encoder` names what wrote it, and `confidence` is a grade. ### `context` @@ -185,7 +191,7 @@ communicative pressure, not target-language register. Entities, groups, places, abstractions and the rest. `kind` is a referent-kind id (the inventory seeds `person`, `group`, `organism`, `place`, `artifact`, `institution`, `ecology`, `route`, `memory`, `event`, `abstract`), `concept` is -a concept id, and `name` is a proper name when the referent has one. +a concept id, and `entity` is the id of the named entity when the referent has one. ### `predications` diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 9279bf2..05be723 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -178,10 +178,12 @@ operator-attested annotations alone. Who annotates is question intake: cast entities from the cast fixtures, public-domain entities by the importer, and in live use by the persona before it names one. A corpus item carries its name spans (byte ranges of its text, each naming an entity and - equal to one of that entity's forms in the item's language), and its speaker + on extended grapheme cluster boundaries of its NFC text and equal to one of that + entity's NFC forms in the item's language; an entity holds at most 8 languages + of BCP 47-form tags, 8 forms each, a form at most 64 bytes and 4 words), and its speaker and prior-turn speakers are entity ids. The packet store refuses a packet whose referent cites an entity the store lacks, and a packet with a - `corpus_item` that cites an entity neither annotated on that item nor one of + `corpus_item` (required, typed `CorpusItemId`) that cites an entity neither annotated on that item nor one of its speakers: an id-set check, not a text rule. The scorer reads only the output and the reference and scores the whole of each. An honest name earns its credit ("Brokkr came back." against itself is 100), a wrong one loses it, diff --git a/schemas/cultnet/weksa.corpus_item.schema.json b/schemas/cultnet/weksa.corpus_item.schema.json index f44f011..40b431d 100644 --- a/schemas/cultnet/weksa.corpus_item.schema.json +++ b/schemas/cultnet/weksa.corpus_item.schema.json @@ -17,6 +17,13 @@ "language": { "type": "string" }, + "names": { + "description": "Which entities the text names, and where. Fixed with the text at intake.", + "type": "array", + "items": { + "$ref": "#/$defs/NameSpan" + } + }, "prior_turns": { "type": "array", "items": { @@ -27,7 +34,7 @@ "$ref": "#/$defs/Source" }, "speaker": { - "type": "string" + "$ref": "#/$defs/EntityId" }, "supersedes": { "description": "Key of the item this one corrects.", @@ -52,9 +59,14 @@ "prior_turns", "source", "admitted_on", - "withdrawn" + "withdrawn", + "names" ], "$defs": { + "EntityId": { + "type": "string", + "pattern": "^entity:[a-z0-9]([a-z0-9-]{0,46}[a-z0-9])?$" + }, "ModelFamily": { "description": "The family of the model that wrote a line.", "oneOf": [ @@ -76,6 +88,30 @@ } ] }, + "NameSpan": { + "description": "A byte range `[start, end)` of an item's text that names an entity. The\nstore admits it only when both ends sit on extended grapheme cluster\nboundaries (UAX #29) of the NFC text and the range's bytes equal one of the\nentity's forms in the item's language.", + "type": "object", + "properties": { + "end": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "entity": { + "$ref": "#/$defs/EntityId" + }, + "start": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "entity", + "start", + "end" + ] + }, "Source": { "oneOf": [ { @@ -196,7 +232,7 @@ "type": "object", "properties": { "speaker": { - "type": "string" + "$ref": "#/$defs/EntityId" }, "text": { "type": "string" diff --git a/schemas/cultnet/weksa.entity.schema.json b/schemas/cultnet/weksa.entity.schema.json new file mode 100644 index 0000000..bb08247 --- /dev/null +++ b/schemas/cultnet/weksa.entity.schema.json @@ -0,0 +1,103 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Entity", + "description": "Key: the entity id. Forms only grow: see `EntityStore::put`.", + "type": "object", + "properties": { + "forms": { + "type": "array", + "items": { + "$ref": "#/$defs/LanguageForms" + } + }, + "id": { + "$ref": "#/$defs/EntityId" + }, + "origin": { + "$ref": "#/$defs/EntityOrigin" + } + }, + "required": [ + "id", + "forms", + "origin" + ], + "$defs": { + "EntityId": { + "type": "string", + "pattern": "^entity:[a-z0-9]([a-z0-9-]{0,46}[a-z0-9])?$" + }, + "EntityOrigin": { + "description": "Where an entity was minted.", + "oneOf": [ + { + "description": "From a cast member's fixture.", + "type": "object", + "properties": { + "Cast": { + "type": "object", + "properties": { + "persona": { + "type": "string" + } + }, + "required": [ + "persona" + ] + } + }, + "additionalProperties": false, + "required": [ + "Cast" + ] + }, + { + "description": "From a public-domain work.", + "type": "object", + "properties": { + "PublicDomain": { + "type": "object", + "properties": { + "work": { + "type": "string" + } + }, + "required": [ + "work" + ] + } + }, + "additionalProperties": false, + "required": [ + "PublicDomain" + ] + }, + { + "description": "Minted live by a persona before it names the entity.", + "type": "string", + "const": "Live" + } + ] + }, + "LanguageForms": { + "description": "An entity's forms in one language, as the corpus item language tag spells it.\nThe citation form comes first.", + "type": "object", + "properties": { + "forms": { + "description": "1 to 64 bytes each, no control character, no edge whitespace, none repeated.", + "type": "array", + "items": { + "type": "string" + } + }, + "language": { + "type": "string" + } + }, + "required": [ + "language", + "forms" + ] + } + } +} diff --git a/schemas/cultnet/weksa.interlingua_packet.schema.json b/schemas/cultnet/weksa.interlingua_packet.schema.json index 634d0d5..24d7507 100644 --- a/schemas/cultnet/weksa.interlingua_packet.schema.json +++ b/schemas/cultnet/weksa.interlingua_packet.schema.json @@ -143,6 +143,10 @@ "audience" ] }, + "CorpusItemId": { + "type": "string", + "pattern": "^item:[a-z0-9]([a-z0-9-]{0,46}[a-z0-9])?:[1-9][0-9]*$" + }, "Discourse": { "type": "object", "properties": { @@ -182,6 +186,10 @@ "type": "string", "pattern": "^encoder:[0-9a-f]{16}$" }, + "EntityId": { + "type": "string", + "pattern": "^entity:[a-z0-9]([a-z0-9-]{0,46}[a-z0-9])?$" + }, "Extension": { "type": "object", "properties": { @@ -323,12 +331,6 @@ "features" ] }, - "ProperName": { - "description": "1 to 4 space-joined tokens of letters, apostrophes and hyphens.", - "type": "string", - "maxLength": 64, - "pattern": "^\\p{L}([\\p{L}'’-]{0,22}\\p{L})?( \\p{L}([\\p{L}'’-]{0,22}\\p{L})?){0,3}$" - }, "Provenance": { "type": "object", "properties": { @@ -336,11 +338,8 @@ "$ref": "#/$defs/Grade" }, "corpus_item": { - "description": "Key of the corpus item this packet was minted for.", - "type": [ - "string", - "null" - ] + "description": "The corpus item this packet was minted for. Required: the packet store\nadmits only packets that name their item.", + "$ref": "#/$defs/CorpusItemId" }, "encoder": { "$ref": "#/$defs/EncoderId" @@ -351,6 +350,7 @@ }, "required": [ "source", + "corpus_item", "encoder", "confidence" ] @@ -371,6 +371,17 @@ "description": "Concept id.", "$ref": "#/$defs/InvId" }, + "entity": { + "description": "The entity this referent names, when it has one. The entity store holds\nits wording; the packet holds only the id.", + "anyOf": [ + { + "$ref": "#/$defs/EntityId" + }, + { + "type": "null" + } + ] + }, "features": { "type": "array", "items": { @@ -383,17 +394,6 @@ "kind": { "description": "Referent-kind id.", "$ref": "#/$defs/InvId" - }, - "name": { - "description": "A proper name, the one place a referent carries wording. It is\nform-bounded text; the scorer masks it.", - "anyOf": [ - { - "$ref": "#/$defs/ProperName" - }, - { - "type": "null" - } - ] } }, "required": [ diff --git a/schemas/interlingua-packet.md b/schemas/interlingua-packet.md index d02c527..0d4a967 100644 --- a/schemas/interlingua-packet.md +++ b/schemas/interlingua-packet.md @@ -36,8 +36,9 @@ phonology, diachrony, and renderer rules. - `extensions`: namespaced client or setting pressure. - `trace`: debug and authoring notes, stripped before lowering. -Every open slot is an inventory id or a node id. The only free text is a -referent's proper `name` and the trace. +Every open slot is an inventory id, a node id, an entity id or a corpus item id. +The only free text is the trace, which the validator neither bounds nor reads +and which is stripped before lowering. ## Extension Contract