GameCult/Weksa
Watch
9
0
Fork
You've already forked Weksa
 0
Code Issues Pull requests 2 Projects Releases Packages Wiki Activity Actions

entity-door-lows (2d): attested origins, every Cf refused in forms, bytes-only form bound, S6 under NFKC + case fold keeping marks, no-alnum texts refused, CLI flags and echoes (squash of weksa-lowering/entity-door-lows f1c9815..57dacf4; Soul s1, s2)

Browse source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Eureka 2026-10-29 18:44:02 +01:00
commit 8d9c0270d1
16 changed files with 1413 additions and 84 deletions
Download patch file Download diff file Expand all files Collapse all files

5
crates/weksa/src/corpus.rs
View file

@ -17,10 +17,13 @@ pub struct Turn {
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub enum Source { pub enum Source {
/// A line from a cast member's hand-authored fixture. /// A line from a cast member's hand-authored fixture, with the human who
/// wrote it. A fixture that records no author is `ModelWritten` with family
/// `Unrecorded`, never this (`corpus-labels-tell-truth`).
CastAuthored { CastAuthored {
persona: String, persona: String,
fixture: String, fixture: String,
author: String,
}, },
OperatorWritten, OperatorWritten,
OperatorApproved { OperatorApproved {

12
crates/weksa/src/entity.rs
View file

@ -13,19 +13,21 @@ use crate::packet::EntityId;
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct LanguageForms { pub struct LanguageForms {
pub language: String, pub language: String,
/// 1 to 64 bytes each, no control character, no edge whitespace, none repeated. /// 1 to 64 bytes each, no control or invisible format character, no edge
/// whitespace, none repeated.
pub forms: Vec<String>, pub forms: Vec<String>,
} }
/// Where an entity was minted. /// Where an entity was minted.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub enum EntityOrigin { pub enum EntityOrigin {
/// From a cast member's fixture. /// From a cast member's fixture: the persona and the fixture path the
Cast { persona: String }, /// entity came from.
Cast { persona: String, fixture: String },
/// From a public-domain work. /// From a public-domain work.
PublicDomain { work: String }, PublicDomain { work: String },
/// Minted live by a persona before it names the entity. /// Minted live by the named persona before it names the entity.
Live, Live { persona: String },
} }
/// Key: the entity id. Forms only grow: see `EntityStore::put`. /// Key: the entity id. Forms only grow: see `EntityStore::put`.

111
crates/weksa/src/form.rs
View file

@ -86,3 +86,114 @@ pub(crate) fn is_slug(s: &str) -> bool {
&& b[0] != b'-' && b[0] != b'-'
&& b[b.len() - 1] != b'-' && b[b.len() - 1] != b'-'
} }
/// A General_Category Cf code point, or the line and paragraph separators
/// U+2028 and U+2029, other than the joiners ZWNJ U+200C and ZWJ U+200D. The
/// ranges are Unicode 15.1 Cf; `tests/format_refused.txt` is the full list from
/// Python's `unicodedata` that the table is swept against.
pub(crate) fn is_format(c: char) -> bool {
matches!(
c,
'\u{ad}'
| '\u{600}'..='\u{605}'
| '\u{61c}'
| '\u{6dd}'
| '\u{70f}'
| '\u{890}'..='\u{891}'
| '\u{8e2}'
| '\u{180e}'
| '\u{200b}'
| '\u{200e}'..='\u{200f}'
| '\u{2028}'..='\u{202e}'
| '\u{2060}'..='\u{2064}'
| '\u{2066}'..='\u{206f}'
| '\u{feff}'
| '\u{fff9}'..='\u{fffb}'
| '\u{110bd}'
| '\u{110cd}'
| '\u{13430}'..='\u{1343f}'
| '\u{1bca0}'..='\u{1bca3}'
| '\u{1d173}'..='\u{1d17a}'
| '\u{e0001}'
| '\u{e0020}'..='\u{e007f}'
)
}
/// ZWNJ and ZWJ: Cf, but scripts and emoji sequences need them inside a form.
pub(crate) fn is_joiner(c: char) -> bool {
matches!(c, '\u{200c}' | '\u{200d}')
}
/// A character a name or form may not hold: every `is_format` code point, and
/// the invisible fillers that are Default_Ignorable without being Cf (Hangul
/// U+115F, U+1160, U+3164, U+FFA0 and Khmer U+17B4, U+17B5). Joiners,
/// variation selectors, CGJ and the Mongolian selectors are not refused.
pub(crate) fn is_refused_format(c: char) -> bool {
is_format(c)
|| matches!(
c,
'\u{115f}' | '\u{1160}' | '\u{17b4}' | '\u{17b5}' | '\u{3164}' | '\u{ffa0}'
)
}
/// A joiner, a variation selector (U+FE00..U+FE0F, U+E0100..U+E01EF), CGJ
/// U+034F or a Mongolian free variation selector (U+180B..U+180D, U+180F): it
/// shapes a neighbour and shows nothing alone.
pub(crate) fn is_joining_only(c: char) -> bool {
is_joiner(c)
|| matches!(
c,
'\u{34f}'
| '\u{180b}'..='\u{180d}'
| '\u{180f}'
| '\u{fe00}'..='\u{fe0f}'
| '\u{e0100}'..='\u{e01ef}'
)
}
/// An attestation string (author, persona, fixture, note, work, url, generator,
/// digest): at most 200 bytes, at least one alphanumeric, no control, format or
/// joiner character. Whitespace between words is fine; it attests something.
pub(crate) fn is_attestation(s: &str) -> bool {
s.len() <= 200
&& s.chars().any(char::is_alphanumeric)
&& !s
.chars()
.any(|c| c.is_control() || is_refused_format(c) || is_joiner(c))
}
#[cfg(test)]
mod tests {
use super::*;
/// The rule against every code point: `is_format` is exactly the listed
/// Cf-minus-joiners-plus-U+2028/9 set, so each range has both of its
/// edges and the gaps between ranges pinned.
#[test]
fn format_rule_matches_the_unicode_list_at_every_code_point() {
let listed: std::collections::BTreeSet<u32> = include_str!("../tests/format_refused.txt")
.lines()
.filter(|l| !l.starts_with('#'))
.map(|l| u32::from_str_radix(l, 16).unwrap())
.collect();
for code in 0..=0x10ffff_u32 {
if let Some(c) = char::from_u32(code) {
assert_eq!(is_format(c), listed.contains(&code), "U+{code:X}");
}
}
assert!(!is_format('\u{200c}') && !is_format('\u{200d}'));
}
#[test]
fn an_attestation_is_bounded_readable_text() {
assert!(is_attestation("a"));
assert!(is_attestation(&"x".repeat(200)));
assert!(!is_attestation(&"x".repeat(201)));
assert!(is_attestation("two words, 2 more"));
for bad in [
"", " ", "\t", "\u{200b}", "\u{200d}", "\0", "--", "a\nb", "a\u{ad}b",
] {
assert!(!is_attestation(bad), "{bad:?}");
}
}
}

70
crates/weksa/src/main.rs
View file

@ -88,9 +88,12 @@ struct PutArgs {
forms: Vec<String>, forms: Vec<String>,
#[arg(long, value_enum)] #[arg(long, value_enum)]
origin: OriginKind, origin: OriginKind,
/// The persona of a `cast` origin; refused with any other origin. /// The persona of a `cast` or `live` origin; refused with any other origin.
#[arg(long)] #[arg(long)]
persona: Option<String>, persona: Option<String>,
/// The fixture path a `cast` entity came from; refused with any other origin.
#[arg(long)]
fixture: Option<String>,
/// The work of a `public-domain` origin; refused with any other origin. /// The work of a `public-domain` origin; refused with any other origin.
#[arg(long)] #[arg(long)]
work: Option<String>, work: Option<String>,
@ -107,9 +110,7 @@ impl PutArgs {
fn entity(self) -> Result<Entity> { fn entity(self) -> Result<Entity> {
let mut forms: Vec<LanguageForms> = Vec::new(); let mut forms: Vec<LanguageForms> = Vec::new();
for f in &self.forms { for f in &self.forms {
let (language, form) = f let (language, form) = f.split_once('=').context("--form is not language=form")?;
.split_once('=')
.with_context(|| format!("--form {f:?} is not language=form"))?;
match forms.iter_mut().find(|lf| lf.language == language) { match forms.iter_mut().find(|lf| lf.language == language) {
Some(lf) => lf.forms.push(form.nfc().collect()), Some(lf) => lf.forms.push(form.nfc().collect()),
None => forms.push(LanguageForms { None => forms.push(LanguageForms {
@ -118,16 +119,19 @@ impl PutArgs {
}), }),
} }
} }
let origin = match (self.origin, self.persona, self.work) { let origin = match (self.origin, self.persona, self.fixture, self.work) {
(OriginKind::Cast, persona, None) => EntityOrigin::Cast { (OriginKind::Cast, persona, fixture, None) => EntityOrigin::Cast {
persona: need(persona, "persona")?, persona: need(persona, "persona")?,
fixture: need(fixture, "fixture")?,
}, },
(OriginKind::PublicDomain, None, work) => EntityOrigin::PublicDomain { (OriginKind::PublicDomain, None, None, work) => EntityOrigin::PublicDomain {
work: need(work, "work")?, work: need(work, "work")?,
}, },
(OriginKind::Live, None, None) => EntityOrigin::Live, (OriginKind::Live, persona, None, None) => EntityOrigin::Live {
persona: need(persona, "persona")?,
},
_ => bail!( _ => bail!(
"--persona is only for --origin cast and --work only for --origin public-domain" "--fixture is only for --origin cast, --persona only for --origin cast or live and --work only for --origin public-domain"
), ),
}; };
Ok(Entity { Ok(Entity {
@ -202,6 +206,7 @@ struct AddArgs {
note: Option<String>, note: Option<String>,
#[arg(long)] #[arg(long)]
work: Option<String>, work: Option<String>,
/// Who wrote the line: for `cast-authored` and `public-domain`.
#[arg(long)] #[arg(long)]
author: Option<String>, author: Option<String>,
#[arg(long)] #[arg(long)]
@ -226,10 +231,37 @@ fn need(value: Option<String>, flag: &str) -> Result<String> {
impl AddArgs { impl AddArgs {
fn source(&self) -> Result<Source> { fn source(&self) -> Result<Source> {
let get = |v: &Option<String>, flag: &str| need(v.clone(), flag); let get = |v: &Option<String>, flag: &str| need(v.clone(), flag);
let applies: &[&str] = match self.source {
SourceKind::CastAuthored => &["persona", "fixture", "author"],
SourceKind::OperatorWritten => &[],
SourceKind::OperatorApproved => &["note"],
SourceKind::PublicDomain => &["work", "author", "edition-url"],
SourceKind::ModelWritten => &["model-family", "generator", "prompt-digest"],
};
let given = [
("persona", self.persona.is_some()),
("fixture", self.fixture.is_some()),
("author", self.author.is_some()),
("note", self.note.is_some()),
("work", self.work.is_some()),
("edition-url", self.edition_url.is_some()),
("model-family", self.family.is_some()),
("generator", self.generator.is_some()),
("prompt-digest", self.prompt_digest.is_some()),
];
let stray: Vec<String> = given
.into_iter()
.filter(|(flag, given)| *given && !applies.contains(flag))
.map(|(flag, _)| format!("--{flag}"))
.collect();
if !stray.is_empty() {
bail!("{} does not apply to this --source", stray.join(", "));
}
Ok(match self.source { Ok(match self.source {
SourceKind::CastAuthored => Source::CastAuthored { SourceKind::CastAuthored => Source::CastAuthored {
persona: get(&self.persona, "persona")?, persona: get(&self.persona, "persona")?,
fixture: get(&self.fixture, "fixture")?, fixture: get(&self.fixture, "fixture")?,
author: get(&self.author, "author")?,
}, },
SourceKind::OperatorWritten => Source::OperatorWritten, SourceKind::OperatorWritten => Source::OperatorWritten,
SourceKind::OperatorApproved => Source::OperatorApproved { SourceKind::OperatorApproved => Source::OperatorApproved {
@ -258,9 +290,11 @@ impl AddArgs {
.map(|t| { .map(|t| {
let (speaker, text) = t let (speaker, text) = t
.split_once('=') .split_once('=')
.with_context(|| format!("--prior-turn {t:?} is not speaker=text"))?; .context("--prior-turn is not speaker=text")?;
Ok(Turn { Ok(Turn {
speaker: speaker.parse()?, speaker: speaker.parse().map_err(|_| {
anyhow::anyhow!("--prior-turn speaker is not entity:<slug>")
})?,
text: text.nfc().collect(), text: text.nfc().collect(),
}) })
}) })
@ -269,13 +303,13 @@ impl AddArgs {
.names .names
.iter() .iter()
.map(|n| { .map(|n| {
let bad = || format!("--name {n:?} is not start..end=entity:<slug>"); let bad = || anyhow::anyhow!("--name is not start..end=entity:<slug>");
let (range, entity) = n.split_once('=').with_context(bad)?; let (range, entity) = n.split_once('=').ok_or_else(bad)?;
let (start, end) = range.split_once("..").with_context(bad)?; let (start, end) = range.split_once("..").ok_or_else(bad)?;
Ok(NameSpan { Ok(NameSpan {
entity: entity.parse()?, entity: entity.parse().map_err(|_| bad())?,
start: start.parse().with_context(bad)?, start: start.parse().map_err(|_| bad())?,
end: end.parse().with_context(bad)?, end: end.parse().map_err(|_| bad())?,
}) })
}) })
.collect::<Result<Vec<_>>>()?; .collect::<Result<Vec<_>>>()?;
@ -338,7 +372,7 @@ fn main() -> Result<()> {
} }
Command::Corpus(CorpusCommand::Add(args)) => { Command::Corpus(CorpusCommand::Add(args)) => {
let mut store = CorpusStore::open(&cli.corpus)?; let mut store = CorpusStore::open(&cli.corpus)?;
let key = store.next_key(&args.slug)?; let key = store.next_key(&args.slug).map_err(refused)?;
let entities = EntityStore::open(&cli.entities)?; let entities = EntityStore::open(&cli.entities)?;
let outcome = store let outcome = store
.add((*args).item(key.clone())?, &entities) .add((*args).item(key.clone())?, &entities)

108
crates/weksa/src/store.rs
View file

@ -6,13 +6,14 @@ use std::collections::BTreeSet;
use std::fmt; use std::fmt;
use std::path::Path; use std::path::Path;
use caseless::default_case_fold_str;
use cultcache_rs::{CultCache, SingleFileMessagePackBackingStore}; use cultcache_rs::{CultCache, SingleFileMessagePackBackingStore};
use unicode_normalization::is_nfc; use unicode_normalization::{UnicodeNormalization, char::is_combining_mark, is_nfc};
use unicode_segmentation::UnicodeSegmentation; use unicode_segmentation::UnicodeSegmentation;
use crate::corpus::{CorpusItem, CorpusItemId, item_key, key_slug}; use crate::corpus::{CorpusItem, CorpusItemId, Source, item_key, key_slug};
use crate::entity::Entity; use crate::entity::{Entity, EntityOrigin};
use crate::form::is_language_tag; use crate::form::{is_attestation, is_joining_only, is_language_tag, is_refused_format};
use crate::inventory::{Change, InvId, Inventory, InventoryEntry, InventoryRevision, revision_key}; use crate::inventory::{Change, InvId, Inventory, InventoryEntry, InventoryRevision, revision_key};
use crate::packet::{EntityId, Packet}; use crate::packet::{EntityId, Packet};
use crate::validate::{Refusal, validate}; use crate::validate::{Refusal, validate};
@ -88,15 +89,12 @@ pub enum EntityBound {
Forms, Forms,
/// A form of more than 64 bytes. /// A form of more than 64 bytes.
FormBytes, FormBytes,
/// A form of more than 4 words.
FormWords,
} }
impl EntityBound { impl EntityBound {
pub const MAX_LANGUAGES: usize = 8; pub const MAX_LANGUAGES: usize = 8;
pub const MAX_FORMS: usize = 8; pub const MAX_FORMS: usize = 8;
pub const MAX_FORM_BYTES: usize = 64; pub const MAX_FORM_BYTES: usize = 64;
pub const MAX_FORM_WORDS: usize = 4;
} }
#[derive(Debug)] #[derive(Debug)]
@ -185,7 +183,7 @@ impl CorpusStore {
} }
/// The next free key for `slug`; withdrawn items keep their numbers. /// The next free key for `slug`; withdrawn items keep their numbers.
pub fn next_key(&self, slug: &str) -> anyhow::Result<String> { pub fn next_key(&self, slug: &str) -> Result<String, StoreError> {
let highest = self let highest = self
.all()? .all()?
.iter() .iter()
@ -193,7 +191,10 @@ impl CorpusStore {
.filter_map(|i| i.id.rsplit_once(':')?.1.parse::<u32>().ok()) .filter_map(|i| i.id.rsplit_once(':')?.1.parse::<u32>().ok())
.max() .max()
.unwrap_or(0); .unwrap_or(0);
Ok(item_key(slug, highest + 1)) let next = highest
.checked_add(1)
.ok_or_else(|| malformed(slug, "slug has no free key"))?;
Ok(item_key(slug, next))
} }
/// The item `id` names, or `UnknownCorpusItem` when it holds nothing. /// The item `id` names, or `UnknownCorpusItem` when it holds nothing.
@ -236,8 +237,14 @@ impl CorpusStore {
if !is_nfc(&item.text) || item.prior_turns.iter().any(|t| !is_nfc(&t.text)) { if !is_nfc(&item.text) || item.prior_turns.iter().any(|t| !is_nfc(&t.text)) {
return Err(StoreRefusal::NotNfc { key: item.id }.into()); return Err(StoreRefusal::NotNfc { key: item.id }.into());
} }
check_entities(&item, entities)?;
let text = normalized(&item.text); let text = normalized(&item.text);
if !text.chars().any(char::is_alphanumeric) {
return Err(malformed(&item.id, "text has no alphanumeric character"));
}
if let Some(why) = empty_attestation(&item.source) {
return Err(malformed(&item.id, why));
}
check_entities(&item, entities)?;
for held in self.all()? { for held in self.all()? {
if normalized(&held.text) == text && item.supersedes.as_deref() != Some(&held.id) { if normalized(&held.text) == text && item.supersedes.as_deref() != Some(&held.id) {
return Err(StoreRefusal::S6TextHeld { return Err(StoreRefusal::S6TextHeld {
@ -350,15 +357,58 @@ fn check_entities(item: &CorpusItem, entities: &EntityStore) -> Result<(), Store
Ok(()) Ok(())
} }
/// Lowercased alphanumerics only, so case, spacing and punctuation variants of /// The S6 key of a text: Unicode compatibility caseless matching (D146: NFKC,
/// a text compare equal. /// default case fold, NFKC) keeping letters, digits and combining marks
/// (General_Category L, N, M) so tone marks and viramas keep words apart. Composed and decomposed,
/// fullwidth, ligature and case-fold variants (straße, STRASSE) and spacing and
/// punctuation variants compare equal. Homoglyphs across scripts (Cyrillic a for
/// Latin a) stay apart: no key here reads what a character looks like.
fn normalized(text: &str) -> String { fn normalized(text: &str) -> String {
text.chars() let folded = default_case_fold_str(&text.nfkc().collect::<String>());
.flat_map(char::to_lowercase) folded
.filter(|c| c.is_alphanumeric()) .nfkc()
.filter(|c| c.is_alphanumeric() || is_combining_mark(*c))
.collect() .collect()
} }
/// The field that holds an unattested provenance string, if any: a string with
/// no alphanumeric, a control or format character, or over 200 bytes attests
/// nothing.
fn empty_attestation(source: &Source) -> Option<&'static str> {
let empty = |s: &String| !is_attestation(s);
let empty_some = |s: &Option<String>| s.as_ref().is_some_and(empty);
match source {
Source::CastAuthored { persona, .. } if empty(persona) => {
Some("a cast-authored source names its persona")
}
Source::CastAuthored { fixture, .. } if empty(fixture) => {
Some("a cast-authored source names its fixture")
}
Source::CastAuthored { author, .. } if empty(author) => {
Some("a cast-authored source names its author")
}
Source::OperatorApproved { note } if empty(note) => {
Some("an operator-approved source carries its note")
}
Source::PublicDomain { work, .. } if empty(work) => {
Some("a public-domain source names its work")
}
Source::PublicDomain { author, .. } if empty(author) => {
Some("a public-domain source names its author")
}
Source::PublicDomain { edition_url, .. } if empty(edition_url) => {
Some("a public-domain source names its edition url")
}
Source::ModelWritten { generator, .. } if empty_some(generator) => {
Some("a model-written source names a non-empty generator")
}
Source::ModelWritten { prompt_digest, .. } if empty_some(prompt_digest) => {
Some("a model-written source carries a non-empty prompt digest")
}
_ => None,
}
}
pub struct EntityStore { pub struct EntityStore {
cache: CultCache, cache: CultCache,
} }
@ -384,8 +434,9 @@ impl EntityStore {
/// every stored language and form in order and origin as it was, and only /// every stored language and form in order and origin as it was, and only
/// appends: forms grow, never shrink, reorder or change. An entity has 1 to /// appends: forms grow, never shrink, reorder or change. An entity has 1 to
/// 8 languages, each a BCP 47-form tag named once, with 1 to 8 NFC forms of /// 8 languages, each a BCP 47-form tag named once, with 1 to 8 NFC forms of
/// 1 to 64 bytes and at most 4 words, no control character, no edge /// 1 to 64 bytes, no control or invisible format character (joiners and
/// whitespace, none repeated. /// variation selectors stay), no edge whitespace, none repeated; the origin
/// attests where it came from with strings that hold an alphanumeric, no control or format character and at most 200 bytes.
pub fn put(&mut self, entity: Entity) -> Result<AddOutcome, StoreError> { pub fn put(&mut self, entity: Entity) -> Result<AddOutcome, StoreError> {
let key = entity.id.to_string(); let key = entity.id.to_string();
check_entity_shape(&entity)?; check_entity_shape(&entity)?;
@ -418,6 +469,20 @@ fn check_entity_shape(entity: &Entity) -> Result<(), StoreError> {
} }
.into() .into()
}; };
let (persona, fixture, work) = match &entity.origin {
EntityOrigin::Cast { persona, fixture } => (Some(persona), Some(fixture), None),
EntityOrigin::Live { persona } => (Some(persona), None, None),
EntityOrigin::PublicDomain { work } => (None, None, Some(work)),
};
if persona.is_some_and(|s| !is_attestation(s)) {
return Err(bad("an origin names its persona"));
}
if fixture.is_some_and(|s| !is_attestation(s)) {
return Err(bad("a cast origin names its fixture"));
}
if work.is_some_and(|s| !is_attestation(s)) {
return Err(bad("a public-domain origin names its work"));
}
if entity.forms.is_empty() { if entity.forms.is_empty() {
return Err(bad("an entity has forms in at least one language")); return Err(bad("an entity has forms in at least one language"));
} }
@ -443,9 +508,6 @@ fn check_entity_shape(entity: &Entity) -> Result<(), StoreError> {
if f.len() > EntityBound::MAX_FORM_BYTES { if f.len() > EntityBound::MAX_FORM_BYTES {
return Err(bound(EntityBound::FormBytes)); return Err(bound(EntityBound::FormBytes));
} }
if f.split_whitespace().count() > EntityBound::MAX_FORM_WORDS {
return Err(bound(EntityBound::FormWords));
}
if !is_nfc(f) { if !is_nfc(f) {
return Err(StoreRefusal::NotNfc { return Err(StoreRefusal::NotNfc {
key: entity.id.to_string(), key: entity.id.to_string(),
@ -457,6 +519,12 @@ fn check_entity_shape(entity: &Entity) -> Result<(), StoreError> {
"a form is not empty and has no control character or edge whitespace", "a form is not empty and has no control character or edge whitespace",
)); ));
} }
if f.chars().any(is_refused_format) {
return Err(bad("a form holds an invisible format character"));
}
if f.chars().all(is_joining_only) {
return Err(bad("a form is more than joiners and selectors"));
}
if !seen.insert(f) { if !seen.insert(f) {
return Err(bad("a form is not repeated within its language")); return Err(bad("a form is not repeated within its language"));
} }

295
crates/weksa/tests/cli.rs
View file

@ -1,6 +1,6 @@
mod common; mod common;
use std::process::{Command, Output}; use std::process::{Command, Output};
use weksa::{InventoryStore, PacketStore}; use weksa::{CorpusStore, EntityStore, InventoryStore, PacketStore};
fn weksa(dir: &std::path::Path, args: &[&str]) -> Output { fn weksa(dir: &std::path::Path, args: &[&str]) -> Output {
Command::new(env!("CARGO_BIN_EXE_weksa")) Command::new(env!("CARGO_BIN_EXE_weksa"))
@ -35,6 +35,8 @@ fn cast_entities(dir: &std::path::Path, slugs: &[&str]) {
"cast", "cast",
"--persona", "--persona",
slug, slug,
"--fixture",
"f.yaml",
], ],
); );
assert!(put.status.success(), "{}", out(&put)); assert!(put.status.success(), "{}", out(&put));
@ -81,7 +83,7 @@ fn seed_add_dump_split_withdraw() {
"{dump}" "{dump}"
); );
// A cast-authored item needs its persona and fixture. // A cast-authored item needs its persona, fixture and author.
let missing = weksa(d.path(), &ADD); let missing = weksa(d.path(), &ADD);
assert!( assert!(
!missing.status.success() && out(&missing).contains("--persona"), !missing.status.success() && out(&missing).contains("--persona"),
@ -89,7 +91,14 @@ fn seed_add_dump_split_withdraw() {
out(&missing) out(&missing)
); );
let mut add = ADD.to_vec(); let mut add = ADD.to_vec();
add.extend(["--persona", "nibu", "--fixture", "f.yaml"]); add.extend([
"--persona",
"nibu",
"--fixture",
"f.yaml",
"--author",
"Ada Writer",
]);
let added = weksa(d.path(), &add); let added = weksa(d.path(), &add);
assert!( assert!(
added.status.success() && out(&added).contains("Added item:nibu:1"), added.status.success() && out(&added).contains("Added item:nibu:1"),
@ -294,7 +303,15 @@ fn schema_writes_the_published_files() {
fn entity_put_and_name_spans() { fn entity_put_and_name_spans() {
let d = tempfile::tempdir().unwrap(); let d = tempfile::tempdir().unwrap();
let put = |args: &[&str]| { let put = |args: &[&str]| {
let mut all = vec!["entity", "put", "entity:kay", "--origin", "cast"]; let mut all = vec![
"entity",
"put",
"entity:kay",
"--origin",
"cast",
"--fixture",
"f.yaml",
];
all.extend_from_slice(args); all.extend_from_slice(args);
weksa(d.path(), &all) weksa(d.path(), &all)
}; };
@ -386,24 +403,59 @@ fn entity_origin_flags_must_match_origin() {
weksa(d.path(), &all) weksa(d.path(), &all)
}; };
for mismatched in [ for mismatched in [
&["--origin", "live", "--persona", "kay"][..], &["--origin", "live"][..],
&["--origin", "live", "--work", "edda"], &["--origin", "live", "--persona", "kay", "--work", "edda"],
&["--origin", "cast", "--persona", "kay", "--work", "edda"],
&[ &[
"--origin", "--origin",
"public-domain", "live",
"--persona", "--persona",
"kay", "kay",
"--fixture",
"f.yaml",
],
&["--origin", "cast", "--persona", "kay"],
&["--origin", "cast", "--fixture", "f.yaml"],
&[
"--origin",
"cast",
"--persona",
"kay",
"--fixture",
"f.yaml",
"--work", "--work",
"edda", "edda",
], ],
&["--origin", "public-domain", "--persona", "kay"], &["--origin", "public-domain"],
&[
"--origin",
"public-domain",
"--work",
"edda",
"--persona",
"kay",
],
&[
"--origin",
"public-domain",
"--work",
"edda",
"--fixture",
"f.yaml",
],
&["--origin", "cast", "--persona", "", "--fixture", "f.yaml"],
&["--origin", "cast", "--persona", "kay", "--fixture", ""],
&["--origin", "live", "--persona", ""],
&["--origin", "public-domain", "--work", ""],
] { ] {
let o = put(mismatched); let o = put(mismatched);
assert!(!o.status.success(), "{mismatched:?}: {}", out(&o)); assert!(!o.status.success(), "{mismatched:?}: {}", out(&o));
} }
assert!(!out(&weksa(d.path(), &["entity", "dump"])).contains("entity:kay")); assert!(!out(&weksa(d.path(), &["entity", "dump"])).contains("entity:kay"));
assert!(put(&["--origin", "live"]).status.success()); assert!(
put(&["--origin", "live", "--persona", "kay"])
.status
.success()
);
let work = weksa( let work = weksa(
d.path(), d.path(),
&[ &[
@ -436,6 +488,8 @@ fn intake_normalizes_to_nfc() {
"fr=Rene\u{301}", "fr=Rene\u{301}",
"--origin", "--origin",
"live", "live",
"--persona",
"rene",
], ],
); );
assert!(put.status.success(), "{}", out(&put)); assert!(put.status.success(), "{}", out(&put));
@ -467,3 +521,224 @@ fn intake_normalizes_to_nfc() {
let dump = out(&weksa(d.path(), &["corpus", "dump"])); let dump = out(&weksa(d.path(), &["corpus", "dump"]));
assert!(dump.contains("Ren\u{e9} part."), "{dump}"); assert!(dump.contains("Ren\u{e9} part."), "{dump}");
} }
/// `corpus add` of "Mind the hatch." by `entity:nibu` under `source`, with
/// `extra` flags.
fn add_with(dir: &std::path::Path, source: &str, extra: &[&str]) -> Output {
let mut args = vec![
"corpus",
"add",
"--slug",
"nibu",
"--text",
"Mind the hatch.",
"--speaker",
"entity:nibu",
"--group",
"scene-1",
"--admitted-on",
"2026-10-29",
"--source",
source,
];
args.extend_from_slice(extra);
weksa(dir, &args)
}
#[test]
fn source_flags_must_match_the_source() {
let d = tempfile::tempdir().unwrap();
cast_entities(d.path(), &["nibu"]);
// A provenance flag that does not apply to --source refuses, naming the flag.
for (source, extra, flag) in [
("operator-written", &["--persona", "p"][..], "--persona"),
(
"operator-approved",
&["--note", "n", "--work", "w"],
"--work",
),
(
"public-domain",
&[
"--work",
"w",
"--author",
"a",
"--edition-url",
"u",
"--note",
"n",
],
"--note",
),
(
"cast-authored",
&[
"--persona",
"p",
"--fixture",
"f",
"--author",
"a",
"--generator",
"g",
],
"--generator",
),
(
"model-written",
&["--model-family", "unrecorded", "--edition-url", "u"],
"--edition-url",
),
(
"operator-written",
&["--model-family", "unrecorded"],
"--model-family",
),
(
"model-written",
&[
"--model-family",
"unrecorded",
"--prompt-digest",
"d",
"--fixture",
"f",
],
"--fixture",
),
// A source missing a flag it needs.
(
"cast-authored",
&["--persona", "p", "--fixture", "f"],
"--author",
),
(
"public-domain",
&["--work", "w", "--edition-url", "u"],
"--author",
),
// Empty provenance strings.
(
"model-written",
&["--model-family", "unrecorded", "--generator", ""],
"",
),
(
"model-written",
&["--model-family", "unrecorded", "--prompt-digest", ""],
"",
),
(
"cast-authored",
&["--persona", "", "--fixture", "f", "--author", "a"],
"",
),
("operator-approved", &["--note", ""], ""),
] {
let o = add_with(d.path(), source, extra);
assert!(!o.status.success(), "{source} {extra:?}: {}", out(&o));
assert!(out(&o).contains(flag), "{source} {extra:?}: {}", out(&o));
}
assert!(!out(&weksa(d.path(), &["corpus", "dump"])).contains("item:"));
// A source whose flags all apply is admitted; --author reaches a
// cast-authored item.
let cast = add_with(
d.path(),
"cast-authored",
&["--persona", "p", "--fixture", "f", "--author", "Ada Writer"],
);
assert!(cast.status.success(), "{}", out(&cast));
let dump = out(&weksa(d.path(), &["corpus", "dump"]));
assert!(dump.contains("author: \"Ada Writer\""), "{dump}");
}
#[test]
fn malformed_intake_never_echoes_its_values() {
const CANARY: &str = "CANARY-7f3a";
let d = tempfile::tempdir().unwrap();
cast_entities(d.path(), &["nibu"]);
let shape = |value: &str| format!("{CANARY}{value}");
let puts = [
(vec![shape("")], "--form"),
(vec![shape("_x=Kay")], ""),
(vec![format!("en-US={CANARY}\u{7}")], ""),
(vec![format!("{CANARY}_x=Kay")], ""),
];
for (forms, flag) in puts {
let mut args = vec![
"entity",
"put",
"entity:kay",
"--origin",
"live",
"--persona",
"kay",
];
for f in &forms {
args.extend(["--form", f]);
}
let o = weksa(d.path(), &args);
assert!(!o.status.success(), "{forms:?}: {}", out(&o));
assert!(!out(&o).contains(CANARY), "{forms:?}: {}", out(&o));
assert!(out(&o).contains(flag), "{forms:?}: {}", out(&o));
}
let bad_turn = [shape(""), format!("{CANARY}=hello")];
let bad_name = [
shape(""),
format!("0..1={CANARY}"),
format!("{CANARY}..1=entity:nibu"),
format!("0..{CANARY}=entity:nibu"),
];
let cases = bad_turn
.iter()
.map(|t| ("--prior-turn", t))
.chain(bad_name.iter().map(|n| ("--name", n)));
for (flag, value) in cases {
let o = add_with(d.path(), "operator-written", &[flag, value]);
assert!(!o.status.success(), "{flag} {value}: {}", out(&o));
assert!(!out(&o).contains(CANARY), "{flag} {value}: {}", out(&o));
assert!(out(&o).contains(flag), "{flag} {value}: {}", out(&o));
}
}
#[test]
fn every_decomposed_form_and_turn_is_stored_composed() {
let d = tempfile::tempdir().unwrap();
cast_entities(d.path(), &["nibu"]);
let put = weksa(
d.path(),
&[
"entity",
"put",
"entity:rene",
"--form",
"fr=Rene\u{301}",
"--form",
"fr=Zoe\u{308}",
"--origin",
"live",
"--persona",
"rene",
],
);
assert!(put.status.success(), "{}", out(&put));
let rene = EntityStore::open(&d.path().join("e.cc"))
.unwrap()
.get(&"entity:rene".parse().unwrap())
.unwrap()
.unwrap();
assert_eq!(rene.forms[0].forms, ["Ren\u{e9}", "Zo\u{eb}"]);
let add = add_with(
d.path(),
"operator-written",
&["--prior-turn", "entity:rene=Cafe\u{301} now"],
);
assert!(add.status.success(), "{}", out(&add));
let item = CorpusStore::open(&d.path().join("c.cc"))
.unwrap()
.get("item:nibu:1")
.unwrap()
.unwrap();
assert_eq!(item.prior_turns[0].text, "Caf\u{e9} now");
}

2
crates/weksa/tests/common/mod.rs
View file

@ -63,6 +63,7 @@ pub fn cast() -> Source {
Source::CastAuthored { Source::CastAuthored {
persona: "nibu".into(), persona: "nibu".into(),
fixture: "f.yaml".into(), fixture: "f.yaml".into(),
author: "Ada Writer".into(),
} }
} }
@ -91,6 +92,7 @@ pub fn ent(slug: &str, language: &str, forms: &[&str]) -> Entity {
}], }],
origin: EntityOrigin::Cast { origin: EntityOrigin::Cast {
persona: slug.into(), persona: slug.into(),
fixture: "f.yaml".into(),
}, },
} }
} }

6
crates/weksa/tests/entities.rs
View file

@ -105,7 +105,11 @@ fn entity_forms_only_grow() {
}, },
|e| e.forms.reverse(), |e| e.forms.reverse(),
|e| e.forms[1].language = "fi".into(), |e| e.forms[1].language = "fi".into(),
|e| e.origin = EntityOrigin::Live, |e| {
e.origin = EntityOrigin::Live {
persona: "kay".into(),
}
},
]; ];
assert_eq!( assert_eq!(
refusal(store.put(ent("kay", "en-US", &["Kay"]))), refusal(store.put(ent("kay", "en-US", &["Kay"]))),

330
crates/weksa/tests/entity_door.rs
View file

@ -163,28 +163,33 @@ fn entity_capacity_is_bounded() {
let make = |forms: Vec<LanguageForms>| Entity { let make = |forms: Vec<LanguageForms>| Entity {
id: eid("big"), id: eid("big"),
forms, forms,
origin: EntityOrigin::Live, origin: EntityOrigin::Live {
persona: "p".into(),
},
}; };
// A whole line is not a name: 7 words. // A form is bounded by bytes, not words: a spaced name of any word count and
let line = "I will never leave you, not ever."; // names in scripts that write no spaces are admitted (the Thai name counts
let e = make(vec![lf("en-US", one(line))]); // five UAX #29 words, a Han name one per character).
assert_eq!( for (i, name) in [
refusal(w.entities.put(e.clone())), "I will never leave you, not ever.",
bounds(&e, EntityBound::FormWords) "Mary of the Isles too",
); "de la Cruz y Garcia",
// Four words is the edge; five is over. 64 bytes is the edge; 65 is over. "\u{e2a}\u{e21}\u{e0a}\u{e32}\u{e22} \u{e43}\u{e08}\u{e14}\u{e35}",
w.entities "\u{5c71}\u{7530}\u{592a}\u{90ce}",
.put(Entity { ]
id: eid("four"), .into_iter()
..make(vec![lf("en-US", one("Mary of the Isles"))]) .enumerate()
}) {
.unwrap(); w.entities
let five = make(vec![lf("en-US", one("Mary of the Isles too"))]); .put(Entity {
assert_eq!( id: eid(&format!("name-{i}")),
refusal(w.entities.put(five.clone())), ..make(vec![lf("en-US", one(name))])
bounds(&five, EntityBound::FormWords) })
); .unwrap();
}
// 64 bytes is the edge; 65 is over, counted in bytes (21 Han characters are
// 63 bytes, 22 are 66).
let wide = make(vec![lf("en-US", one(&"a".repeat(65)))]); let wide = make(vec![lf("en-US", one(&"a".repeat(65)))]);
assert_eq!( assert_eq!(
refusal(w.entities.put(wide.clone())), refusal(w.entities.put(wide.clone())),
@ -196,6 +201,17 @@ fn entity_capacity_is_bounded() {
..make(vec![lf("en-US", one(&"a".repeat(64)))]) ..make(vec![lf("en-US", one(&"a".repeat(64)))])
}) })
.unwrap(); .unwrap();
w.entities
.put(Entity {
id: eid("han-edge"),
..make(vec![lf("ja", one(&"\u{5c71}".repeat(21)))])
})
.unwrap();
let han = make(vec![lf("ja", one(&"\u{5c71}".repeat(22)))]);
assert_eq!(
refusal(w.entities.put(han.clone())),
bounds(&han, EntityBound::FormBytes)
);
// 8 forms per language is the edge. // 8 forms per language is the edge.
let forms = |n: usize| (0..n).map(|i| format!("form{i}")).collect::<Vec<_>>(); let forms = |n: usize| (0..n).map(|i| format!("form{i}")).collect::<Vec<_>>();
@ -321,6 +337,7 @@ fn entities_are_append_only_and_replay_needs_them() {
let mut persona = base.clone(); let mut persona = base.clone();
persona.origin = EntityOrigin::Cast { persona.origin = EntityOrigin::Cast {
persona: "someone-else".into(), persona: "someone-else".into(),
fixture: "f.yaml".into(),
}; };
assert_eq!(refusal(w.entities.put(persona)), not_append); assert_eq!(refusal(w.entities.put(persona)), not_append);
let mut duplicate = base.clone(); let mut duplicate = base.clone();
@ -346,3 +363,276 @@ fn entities_are_append_only_and_replay_needs_them() {
} }
); );
} }
/// One entity holding `form` as its only en-US form.
fn with_form(slug: &str, form: &str) -> Entity {
Entity {
id: eid(slug),
forms: vec![LanguageForms {
language: "en-US".into(),
forms: vec![form.into()],
}],
origin: EntityOrigin::Live {
persona: "p".into(),
},
}
}
#[test]
fn invisible_format_characters_are_refused_in_forms() {
let mut w = world();
let hidden = |slug: &str| StoreRefusal::Malformed {
key: format!("entity:{slug}"),
why: "a form holds an invisible format character",
};
let at = |code: u32| char::from_u32(code).unwrap();
// Every Cf code point except the joiners, plus U+2028 and U+2029, listed
// from Python's unicodedata. The code-point sweep in form.rs pins the edges
// of the rule; this checks the door applies it, in each position.
let listed: Vec<u32> = include_str!("format_refused.txt")
.lines()
.filter(|l| !l.starts_with('#'))
.map(|l| u32::from_str_radix(l, 16).unwrap())
.collect();
assert_eq!(listed.len(), 170);
for (n, code) in listed.iter().enumerate() {
let c = at(*code);
let mut forms = vec![format!("Ka{c}y")];
if !c.is_whitespace() {
forms.extend([format!("{c}Kay"), format!("Kay{c}")]);
}
for form in forms {
assert_eq!(
refusal(w.entities.put(with_form("fmt", &form))),
hidden("fmt"),
"U+{code:X} (#{n}) in {form:?}"
);
}
}
// Default_Ignorable fillers that are not Cf: Hangul and Khmer.
for code in [0x115f, 0x1160, 0x17b4, 0x17b5, 0x3164, 0xffa0] {
assert_eq!(
refusal(
w.entities
.put(with_form("fmt", &format!("Ka{}y", at(code))))
),
hidden("fmt"),
"U+{code:X}"
);
}
assert!(w.entities.get(&eid("fmt")).unwrap().is_none());
// The neighbours of the refused ranges, ZWNJ, ZWJ, CGJ, variation
// selectors and the Mongolian selectors (all Mn) are admitted in a form.
let admitted: [u32; 33] = [
0xac,
0xae,
0x34f,
0x5ff,
0x606,
0x61b,
0x61d,
0x6dc,
0x6de,
0x70e,
0x710,
0x88f,
0x892,
0x8e1,
0x8e3,
0x180b,
0x180c,
0x180d,
0x180f,
0x200a,
0x200c,
0x200d,
0x2010,
0x202f,
0x2065,
0x2070,
0xfe00,
0xfe0f,
0xfeff - 1,
0xfffc,
0x110be,
0x1d172,
0xe0100,
];
for (n, code) in admitted.into_iter().enumerate() {
let form = format!("Ka{}y", at(code));
w.entities
.put(with_form(&format!("ok-{n}"), &form))
.unwrap_or_else(|e| panic!("U+{code:X}: {e}"));
}
// ZWNJ in a Persian form, ZWJ in an emoji sequence, VS16 and a Mongolian
// free variation selector beside a letter.
for (n, form) in [
"\u{645}\u{6cc}\u{200c}\u{62e}\u{648}\u{627}\u{647}\u{645}",
"\u{1f469}\u{200d}\u{1f4bb}",
"\u{2764}\u{fe0f}",
"\u{1820}\u{180b}",
"\u{1f600}\u{e0100}",
]
.into_iter()
.enumerate()
{
w.entities
.put(with_form(&format!("script-{n}"), form))
.unwrap();
}
// A form of only joiners, variation selectors, CGJ or Mongolian selectors
// shows nothing and is refused.
let empty_looking = |slug: &str| StoreRefusal::Malformed {
key: format!("entity:{slug}"),
why: "a form is more than joiners and selectors",
};
for form in [
"\u{200d}",
"\u{200c}",
"\u{200c}\u{200d}",
"\u{fe0f}",
"\u{fe00}\u{fe0f}",
"\u{34f}",
"\u{180b}\u{180c}\u{180d}\u{180f}",
"\u{e0100}",
"\u{e01ef}\u{34f}\u{200d}",
] {
assert_eq!(
refusal(w.entities.put(with_form("blank", form))),
empty_looking("blank"),
"{form:?}"
);
}
assert!(w.entities.get(&eid("blank")).unwrap().is_none());
}
#[test]
fn an_origin_attests_where_the_entity_came_from() {
let mut w = world();
let bad = |slug: &str, why: &'static str| StoreRefusal::Malformed {
key: format!("entity:{slug}"),
why,
};
let with = |slug: &str, origin: EntityOrigin| Entity {
origin,
..with_form(slug, "Kay")
};
let cast = |persona: &str, fixture: &str| EntityOrigin::Cast {
persona: persona.into(),
fixture: fixture.into(),
};
assert_eq!(
refusal(w.entities.put(with("a", cast("", "f.yaml")))),
bad("a", "an origin names its persona")
);
assert_eq!(
refusal(w.entities.put(with("b", cast("kay", "")))),
bad("b", "a cast origin names its fixture")
);
assert_eq!(
refusal(w.entities.put(with(
"c",
EntityOrigin::Live {
persona: String::new()
}
))),
bad("c", "an origin names its persona")
);
assert_eq!(
refusal(w.entities.put(with(
"d",
EntityOrigin::PublicDomain {
work: String::new()
}
))),
bad("d", "a public-domain origin names its work")
);
for slug in ["a", "b", "c", "d"] {
assert!(w.entities.get(&eid(slug)).unwrap().is_none());
}
w.entities.put(with("e", cast("kay", "f.yaml"))).unwrap();
w.entities
.put(with(
"f",
EntityOrigin::PublicDomain {
work: "edda".into(),
},
))
.unwrap();
}
#[test]
fn an_origin_attestation_holds_something_readable() {
let mut w = world();
let with = |slug: &str, origin: EntityOrigin| Entity {
origin,
..with_form(slug, "Kay")
};
type Build = fn(String) -> EntityOrigin;
let origins: [(Build, &str); 4] = [
(
|v| EntityOrigin::Cast {
persona: v,
fixture: "f.yaml".into(),
},
"an origin names its persona",
),
(
|v| EntityOrigin::Cast {
persona: "kay".into(),
fixture: v,
},
"a cast origin names its fixture",
),
(
|v| EntityOrigin::Live { persona: v },
"an origin names its persona",
),
(
|v| EntityOrigin::PublicDomain { work: v },
"a public-domain origin names its work",
),
];
let unreadable = [
" ".to_string(),
"\t".into(),
"\n".into(),
"\u{200b}".into(),
"\u{200d}".into(),
"\0".into(),
"--".into(),
"ok\u{200b}ok".into(),
"ok\u{2028}ok".into(),
"ok\0ok".into(),
"x".repeat(201),
"x".repeat(2 * 1024 * 1024),
];
let mut n = 0;
for (build, why) in &origins {
for bad in &unreadable {
n += 1;
let slug = format!("o-{n}");
assert_eq!(
refusal(w.entities.put(with(&slug, build(bad.clone())))),
StoreRefusal::Malformed {
key: format!("entity:{slug}"),
why
},
"{why}: {:?}",
bad.chars().take(12).collect::<String>()
);
assert!(w.entities.get(&eid(&slug)).unwrap().is_none());
}
}
for (m, (build, _)) in origins.iter().enumerate() {
for (k, good) in ["x".repeat(200), "two words, 2 more".to_string()]
.into_iter()
.enumerate()
{
w.entities
.put(with(&format!("g-{}", 2 * m + k), build(good)))
.unwrap();
}
}
}

171
crates/weksa/tests/format_refused.txt Normal file
View file

@ -0,0 +1,171 @@
# Cf minus ZWNJ/ZWJ, plus U+2028 and U+2029; Python 3.13 unicodedata 15.1.0 over 0..10FFFF
AD
600
601
602
603
604
605
61C
6DD
70F
890
891
8E2
180E
200B
200E
200F
2028
2029
202A
202B
202C
202D
202E
2060
2061
2062
2063
2064
2066
2067
2068
2069
206A
206B
206C
206D
206E
206F
FEFF
FFF9
FFFA
FFFB
110BD
110CD
13430
13431
13432
13433
13434
13435
13436
13437
13438
13439
1343A
1343B
1343C
1343D
1343E
1343F
1BCA0
1BCA1
1BCA2
1BCA3
1D173
1D174
1D175
1D176
1D177
1D178
1D179
1D17A
E0001
E0020
E0021
E0022
E0023
E0024
E0025
E0026
E0027
E0028
E0029
E002A
E002B
E002C
E002D
E002E
E002F
E0030
E0031
E0032
E0033
E0034
E0035
E0036
E0037
E0038
E0039
E003A
E003B
E003C
E003D
E003E
E003F
E0040
E0041
E0042
E0043
E0044
E0045
E0046
E0047
E0048
E0049
E004A
E004B
E004C
E004D
E004E
E004F
E0050
E0051
E0052
E0053
E0054
E0055
E0056
E0057
E0058
E0059
E005A
E005B
E005C
E005D
E005E
E005F
E0060
E0061
E0062
E0063
E0064
E0065
E0066
E0067
E0068
E0069
E006A
E006B
E006C
E006D
E006E
E006F
E0070
E0071
E0072
E0073
E0074
E0075
E0076
E0077
E0078
E0079
E007A
E007B
E007C
E007D
E007E
E007F

1
crates/weksa/tests/split_schema.rs
View file

@ -22,6 +22,7 @@ fn cast() -> Source {
Source::CastAuthored { Source::CastAuthored {
persona: "p".into(), persona: "p".into(),
fixture: "f".into(), fixture: "f".into(),
author: "a".into(),
} }
} }

344
crates/weksa/tests/stores.rs
View file

@ -143,6 +143,17 @@ fn corpus_keys_are_canonical() {
} }
// An alias of a stored key is not a corpus item id at all. // An alias of a stored key is not a corpus item id at all.
assert!(weksa::CorpusItemId::parse("item:nibu:01").is_err()); assert!(weksa::CorpusItemId::parse("item:nibu:01").is_err());
// The last free number is handed out; past u32::MAX the slug has no free
// key, and asking neither wraps nor panics.
store
.add(item("item:b:4294967294", "one before the last"), &ents(&d))
.unwrap();
assert_eq!(store.next_key("b").unwrap(), "item:b:4294967295");
let full = "a".repeat(48);
assert_eq!(
refusal(store.next_key(&full)),
malformed(&full, "slug has no free key")
);
} }
#[test] #[test]
@ -263,6 +274,322 @@ fn corpus_admission_rules() {
); );
} }
#[test]
fn s6_reads_text_under_compatibility_case_folding() {
let d = dir();
let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap();
// Each pair is one text to the key: fullwidth, ligature, superscript and
// circled-digit compatibility forms and full case folds (straße, STRASSE).
let pairs = [
("Brokkr", "\u{ff22}\u{ff52}\u{ff4f}\u{ff4b}\u{ff4b}\u{ff52}"),
("fine", "\u{fb01}ne"),
("room 2", "room \u{b2}"),
("gate 1", "gate \u{2460}"),
("stra\u{df}e", "STRASSE"),
(
"\u{3a3}\u{3af}\u{3c3}\u{3c5}\u{3c6}\u{3bf}\u{3c2}",
"\u{3c3}\u{3af}\u{3c3}\u{3c5}\u{3c6}\u{3bf}\u{3c3}",
),
// The first NFKC step: a compatibility letter folds only after it is
// mapped (math-bold S has no case fold of its own; black-letter H is
// not a case variant of h until NFKC reads it as H).
(
"sigrun",
"\u{1d412}\u{1d422}\u{1d420}\u{1d42b}\u{1d42e}\u{1d427}",
),
("h", "\u{210c}"),
// The second NFKC step: a full case fold can leave a decomposed or
// reordered sequence that only a second NFKC composes to the same key
// (dialytika tonos, caron under line, macron below).
("\u{390}", "\u{3aa}\u{301}"),
("\u{3b0}", "\u{3ab}\u{301}"),
("\u{1e96}\u{30c}", "\u{21e}\u{331}"),
];
for (n, (a, b)) in pairs.into_iter().enumerate() {
let (first, second) = (
format!("item:s6:{}", 2 * n + 1),
format!("item:s6:{}", 2 * n + 2),
);
store.add(item(&first, a), &ents(&d)).unwrap();
assert_eq!(
refusal(store.add(item(&second, b), &ents(&d))),
StoreRefusal::S6TextHeld {
key: second.clone(),
held_by: first.clone()
},
"{a:?} vs {b:?}"
);
assert!(store.get(&second).unwrap().is_none());
}
// A decomposed text never reaches the key: the door refuses it as not NFC.
assert_eq!(
refusal(store.add(item("item:s6:90", "Bro\u{308}kkr"), &ents(&d))),
StoreRefusal::NotNfc {
key: "item:s6:90".into()
}
);
// Cyrillic a is not Latin a: homoglyphs across scripts stay apart.
store.add(item("item:s6:91", "cat"), &ents(&d)).unwrap();
store
.add(item("item:s6:92", "c\u{430}t"), &ents(&d))
.unwrap();
}
#[test]
fn s6_keeps_tone_marks_and_viramas_apart() {
let d = dir();
let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap();
// Minimal pairs that differ only by a combining mark: Thai mai tho and mai
// ek on one word, Thai tone on a short vowel, a Devanagari virama.
let pairs = [
("\u{e19}\u{e49}\u{e33}", "\u{e19}\u{e33}"),
("\u{e44}\u{e21}\u{e49}", "\u{e44}\u{e21}\u{e48}"),
("\u{915}\u{94d}\u{92f}", "\u{915}\u{92f}"),
];
for (n, (a, b)) in pairs.into_iter().enumerate() {
store
.add(item(&format!("item:mk:{}", 2 * n + 1), a), &ents(&d))
.unwrap_or_else(|e| panic!("{a:?}: {e}"));
store
.add(item(&format!("item:mk:{}", 2 * n + 2), b), &ents(&d))
.unwrap_or_else(|e| panic!("{b:?} held by {a:?}: {e}"));
}
// A mark alone is still no text.
for (n, text) in ["\u{e49}", "\u{94d}"].into_iter().enumerate() {
let key = format!("item:mk:{}", 90 + n);
assert_eq!(
refusal(store.add(item(&key, text), &ents(&d))),
malformed(&key, "text has no alphanumeric character")
);
}
}
#[test]
fn an_attestation_holds_something_readable() {
let d = dir();
let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap();
let s = |x: &str| x.to_string();
type Build = fn(String) -> Source;
let builders: [(Build, &'static str); 9] = [
(
|v| Source::CastAuthored {
persona: v,
fixture: "f".into(),
author: "a".into(),
},
"a cast-authored source names its persona",
),
(
|v| Source::CastAuthored {
persona: "p".into(),
fixture: v,
author: "a".into(),
},
"a cast-authored source names its fixture",
),
(
|v| Source::CastAuthored {
persona: "p".into(),
fixture: "f".into(),
author: v,
},
"a cast-authored source names its author",
),
(
|v| Source::OperatorApproved { note: v },
"an operator-approved source carries its note",
),
(
|v| Source::PublicDomain {
work: v,
author: "a".into(),
edition_url: "u".into(),
},
"a public-domain source names its work",
),
(
|v| Source::PublicDomain {
work: "w".into(),
author: v,
edition_url: "u".into(),
},
"a public-domain source names its author",
),
(
|v| Source::PublicDomain {
work: "w".into(),
author: "a".into(),
edition_url: v,
},
"a public-domain source names its edition url",
),
(
|v| Source::ModelWritten {
family: ModelFamily::Unrecorded,
generator: Some(v),
prompt_digest: None,
},
"a model-written source names a non-empty generator",
),
(
|v| Source::ModelWritten {
family: ModelFamily::Unrecorded,
generator: None,
prompt_digest: Some(v),
},
"a model-written source carries a non-empty prompt digest",
),
];
let unreadable = [
s(" "),
s("\t"),
s("\n"),
s("\u{200b}"),
s("\u{200d}"),
s("\0"),
s("--"),
s("\u{e49}"),
s("ok\u{200b}ok"),
s("ok\u{200d}ok"),
s("ok\u{2028}ok"),
s("ok\nok"),
s("ok\0ok"),
"x".repeat(201),
"x".repeat(2 * 1024 * 1024),
];
let mut n = 0;
for (build, why) in &builders {
for bad in &unreadable {
n += 1;
let key = format!("item:att:{n}");
let refused = item_in(&key, &format!("line {n}"), "scene-1", build(bad.clone()));
assert_eq!(
refusal(store.add(refused, &ents(&d))),
malformed(&key, why),
"{why}: {:?}",
bad.chars().take(12).collect::<String>()
);
}
}
assert!(store.all().unwrap().is_empty());
// 200 bytes and plain words with spaces attest; 201 does not.
for (m, (build, _)) in builders.iter().enumerate() {
for (k, good) in ["x".repeat(200), s("two words, 2 more")]
.into_iter()
.enumerate()
{
let key = format!("item:att-ok:{}", 2 * m + k + 1);
store
.add(
item_in(&key, &format!("fine {m} {k}"), "scene-1", build(good)),
&ents(&d),
)
.unwrap_or_else(|e| panic!("{key}: {e}"));
}
}
}
#[test]
fn a_text_with_no_alphanumeric_character_is_refused() {
let d = dir();
let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap();
for (n, text) in ["...", "?!", " ", "\u{2014}\u{2014}", "\u{ff01}"]
.into_iter()
.enumerate()
{
let key = format!("item:p:{}", n + 1);
assert_eq!(
refusal(store.add(item(&key, text), &ents(&d))),
malformed(&key, "text has no alphanumeric character"),
"{text:?}"
);
}
assert!(store.all().unwrap().is_empty());
store.add(item("item:p:1", "a..."), &ents(&d)).unwrap();
}
#[test]
fn an_item_source_attests_with_non_empty_strings() {
let d = dir();
let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap();
let s = |x: &str| x.to_string();
let cast = |persona: &str, fixture: &str, author: &str| Source::CastAuthored {
persona: s(persona),
fixture: s(fixture),
author: s(author),
};
let model = |generator: Option<&str>, digest: Option<&str>| Source::ModelWritten {
family: ModelFamily::Unrecorded,
generator: generator.map(s),
prompt_digest: digest.map(s),
};
let public = |work: &str, author: &str, url: &str| Source::PublicDomain {
work: s(work),
author: s(author),
edition_url: s(url),
};
let cases: [(Source, &'static str); 9] = [
(
cast("", "f", "a"),
"a cast-authored source names its persona",
),
(
cast("p", "", "a"),
"a cast-authored source names its fixture",
),
(
cast("p", "f", ""),
"a cast-authored source names its author",
),
(
Source::OperatorApproved { note: s("") },
"an operator-approved source carries its note",
),
(
public("", "a", "u"),
"a public-domain source names its work",
),
(
public("w", "", "u"),
"a public-domain source names its author",
),
(
public("w", "a", ""),
"a public-domain source names its edition url",
),
(
model(Some(""), None),
"a model-written source names a non-empty generator",
),
(
model(None, Some("")),
"a model-written source carries a non-empty prompt digest",
),
];
for (n, (source, why)) in cases.into_iter().enumerate() {
let key = format!("item:prov:{}", n + 1);
let refused = item_in(&key, &format!("line {n}"), "scene-1", source);
assert_eq!(refusal(store.add(refused, &ents(&d))), malformed(&key, why));
}
assert!(store.all().unwrap().is_empty());
// The same sources with every string present are admitted.
for (n, source) in [
cast("p", "f", "a"),
Source::OperatorApproved { note: s("n") },
Source::OperatorWritten,
public("w", "a", "u"),
model(Some("g"), Some("d")),
model(None, None),
]
.into_iter()
.enumerate()
{
let key = format!("item:prov:{}", n + 1);
let item = item_in(&key, &format!("line {n}"), "scene-1", source);
assert_eq!(store.add(item, &ents(&d)).unwrap(), AddOutcome::Added);
}
}
#[test] #[test]
fn s5_supersede_keeps_the_group() { fn s5_supersede_keeps_the_group() {
let d = dir(); let d = dir();
@ -842,6 +1169,23 @@ fn the_committed_stores_load_and_hold_clean_data() {
"no item claims a recorded human author" "no item claims a recorded human author"
); );
assert_eq!(items.len(), 3); assert_eq!(items.len(), 3);
// Every committed entity attests, with non-empty strings, where it came
// from, and a cast entity's fixture is a file of this repository.
let entities = EntityStore::open(&root.join("data/entity/entity.cc"))
.unwrap()
.all()
.unwrap();
assert!(!entities.is_empty());
for e in &entities {
match &e.origin {
weksa::entity::EntityOrigin::Cast { persona, fixture } => {
assert!(!persona.is_empty(), "{}", e.id);
assert!(root.join(fixture).is_file(), "{}: {fixture}", e.id);
}
weksa::entity::EntityOrigin::Live { persona } => assert!(!persona.is_empty()),
weksa::entity::EntityOrigin::PublicDomain { work } => assert!(!work.is_empty()),
}
}
} }
#[test] #[test]

2
data/entity/entity.cc
View file

@ -1 +1 @@
“²cultcache.store.v1‘—¬weksa.entity¬weksa.entity¯weksa.entity.v1¬weksa.entityÙL{"schemaName":"weksa.entity","schemaVersion":"weksa.entity.v1","members":[]}‘¬weksa.entity<EFBFBD>’”­entity:brokkr¬weksa.entity´2026-10-29T14:59:18ZÄ-“­entity:brokkr‘’¥en-US‘¦Brokkr<EFBFBD>¤Cast‘¦brokkr”«entity:nibu¬weksa.entity´2026-10-29T14:59:18ZÄ'“«entity:nibu‘’¥en-US‘¤Nibu<EFBFBD>¤Cast‘¤nibu “²cultcache.store.v1‘—¬weksa.entity¬weksa.entity¯weksa.entity.v1¬weksa.entityÙL{"schemaName":"weksa.entity","schemaVersion":"weksa.entity.v1","members":[]}‘¬weksa.entity<EFBFBD>’”­entity:brokkr¬weksa.entity´2026-10-29T16:43:28ZÄn“­entity:brokkr‘’¥en-US‘¦Brokkr<EFBFBD>¤Cast’¦brokkrÙ?examples/persona-lowering/brokkr-edda-voice/lowered-output.yaml”«entity:nibu¬weksa.entity´2026-10-29T16:43:28ZÄe“«entity:nibu‘’¥en-US‘¤Nibu<EFBFBD>¤Cast’¤nibuÙ<examples/english-lowering/nibu-proof-001/lowered-output.yaml

2
docs/targets/interlingua-lowering-map.md
View file

@ -180,7 +180,7 @@ operator-attested annotations alone. Who annotates is question
carries its name spans (byte ranges of its text, each naming an entity and carries its name spans (byte ranges of its text, each naming an entity and
on extended grapheme cluster boundaries of its NFC text and equal to one of that on extended grapheme cluster boundaries of its NFC text and equal to one of that
entity's NFC forms in the item's language; an entity holds at most 8 languages entity's NFC forms in the item's language; an entity holds at most 8 languages
of BCP 47-form tags, 8 forms each, a form at most 64 bytes and 4 words), and its speaker of BCP 47-form tags, 8 forms each, a form at most 64 bytes with no invisible format character), and its speaker
and prior-turn speakers are entity ids. The packet store refuses a packet and prior-turn speakers are entity ids. The packet store refuses a packet
whose referent cites an entity the store lacks, and a packet with a whose referent cites an entity the store lacks, and a packet with a
`corpus_item` (required, typed `CorpusItemId`) that cites an entity neither annotated on that item nor one of `corpus_item` (required, typed `CorpusItemId`) that cites an entity neither annotated on that item nor one of

8
schemas/cultnet/weksa.corpus_item.schema.json
View file

@ -121,12 +121,15 @@
] ]
}, },
{ {
"description": "A line from a cast member's hand-authored fixture.", "description": "A line from a cast member's hand-authored fixture, with the human who\nwrote it. A fixture that records no author is `ModelWritten` with family\n`Unrecorded`, never this (`corpus-labels-tell-truth`).",
"type": "object", "type": "object",
"properties": { "properties": {
"CastAuthored": { "CastAuthored": {
"type": "object", "type": "object",
"properties": { "properties": {
"author": {
"type": "string"
},
"fixture": { "fixture": {
"type": "string" "type": "string"
}, },
@ -136,7 +139,8 @@
}, },
"required": [ "required": [
"persona", "persona",
"fixture" "fixture",
"author"
] ]
} }
}, },

32
schemas/cultnet/weksa.entity.schema.json
View file

@ -31,18 +31,22 @@
"description": "Where an entity was minted.", "description": "Where an entity was minted.",
"oneOf": [ "oneOf": [
{ {
"description": "From a cast member's fixture.", "description": "From a cast member's fixture: the persona and the fixture path the\nentity came from.",
"type": "object", "type": "object",
"properties": { "properties": {
"Cast": { "Cast": {
"type": "object", "type": "object",
"properties": { "properties": {
"fixture": {
"type": "string"
},
"persona": { "persona": {
"type": "string" "type": "string"
} }
}, },
"required": [ "required": [
"persona" "persona",
"fixture"
] ]
} }
}, },
@ -73,9 +77,25 @@
] ]
}, },
{ {
"description": "Minted live by a persona before it names the entity.", "description": "Minted live by the named persona before it names the entity.",
"type": "string", "type": "object",
"const": "Live" "properties": {
"Live": {
"type": "object",
"properties": {
"persona": {
"type": "string"
}
},
"required": [
"persona"
]
}
},
"additionalProperties": false,
"required": [
"Live"
]
} }
] ]
}, },
@ -84,7 +104,7 @@
"type": "object", "type": "object",
"properties": { "properties": {
"forms": { "forms": {
"description": "1 to 64 bytes each, no control character, no edge whitespace, none repeated.", "description": "1 to 64 bytes each, no control or invisible format character, no edge\nwhitespace, none repeated.",
"type": "array", "type": "array",
"items": { "items": {
"type": "string" "type": "string"