diff --git a/crates/weksa/src/corpus.rs b/crates/weksa/src/corpus.rs index 403cdb8..d67cc60 100644 --- a/crates/weksa/src/corpus.rs +++ b/crates/weksa/src/corpus.rs @@ -17,10 +17,13 @@ pub struct Turn { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub enum Source { - /// A line from a cast member's hand-authored fixture. + /// A line from a cast member's hand-authored fixture, with the human who + /// wrote it. A fixture that records no author is `ModelWritten` with family + /// `Unrecorded`, never this (`corpus-labels-tell-truth`). CastAuthored { persona: String, fixture: String, + author: String, }, OperatorWritten, OperatorApproved { diff --git a/crates/weksa/src/entity.rs b/crates/weksa/src/entity.rs index 4c51e84..33d22e1 100644 --- a/crates/weksa/src/entity.rs +++ b/crates/weksa/src/entity.rs @@ -13,19 +13,21 @@ use crate::packet::EntityId; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub struct LanguageForms { pub language: String, - /// 1 to 64 bytes each, no control character, no edge whitespace, none repeated. + /// 1 to 64 bytes each, no control or invisible format character, no edge + /// whitespace, none repeated. pub forms: Vec, } /// Where an entity was minted. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub enum EntityOrigin { - /// From a cast member's fixture. - Cast { persona: String }, + /// From a cast member's fixture: the persona and the fixture path the + /// entity came from. + Cast { persona: String, fixture: String }, /// From a public-domain work. PublicDomain { work: String }, - /// Minted live by a persona before it names the entity. - Live, + /// Minted live by the named persona before it names the entity. + Live { persona: String }, } /// Key: the entity id. Forms only grow: see `EntityStore::put`. diff --git a/crates/weksa/src/form.rs b/crates/weksa/src/form.rs index d8336fb..9de8eb8 100644 --- a/crates/weksa/src/form.rs +++ b/crates/weksa/src/form.rs @@ -86,3 +86,114 @@ pub(crate) fn is_slug(s: &str) -> bool { && b[0] != b'-' && b[b.len() - 1] != b'-' } + +/// A General_Category Cf code point, or the line and paragraph separators +/// U+2028 and U+2029, other than the joiners ZWNJ U+200C and ZWJ U+200D. The +/// ranges are Unicode 15.1 Cf; `tests/format_refused.txt` is the full list from +/// Python's `unicodedata` that the table is swept against. +pub(crate) fn is_format(c: char) -> bool { + matches!( + c, + '\u{ad}' + | '\u{600}'..='\u{605}' + | '\u{61c}' + | '\u{6dd}' + | '\u{70f}' + | '\u{890}'..='\u{891}' + | '\u{8e2}' + | '\u{180e}' + | '\u{200b}' + | '\u{200e}'..='\u{200f}' + | '\u{2028}'..='\u{202e}' + | '\u{2060}'..='\u{2064}' + | '\u{2066}'..='\u{206f}' + | '\u{feff}' + | '\u{fff9}'..='\u{fffb}' + | '\u{110bd}' + | '\u{110cd}' + | '\u{13430}'..='\u{1343f}' + | '\u{1bca0}'..='\u{1bca3}' + | '\u{1d173}'..='\u{1d17a}' + | '\u{e0001}' + | '\u{e0020}'..='\u{e007f}' + ) +} + +/// ZWNJ and ZWJ: Cf, but scripts and emoji sequences need them inside a form. +pub(crate) fn is_joiner(c: char) -> bool { + matches!(c, '\u{200c}' | '\u{200d}') +} + +/// A character a name or form may not hold: every `is_format` code point, and +/// the invisible fillers that are Default_Ignorable without being Cf (Hangul +/// U+115F, U+1160, U+3164, U+FFA0 and Khmer U+17B4, U+17B5). Joiners, +/// variation selectors, CGJ and the Mongolian selectors are not refused. +pub(crate) fn is_refused_format(c: char) -> bool { + is_format(c) + || matches!( + c, + '\u{115f}' | '\u{1160}' | '\u{17b4}' | '\u{17b5}' | '\u{3164}' | '\u{ffa0}' + ) +} + +/// A joiner, a variation selector (U+FE00..U+FE0F, U+E0100..U+E01EF), CGJ +/// U+034F or a Mongolian free variation selector (U+180B..U+180D, U+180F): it +/// shapes a neighbour and shows nothing alone. +pub(crate) fn is_joining_only(c: char) -> bool { + is_joiner(c) + || matches!( + c, + '\u{34f}' + | '\u{180b}'..='\u{180d}' + | '\u{180f}' + | '\u{fe00}'..='\u{fe0f}' + | '\u{e0100}'..='\u{e01ef}' + ) +} + +/// An attestation string (author, persona, fixture, note, work, url, generator, +/// digest): at most 200 bytes, at least one alphanumeric, no control, format or +/// joiner character. Whitespace between words is fine; it attests something. +pub(crate) fn is_attestation(s: &str) -> bool { + s.len() <= 200 + && s.chars().any(char::is_alphanumeric) + && !s + .chars() + .any(|c| c.is_control() || is_refused_format(c) || is_joiner(c)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The rule against every code point: `is_format` is exactly the listed + /// Cf-minus-joiners-plus-U+2028/9 set, so each range has both of its + /// edges and the gaps between ranges pinned. + #[test] + fn format_rule_matches_the_unicode_list_at_every_code_point() { + let listed: std::collections::BTreeSet = include_str!("../tests/format_refused.txt") + .lines() + .filter(|l| !l.starts_with('#')) + .map(|l| u32::from_str_radix(l, 16).unwrap()) + .collect(); + for code in 0..=0x10ffff_u32 { + if let Some(c) = char::from_u32(code) { + assert_eq!(is_format(c), listed.contains(&code), "U+{code:X}"); + } + } + assert!(!is_format('\u{200c}') && !is_format('\u{200d}')); + } + + #[test] + fn an_attestation_is_bounded_readable_text() { + assert!(is_attestation("a")); + assert!(is_attestation(&"x".repeat(200))); + assert!(!is_attestation(&"x".repeat(201))); + assert!(is_attestation("two words, 2 more")); + for bad in [ + "", " ", "\t", "\u{200b}", "\u{200d}", "\0", "--", "a\nb", "a\u{ad}b", + ] { + assert!(!is_attestation(bad), "{bad:?}"); + } + } +} diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 48fce8f..a559e98 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -88,9 +88,12 @@ struct PutArgs { forms: Vec, #[arg(long, value_enum)] origin: OriginKind, - /// The persona of a `cast` origin; refused with any other origin. + /// The persona of a `cast` or `live` origin; refused with any other origin. #[arg(long)] persona: Option, + /// The fixture path a `cast` entity came from; refused with any other origin. + #[arg(long)] + fixture: Option, /// The work of a `public-domain` origin; refused with any other origin. #[arg(long)] work: Option, @@ -107,9 +110,7 @@ impl PutArgs { fn entity(self) -> Result { let mut forms: Vec = Vec::new(); for f in &self.forms { - let (language, form) = f - .split_once('=') - .with_context(|| format!("--form {f:?} is not language=form"))?; + let (language, form) = f.split_once('=').context("--form is not language=form")?; match forms.iter_mut().find(|lf| lf.language == language) { Some(lf) => lf.forms.push(form.nfc().collect()), None => forms.push(LanguageForms { @@ -118,16 +119,19 @@ impl PutArgs { }), } } - let origin = match (self.origin, self.persona, self.work) { - (OriginKind::Cast, persona, None) => EntityOrigin::Cast { + let origin = match (self.origin, self.persona, self.fixture, self.work) { + (OriginKind::Cast, persona, fixture, None) => EntityOrigin::Cast { persona: need(persona, "persona")?, + fixture: need(fixture, "fixture")?, }, - (OriginKind::PublicDomain, None, work) => EntityOrigin::PublicDomain { + (OriginKind::PublicDomain, None, None, work) => EntityOrigin::PublicDomain { work: need(work, "work")?, }, - (OriginKind::Live, None, None) => EntityOrigin::Live, + (OriginKind::Live, persona, None, None) => EntityOrigin::Live { + persona: need(persona, "persona")?, + }, _ => bail!( - "--persona is only for --origin cast and --work only for --origin public-domain" + "--fixture is only for --origin cast, --persona only for --origin cast or live and --work only for --origin public-domain" ), }; Ok(Entity { @@ -202,6 +206,7 @@ struct AddArgs { note: Option, #[arg(long)] work: Option, + /// Who wrote the line: for `cast-authored` and `public-domain`. #[arg(long)] author: Option, #[arg(long)] @@ -226,10 +231,37 @@ fn need(value: Option, flag: &str) -> Result { impl AddArgs { fn source(&self) -> Result { let get = |v: &Option, flag: &str| need(v.clone(), flag); + let applies: &[&str] = match self.source { + SourceKind::CastAuthored => &["persona", "fixture", "author"], + SourceKind::OperatorWritten => &[], + SourceKind::OperatorApproved => &["note"], + SourceKind::PublicDomain => &["work", "author", "edition-url"], + SourceKind::ModelWritten => &["model-family", "generator", "prompt-digest"], + }; + let given = [ + ("persona", self.persona.is_some()), + ("fixture", self.fixture.is_some()), + ("author", self.author.is_some()), + ("note", self.note.is_some()), + ("work", self.work.is_some()), + ("edition-url", self.edition_url.is_some()), + ("model-family", self.family.is_some()), + ("generator", self.generator.is_some()), + ("prompt-digest", self.prompt_digest.is_some()), + ]; + let stray: Vec = given + .into_iter() + .filter(|(flag, given)| *given && !applies.contains(flag)) + .map(|(flag, _)| format!("--{flag}")) + .collect(); + if !stray.is_empty() { + bail!("{} does not apply to this --source", stray.join(", ")); + } Ok(match self.source { SourceKind::CastAuthored => Source::CastAuthored { persona: get(&self.persona, "persona")?, fixture: get(&self.fixture, "fixture")?, + author: get(&self.author, "author")?, }, SourceKind::OperatorWritten => Source::OperatorWritten, SourceKind::OperatorApproved => Source::OperatorApproved { @@ -258,9 +290,11 @@ impl AddArgs { .map(|t| { let (speaker, text) = t .split_once('=') - .with_context(|| format!("--prior-turn {t:?} is not speaker=text"))?; + .context("--prior-turn is not speaker=text")?; Ok(Turn { - speaker: speaker.parse()?, + speaker: speaker.parse().map_err(|_| { + anyhow::anyhow!("--prior-turn speaker is not entity:") + })?, text: text.nfc().collect(), }) }) @@ -269,13 +303,13 @@ impl AddArgs { .names .iter() .map(|n| { - let bad = || format!("--name {n:?} is not start..end=entity:"); - let (range, entity) = n.split_once('=').with_context(bad)?; - let (start, end) = range.split_once("..").with_context(bad)?; + let bad = || anyhow::anyhow!("--name is not start..end=entity:"); + let (range, entity) = n.split_once('=').ok_or_else(bad)?; + let (start, end) = range.split_once("..").ok_or_else(bad)?; Ok(NameSpan { - entity: entity.parse()?, - start: start.parse().with_context(bad)?, - end: end.parse().with_context(bad)?, + entity: entity.parse().map_err(|_| bad())?, + start: start.parse().map_err(|_| bad())?, + end: end.parse().map_err(|_| bad())?, }) }) .collect::>>()?; @@ -338,7 +372,7 @@ fn main() -> Result<()> { } Command::Corpus(CorpusCommand::Add(args)) => { let mut store = CorpusStore::open(&cli.corpus)?; - let key = store.next_key(&args.slug)?; + let key = store.next_key(&args.slug).map_err(refused)?; let entities = EntityStore::open(&cli.entities)?; let outcome = store .add((*args).item(key.clone())?, &entities) diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index 6b0c858..138ff62 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -6,13 +6,14 @@ use std::collections::BTreeSet; use std::fmt; use std::path::Path; +use caseless::default_case_fold_str; use cultcache_rs::{CultCache, SingleFileMessagePackBackingStore}; -use unicode_normalization::is_nfc; +use unicode_normalization::{UnicodeNormalization, char::is_combining_mark, is_nfc}; use unicode_segmentation::UnicodeSegmentation; -use crate::corpus::{CorpusItem, CorpusItemId, item_key, key_slug}; -use crate::entity::Entity; -use crate::form::is_language_tag; +use crate::corpus::{CorpusItem, CorpusItemId, Source, item_key, key_slug}; +use crate::entity::{Entity, EntityOrigin}; +use crate::form::{is_attestation, is_joining_only, is_language_tag, is_refused_format}; use crate::inventory::{Change, InvId, Inventory, InventoryEntry, InventoryRevision, revision_key}; use crate::packet::{EntityId, Packet}; use crate::validate::{Refusal, validate}; @@ -88,15 +89,12 @@ pub enum EntityBound { Forms, /// A form of more than 64 bytes. FormBytes, - /// A form of more than 4 words. - FormWords, } impl EntityBound { pub const MAX_LANGUAGES: usize = 8; pub const MAX_FORMS: usize = 8; pub const MAX_FORM_BYTES: usize = 64; - pub const MAX_FORM_WORDS: usize = 4; } #[derive(Debug)] @@ -185,7 +183,7 @@ impl CorpusStore { } /// The next free key for `slug`; withdrawn items keep their numbers. - pub fn next_key(&self, slug: &str) -> anyhow::Result { + pub fn next_key(&self, slug: &str) -> Result { let highest = self .all()? .iter() @@ -193,7 +191,10 @@ impl CorpusStore { .filter_map(|i| i.id.rsplit_once(':')?.1.parse::().ok()) .max() .unwrap_or(0); - Ok(item_key(slug, highest + 1)) + let next = highest + .checked_add(1) + .ok_or_else(|| malformed(slug, "slug has no free key"))?; + Ok(item_key(slug, next)) } /// The item `id` names, or `UnknownCorpusItem` when it holds nothing. @@ -236,8 +237,14 @@ impl CorpusStore { if !is_nfc(&item.text) || item.prior_turns.iter().any(|t| !is_nfc(&t.text)) { return Err(StoreRefusal::NotNfc { key: item.id }.into()); } - check_entities(&item, entities)?; let text = normalized(&item.text); + if !text.chars().any(char::is_alphanumeric) { + return Err(malformed(&item.id, "text has no alphanumeric character")); + } + if let Some(why) = empty_attestation(&item.source) { + return Err(malformed(&item.id, why)); + } + check_entities(&item, entities)?; for held in self.all()? { if normalized(&held.text) == text && item.supersedes.as_deref() != Some(&held.id) { return Err(StoreRefusal::S6TextHeld { @@ -350,15 +357,58 @@ fn check_entities(item: &CorpusItem, entities: &EntityStore) -> Result<(), Store Ok(()) } -/// Lowercased alphanumerics only, so case, spacing and punctuation variants of -/// a text compare equal. +/// The S6 key of a text: Unicode compatibility caseless matching (D146: NFKC, +/// default case fold, NFKC) keeping letters, digits and combining marks +/// (General_Category L, N, M) so tone marks and viramas keep words apart. Composed and decomposed, +/// fullwidth, ligature and case-fold variants (straße, STRASSE) and spacing and +/// punctuation variants compare equal. Homoglyphs across scripts (Cyrillic a for +/// Latin a) stay apart: no key here reads what a character looks like. fn normalized(text: &str) -> String { - text.chars() - .flat_map(char::to_lowercase) - .filter(|c| c.is_alphanumeric()) + let folded = default_case_fold_str(&text.nfkc().collect::()); + folded + .nfkc() + .filter(|c| c.is_alphanumeric() || is_combining_mark(*c)) .collect() } +/// The field that holds an unattested provenance string, if any: a string with +/// no alphanumeric, a control or format character, or over 200 bytes attests +/// nothing. +fn empty_attestation(source: &Source) -> Option<&'static str> { + let empty = |s: &String| !is_attestation(s); + let empty_some = |s: &Option| s.as_ref().is_some_and(empty); + match source { + Source::CastAuthored { persona, .. } if empty(persona) => { + Some("a cast-authored source names its persona") + } + Source::CastAuthored { fixture, .. } if empty(fixture) => { + Some("a cast-authored source names its fixture") + } + Source::CastAuthored { author, .. } if empty(author) => { + Some("a cast-authored source names its author") + } + Source::OperatorApproved { note } if empty(note) => { + Some("an operator-approved source carries its note") + } + Source::PublicDomain { work, .. } if empty(work) => { + Some("a public-domain source names its work") + } + Source::PublicDomain { author, .. } if empty(author) => { + Some("a public-domain source names its author") + } + Source::PublicDomain { edition_url, .. } if empty(edition_url) => { + Some("a public-domain source names its edition url") + } + Source::ModelWritten { generator, .. } if empty_some(generator) => { + Some("a model-written source names a non-empty generator") + } + Source::ModelWritten { prompt_digest, .. } if empty_some(prompt_digest) => { + Some("a model-written source carries a non-empty prompt digest") + } + _ => None, + } +} + pub struct EntityStore { cache: CultCache, } @@ -384,8 +434,9 @@ impl EntityStore { /// every stored language and form in order and origin as it was, and only /// appends: forms grow, never shrink, reorder or change. An entity has 1 to /// 8 languages, each a BCP 47-form tag named once, with 1 to 8 NFC forms of - /// 1 to 64 bytes and at most 4 words, no control character, no edge - /// whitespace, none repeated. + /// 1 to 64 bytes, no control or invisible format character (joiners and + /// variation selectors stay), no edge whitespace, none repeated; the origin + /// attests where it came from with strings that hold an alphanumeric, no control or format character and at most 200 bytes. pub fn put(&mut self, entity: Entity) -> Result { let key = entity.id.to_string(); check_entity_shape(&entity)?; @@ -418,6 +469,20 @@ fn check_entity_shape(entity: &Entity) -> Result<(), StoreError> { } .into() }; + let (persona, fixture, work) = match &entity.origin { + EntityOrigin::Cast { persona, fixture } => (Some(persona), Some(fixture), None), + EntityOrigin::Live { persona } => (Some(persona), None, None), + EntityOrigin::PublicDomain { work } => (None, None, Some(work)), + }; + if persona.is_some_and(|s| !is_attestation(s)) { + return Err(bad("an origin names its persona")); + } + if fixture.is_some_and(|s| !is_attestation(s)) { + return Err(bad("a cast origin names its fixture")); + } + if work.is_some_and(|s| !is_attestation(s)) { + return Err(bad("a public-domain origin names its work")); + } if entity.forms.is_empty() { return Err(bad("an entity has forms in at least one language")); } @@ -443,9 +508,6 @@ fn check_entity_shape(entity: &Entity) -> Result<(), StoreError> { if f.len() > EntityBound::MAX_FORM_BYTES { return Err(bound(EntityBound::FormBytes)); } - if f.split_whitespace().count() > EntityBound::MAX_FORM_WORDS { - return Err(bound(EntityBound::FormWords)); - } if !is_nfc(f) { return Err(StoreRefusal::NotNfc { key: entity.id.to_string(), @@ -457,6 +519,12 @@ fn check_entity_shape(entity: &Entity) -> Result<(), StoreError> { "a form is not empty and has no control character or edge whitespace", )); } + if f.chars().any(is_refused_format) { + return Err(bad("a form holds an invisible format character")); + } + if f.chars().all(is_joining_only) { + return Err(bad("a form is more than joiners and selectors")); + } if !seen.insert(f) { return Err(bad("a form is not repeated within its language")); } diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 68229fa..ab31862 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -1,6 +1,6 @@ mod common; use std::process::{Command, Output}; -use weksa::{InventoryStore, PacketStore}; +use weksa::{CorpusStore, EntityStore, InventoryStore, PacketStore}; fn weksa(dir: &std::path::Path, args: &[&str]) -> Output { Command::new(env!("CARGO_BIN_EXE_weksa")) @@ -35,6 +35,8 @@ fn cast_entities(dir: &std::path::Path, slugs: &[&str]) { "cast", "--persona", slug, + "--fixture", + "f.yaml", ], ); assert!(put.status.success(), "{}", out(&put)); @@ -81,7 +83,7 @@ fn seed_add_dump_split_withdraw() { "{dump}" ); - // A cast-authored item needs its persona and fixture. + // A cast-authored item needs its persona, fixture and author. let missing = weksa(d.path(), &ADD); assert!( !missing.status.success() && out(&missing).contains("--persona"), @@ -89,7 +91,14 @@ fn seed_add_dump_split_withdraw() { out(&missing) ); let mut add = ADD.to_vec(); - add.extend(["--persona", "nibu", "--fixture", "f.yaml"]); + add.extend([ + "--persona", + "nibu", + "--fixture", + "f.yaml", + "--author", + "Ada Writer", + ]); let added = weksa(d.path(), &add); assert!( added.status.success() && out(&added).contains("Added item:nibu:1"), @@ -294,7 +303,15 @@ fn schema_writes_the_published_files() { fn entity_put_and_name_spans() { let d = tempfile::tempdir().unwrap(); let put = |args: &[&str]| { - let mut all = vec!["entity", "put", "entity:kay", "--origin", "cast"]; + let mut all = vec![ + "entity", + "put", + "entity:kay", + "--origin", + "cast", + "--fixture", + "f.yaml", + ]; all.extend_from_slice(args); weksa(d.path(), &all) }; @@ -386,24 +403,59 @@ fn entity_origin_flags_must_match_origin() { weksa(d.path(), &all) }; for mismatched in [ - &["--origin", "live", "--persona", "kay"][..], - &["--origin", "live", "--work", "edda"], - &["--origin", "cast", "--persona", "kay", "--work", "edda"], + &["--origin", "live"][..], + &["--origin", "live", "--persona", "kay", "--work", "edda"], &[ "--origin", - "public-domain", + "live", "--persona", "kay", + "--fixture", + "f.yaml", + ], + &["--origin", "cast", "--persona", "kay"], + &["--origin", "cast", "--fixture", "f.yaml"], + &[ + "--origin", + "cast", + "--persona", + "kay", + "--fixture", + "f.yaml", "--work", "edda", ], - &["--origin", "public-domain", "--persona", "kay"], + &["--origin", "public-domain"], + &[ + "--origin", + "public-domain", + "--work", + "edda", + "--persona", + "kay", + ], + &[ + "--origin", + "public-domain", + "--work", + "edda", + "--fixture", + "f.yaml", + ], + &["--origin", "cast", "--persona", "", "--fixture", "f.yaml"], + &["--origin", "cast", "--persona", "kay", "--fixture", ""], + &["--origin", "live", "--persona", ""], + &["--origin", "public-domain", "--work", ""], ] { let o = put(mismatched); assert!(!o.status.success(), "{mismatched:?}: {}", out(&o)); } assert!(!out(&weksa(d.path(), &["entity", "dump"])).contains("entity:kay")); - assert!(put(&["--origin", "live"]).status.success()); + assert!( + put(&["--origin", "live", "--persona", "kay"]) + .status + .success() + ); let work = weksa( d.path(), &[ @@ -436,6 +488,8 @@ fn intake_normalizes_to_nfc() { "fr=Rene\u{301}", "--origin", "live", + "--persona", + "rene", ], ); assert!(put.status.success(), "{}", out(&put)); @@ -467,3 +521,224 @@ fn intake_normalizes_to_nfc() { let dump = out(&weksa(d.path(), &["corpus", "dump"])); assert!(dump.contains("Ren\u{e9} part."), "{dump}"); } + +/// `corpus add` of "Mind the hatch." by `entity:nibu` under `source`, with +/// `extra` flags. +fn add_with(dir: &std::path::Path, source: &str, extra: &[&str]) -> Output { + let mut args = vec![ + "corpus", + "add", + "--slug", + "nibu", + "--text", + "Mind the hatch.", + "--speaker", + "entity:nibu", + "--group", + "scene-1", + "--admitted-on", + "2026-10-29", + "--source", + source, + ]; + args.extend_from_slice(extra); + weksa(dir, &args) +} + +#[test] +fn source_flags_must_match_the_source() { + let d = tempfile::tempdir().unwrap(); + cast_entities(d.path(), &["nibu"]); + // A provenance flag that does not apply to --source refuses, naming the flag. + for (source, extra, flag) in [ + ("operator-written", &["--persona", "p"][..], "--persona"), + ( + "operator-approved", + &["--note", "n", "--work", "w"], + "--work", + ), + ( + "public-domain", + &[ + "--work", + "w", + "--author", + "a", + "--edition-url", + "u", + "--note", + "n", + ], + "--note", + ), + ( + "cast-authored", + &[ + "--persona", + "p", + "--fixture", + "f", + "--author", + "a", + "--generator", + "g", + ], + "--generator", + ), + ( + "model-written", + &["--model-family", "unrecorded", "--edition-url", "u"], + "--edition-url", + ), + ( + "operator-written", + &["--model-family", "unrecorded"], + "--model-family", + ), + ( + "model-written", + &[ + "--model-family", + "unrecorded", + "--prompt-digest", + "d", + "--fixture", + "f", + ], + "--fixture", + ), + // A source missing a flag it needs. + ( + "cast-authored", + &["--persona", "p", "--fixture", "f"], + "--author", + ), + ( + "public-domain", + &["--work", "w", "--edition-url", "u"], + "--author", + ), + // Empty provenance strings. + ( + "model-written", + &["--model-family", "unrecorded", "--generator", ""], + "", + ), + ( + "model-written", + &["--model-family", "unrecorded", "--prompt-digest", ""], + "", + ), + ( + "cast-authored", + &["--persona", "", "--fixture", "f", "--author", "a"], + "", + ), + ("operator-approved", &["--note", ""], ""), + ] { + let o = add_with(d.path(), source, extra); + assert!(!o.status.success(), "{source} {extra:?}: {}", out(&o)); + assert!(out(&o).contains(flag), "{source} {extra:?}: {}", out(&o)); + } + assert!(!out(&weksa(d.path(), &["corpus", "dump"])).contains("item:")); + // A source whose flags all apply is admitted; --author reaches a + // cast-authored item. + let cast = add_with( + d.path(), + "cast-authored", + &["--persona", "p", "--fixture", "f", "--author", "Ada Writer"], + ); + assert!(cast.status.success(), "{}", out(&cast)); + let dump = out(&weksa(d.path(), &["corpus", "dump"])); + assert!(dump.contains("author: \"Ada Writer\""), "{dump}"); +} + +#[test] +fn malformed_intake_never_echoes_its_values() { + const CANARY: &str = "CANARY-7f3a"; + let d = tempfile::tempdir().unwrap(); + cast_entities(d.path(), &["nibu"]); + let shape = |value: &str| format!("{CANARY}{value}"); + let puts = [ + (vec![shape("")], "--form"), + (vec![shape("_x=Kay")], ""), + (vec![format!("en-US={CANARY}\u{7}")], ""), + (vec![format!("{CANARY}_x=Kay")], ""), + ]; + for (forms, flag) in puts { + let mut args = vec![ + "entity", + "put", + "entity:kay", + "--origin", + "live", + "--persona", + "kay", + ]; + for f in &forms { + args.extend(["--form", f]); + } + let o = weksa(d.path(), &args); + assert!(!o.status.success(), "{forms:?}: {}", out(&o)); + assert!(!out(&o).contains(CANARY), "{forms:?}: {}", out(&o)); + assert!(out(&o).contains(flag), "{forms:?}: {}", out(&o)); + } + let bad_turn = [shape(""), format!("{CANARY}=hello")]; + let bad_name = [ + shape(""), + format!("0..1={CANARY}"), + format!("{CANARY}..1=entity:nibu"), + format!("0..{CANARY}=entity:nibu"), + ]; + let cases = bad_turn + .iter() + .map(|t| ("--prior-turn", t)) + .chain(bad_name.iter().map(|n| ("--name", n))); + for (flag, value) in cases { + let o = add_with(d.path(), "operator-written", &[flag, value]); + assert!(!o.status.success(), "{flag} {value}: {}", out(&o)); + assert!(!out(&o).contains(CANARY), "{flag} {value}: {}", out(&o)); + assert!(out(&o).contains(flag), "{flag} {value}: {}", out(&o)); + } +} + +#[test] +fn every_decomposed_form_and_turn_is_stored_composed() { + let d = tempfile::tempdir().unwrap(); + cast_entities(d.path(), &["nibu"]); + let put = weksa( + d.path(), + &[ + "entity", + "put", + "entity:rene", + "--form", + "fr=Rene\u{301}", + "--form", + "fr=Zoe\u{308}", + "--origin", + "live", + "--persona", + "rene", + ], + ); + assert!(put.status.success(), "{}", out(&put)); + let rene = EntityStore::open(&d.path().join("e.cc")) + .unwrap() + .get(&"entity:rene".parse().unwrap()) + .unwrap() + .unwrap(); + assert_eq!(rene.forms[0].forms, ["Ren\u{e9}", "Zo\u{eb}"]); + let add = add_with( + d.path(), + "operator-written", + &["--prior-turn", "entity:rene=Cafe\u{301} now"], + ); + assert!(add.status.success(), "{}", out(&add)); + let item = CorpusStore::open(&d.path().join("c.cc")) + .unwrap() + .get("item:nibu:1") + .unwrap() + .unwrap(); + assert_eq!(item.prior_turns[0].text, "Caf\u{e9} now"); +} diff --git a/crates/weksa/tests/common/mod.rs b/crates/weksa/tests/common/mod.rs index a26c462..13bad1f 100644 --- a/crates/weksa/tests/common/mod.rs +++ b/crates/weksa/tests/common/mod.rs @@ -63,6 +63,7 @@ pub fn cast() -> Source { Source::CastAuthored { persona: "nibu".into(), fixture: "f.yaml".into(), + author: "Ada Writer".into(), } } @@ -91,6 +92,7 @@ pub fn ent(slug: &str, language: &str, forms: &[&str]) -> Entity { }], origin: EntityOrigin::Cast { persona: slug.into(), + fixture: "f.yaml".into(), }, } } diff --git a/crates/weksa/tests/entities.rs b/crates/weksa/tests/entities.rs index 30d43e3..4bf0569 100644 --- a/crates/weksa/tests/entities.rs +++ b/crates/weksa/tests/entities.rs @@ -105,7 +105,11 @@ fn entity_forms_only_grow() { }, |e| e.forms.reverse(), |e| e.forms[1].language = "fi".into(), - |e| e.origin = EntityOrigin::Live, + |e| { + e.origin = EntityOrigin::Live { + persona: "kay".into(), + } + }, ]; assert_eq!( refusal(store.put(ent("kay", "en-US", &["Kay"]))), diff --git a/crates/weksa/tests/entity_door.rs b/crates/weksa/tests/entity_door.rs index a0b3e9e..bcd53e8 100644 --- a/crates/weksa/tests/entity_door.rs +++ b/crates/weksa/tests/entity_door.rs @@ -163,28 +163,33 @@ fn entity_capacity_is_bounded() { let make = |forms: Vec| Entity { id: eid("big"), forms, - origin: EntityOrigin::Live, + origin: EntityOrigin::Live { + persona: "p".into(), + }, }; - // A whole line is not a name: 7 words. - let line = "I will never leave you, not ever."; - let e = make(vec![lf("en-US", one(line))]); - assert_eq!( - refusal(w.entities.put(e.clone())), - bounds(&e, EntityBound::FormWords) - ); - // Four words is the edge; five is over. 64 bytes is the edge; 65 is over. - w.entities - .put(Entity { - id: eid("four"), - ..make(vec![lf("en-US", one("Mary of the Isles"))]) - }) - .unwrap(); - let five = make(vec![lf("en-US", one("Mary of the Isles too"))]); - assert_eq!( - refusal(w.entities.put(five.clone())), - bounds(&five, EntityBound::FormWords) - ); + // A form is bounded by bytes, not words: a spaced name of any word count and + // names in scripts that write no spaces are admitted (the Thai name counts + // five UAX #29 words, a Han name one per character). + for (i, name) in [ + "I will never leave you, not ever.", + "Mary of the Isles too", + "de la Cruz y Garcia", + "\u{e2a}\u{e21}\u{e0a}\u{e32}\u{e22} \u{e43}\u{e08}\u{e14}\u{e35}", + "\u{5c71}\u{7530}\u{592a}\u{90ce}", + ] + .into_iter() + .enumerate() + { + w.entities + .put(Entity { + id: eid(&format!("name-{i}")), + ..make(vec![lf("en-US", one(name))]) + }) + .unwrap(); + } + // 64 bytes is the edge; 65 is over, counted in bytes (21 Han characters are + // 63 bytes, 22 are 66). let wide = make(vec![lf("en-US", one(&"a".repeat(65)))]); assert_eq!( refusal(w.entities.put(wide.clone())), @@ -196,6 +201,17 @@ fn entity_capacity_is_bounded() { ..make(vec![lf("en-US", one(&"a".repeat(64)))]) }) .unwrap(); + w.entities + .put(Entity { + id: eid("han-edge"), + ..make(vec![lf("ja", one(&"\u{5c71}".repeat(21)))]) + }) + .unwrap(); + let han = make(vec![lf("ja", one(&"\u{5c71}".repeat(22)))]); + assert_eq!( + refusal(w.entities.put(han.clone())), + bounds(&han, EntityBound::FormBytes) + ); // 8 forms per language is the edge. let forms = |n: usize| (0..n).map(|i| format!("form{i}")).collect::>(); @@ -321,6 +337,7 @@ fn entities_are_append_only_and_replay_needs_them() { let mut persona = base.clone(); persona.origin = EntityOrigin::Cast { persona: "someone-else".into(), + fixture: "f.yaml".into(), }; assert_eq!(refusal(w.entities.put(persona)), not_append); let mut duplicate = base.clone(); @@ -346,3 +363,276 @@ fn entities_are_append_only_and_replay_needs_them() { } ); } + +/// One entity holding `form` as its only en-US form. +fn with_form(slug: &str, form: &str) -> Entity { + Entity { + id: eid(slug), + forms: vec![LanguageForms { + language: "en-US".into(), + forms: vec![form.into()], + }], + origin: EntityOrigin::Live { + persona: "p".into(), + }, + } +} + +#[test] +fn invisible_format_characters_are_refused_in_forms() { + let mut w = world(); + let hidden = |slug: &str| StoreRefusal::Malformed { + key: format!("entity:{slug}"), + why: "a form holds an invisible format character", + }; + let at = |code: u32| char::from_u32(code).unwrap(); + // Every Cf code point except the joiners, plus U+2028 and U+2029, listed + // from Python's unicodedata. The code-point sweep in form.rs pins the edges + // of the rule; this checks the door applies it, in each position. + let listed: Vec = include_str!("format_refused.txt") + .lines() + .filter(|l| !l.starts_with('#')) + .map(|l| u32::from_str_radix(l, 16).unwrap()) + .collect(); + assert_eq!(listed.len(), 170); + for (n, code) in listed.iter().enumerate() { + let c = at(*code); + let mut forms = vec![format!("Ka{c}y")]; + if !c.is_whitespace() { + forms.extend([format!("{c}Kay"), format!("Kay{c}")]); + } + for form in forms { + assert_eq!( + refusal(w.entities.put(with_form("fmt", &form))), + hidden("fmt"), + "U+{code:X} (#{n}) in {form:?}" + ); + } + } + // Default_Ignorable fillers that are not Cf: Hangul and Khmer. + for code in [0x115f, 0x1160, 0x17b4, 0x17b5, 0x3164, 0xffa0] { + assert_eq!( + refusal( + w.entities + .put(with_form("fmt", &format!("Ka{}y", at(code)))) + ), + hidden("fmt"), + "U+{code:X}" + ); + } + assert!(w.entities.get(&eid("fmt")).unwrap().is_none()); + + // The neighbours of the refused ranges, ZWNJ, ZWJ, CGJ, variation + // selectors and the Mongolian selectors (all Mn) are admitted in a form. + let admitted: [u32; 33] = [ + 0xac, + 0xae, + 0x34f, + 0x5ff, + 0x606, + 0x61b, + 0x61d, + 0x6dc, + 0x6de, + 0x70e, + 0x710, + 0x88f, + 0x892, + 0x8e1, + 0x8e3, + 0x180b, + 0x180c, + 0x180d, + 0x180f, + 0x200a, + 0x200c, + 0x200d, + 0x2010, + 0x202f, + 0x2065, + 0x2070, + 0xfe00, + 0xfe0f, + 0xfeff - 1, + 0xfffc, + 0x110be, + 0x1d172, + 0xe0100, + ]; + for (n, code) in admitted.into_iter().enumerate() { + let form = format!("Ka{}y", at(code)); + w.entities + .put(with_form(&format!("ok-{n}"), &form)) + .unwrap_or_else(|e| panic!("U+{code:X}: {e}")); + } + // ZWNJ in a Persian form, ZWJ in an emoji sequence, VS16 and a Mongolian + // free variation selector beside a letter. + for (n, form) in [ + "\u{645}\u{6cc}\u{200c}\u{62e}\u{648}\u{627}\u{647}\u{645}", + "\u{1f469}\u{200d}\u{1f4bb}", + "\u{2764}\u{fe0f}", + "\u{1820}\u{180b}", + "\u{1f600}\u{e0100}", + ] + .into_iter() + .enumerate() + { + w.entities + .put(with_form(&format!("script-{n}"), form)) + .unwrap(); + } + // A form of only joiners, variation selectors, CGJ or Mongolian selectors + // shows nothing and is refused. + let empty_looking = |slug: &str| StoreRefusal::Malformed { + key: format!("entity:{slug}"), + why: "a form is more than joiners and selectors", + }; + for form in [ + "\u{200d}", + "\u{200c}", + "\u{200c}\u{200d}", + "\u{fe0f}", + "\u{fe00}\u{fe0f}", + "\u{34f}", + "\u{180b}\u{180c}\u{180d}\u{180f}", + "\u{e0100}", + "\u{e01ef}\u{34f}\u{200d}", + ] { + assert_eq!( + refusal(w.entities.put(with_form("blank", form))), + empty_looking("blank"), + "{form:?}" + ); + } + assert!(w.entities.get(&eid("blank")).unwrap().is_none()); +} + +#[test] +fn an_origin_attests_where_the_entity_came_from() { + let mut w = world(); + let bad = |slug: &str, why: &'static str| StoreRefusal::Malformed { + key: format!("entity:{slug}"), + why, + }; + let with = |slug: &str, origin: EntityOrigin| Entity { + origin, + ..with_form(slug, "Kay") + }; + let cast = |persona: &str, fixture: &str| EntityOrigin::Cast { + persona: persona.into(), + fixture: fixture.into(), + }; + assert_eq!( + refusal(w.entities.put(with("a", cast("", "f.yaml")))), + bad("a", "an origin names its persona") + ); + assert_eq!( + refusal(w.entities.put(with("b", cast("kay", "")))), + bad("b", "a cast origin names its fixture") + ); + assert_eq!( + refusal(w.entities.put(with( + "c", + EntityOrigin::Live { + persona: String::new() + } + ))), + bad("c", "an origin names its persona") + ); + assert_eq!( + refusal(w.entities.put(with( + "d", + EntityOrigin::PublicDomain { + work: String::new() + } + ))), + bad("d", "a public-domain origin names its work") + ); + for slug in ["a", "b", "c", "d"] { + assert!(w.entities.get(&eid(slug)).unwrap().is_none()); + } + w.entities.put(with("e", cast("kay", "f.yaml"))).unwrap(); + w.entities + .put(with( + "f", + EntityOrigin::PublicDomain { + work: "edda".into(), + }, + )) + .unwrap(); +} + +#[test] +fn an_origin_attestation_holds_something_readable() { + let mut w = world(); + let with = |slug: &str, origin: EntityOrigin| Entity { + origin, + ..with_form(slug, "Kay") + }; + type Build = fn(String) -> EntityOrigin; + let origins: [(Build, &str); 4] = [ + ( + |v| EntityOrigin::Cast { + persona: v, + fixture: "f.yaml".into(), + }, + "an origin names its persona", + ), + ( + |v| EntityOrigin::Cast { + persona: "kay".into(), + fixture: v, + }, + "a cast origin names its fixture", + ), + ( + |v| EntityOrigin::Live { persona: v }, + "an origin names its persona", + ), + ( + |v| EntityOrigin::PublicDomain { work: v }, + "a public-domain origin names its work", + ), + ]; + let unreadable = [ + " ".to_string(), + "\t".into(), + "\n".into(), + "\u{200b}".into(), + "\u{200d}".into(), + "\0".into(), + "--".into(), + "ok\u{200b}ok".into(), + "ok\u{2028}ok".into(), + "ok\0ok".into(), + "x".repeat(201), + "x".repeat(2 * 1024 * 1024), + ]; + let mut n = 0; + for (build, why) in &origins { + for bad in &unreadable { + n += 1; + let slug = format!("o-{n}"); + assert_eq!( + refusal(w.entities.put(with(&slug, build(bad.clone())))), + StoreRefusal::Malformed { + key: format!("entity:{slug}"), + why + }, + "{why}: {:?}", + bad.chars().take(12).collect::() + ); + assert!(w.entities.get(&eid(&slug)).unwrap().is_none()); + } + } + for (m, (build, _)) in origins.iter().enumerate() { + for (k, good) in ["x".repeat(200), "two words, 2 more".to_string()] + .into_iter() + .enumerate() + { + w.entities + .put(with(&format!("g-{}", 2 * m + k), build(good))) + .unwrap(); + } + } +} diff --git a/crates/weksa/tests/format_refused.txt b/crates/weksa/tests/format_refused.txt new file mode 100644 index 0000000..b994f95 --- /dev/null +++ b/crates/weksa/tests/format_refused.txt @@ -0,0 +1,171 @@ +# Cf minus ZWNJ/ZWJ, plus U+2028 and U+2029; Python 3.13 unicodedata 15.1.0 over 0..10FFFF +AD +600 +601 +602 +603 +604 +605 +61C +6DD +70F +890 +891 +8E2 +180E +200B +200E +200F +2028 +2029 +202A +202B +202C +202D +202E +2060 +2061 +2062 +2063 +2064 +2066 +2067 +2068 +2069 +206A +206B +206C +206D +206E +206F +FEFF +FFF9 +FFFA +FFFB +110BD +110CD +13430 +13431 +13432 +13433 +13434 +13435 +13436 +13437 +13438 +13439 +1343A +1343B +1343C +1343D +1343E +1343F +1BCA0 +1BCA1 +1BCA2 +1BCA3 +1D173 +1D174 +1D175 +1D176 +1D177 +1D178 +1D179 +1D17A +E0001 +E0020 +E0021 +E0022 +E0023 +E0024 +E0025 +E0026 +E0027 +E0028 +E0029 +E002A +E002B +E002C +E002D +E002E +E002F +E0030 +E0031 +E0032 +E0033 +E0034 +E0035 +E0036 +E0037 +E0038 +E0039 +E003A +E003B +E003C +E003D +E003E +E003F +E0040 +E0041 +E0042 +E0043 +E0044 +E0045 +E0046 +E0047 +E0048 +E0049 +E004A +E004B +E004C +E004D +E004E +E004F +E0050 +E0051 +E0052 +E0053 +E0054 +E0055 +E0056 +E0057 +E0058 +E0059 +E005A +E005B +E005C +E005D +E005E +E005F +E0060 +E0061 +E0062 +E0063 +E0064 +E0065 +E0066 +E0067 +E0068 +E0069 +E006A +E006B +E006C +E006D +E006E +E006F +E0070 +E0071 +E0072 +E0073 +E0074 +E0075 +E0076 +E0077 +E0078 +E0079 +E007A +E007B +E007C +E007D +E007E +E007F diff --git a/crates/weksa/tests/split_schema.rs b/crates/weksa/tests/split_schema.rs index d69c5cb..d1e638a 100644 --- a/crates/weksa/tests/split_schema.rs +++ b/crates/weksa/tests/split_schema.rs @@ -22,6 +22,7 @@ fn cast() -> Source { Source::CastAuthored { persona: "p".into(), fixture: "f".into(), + author: "a".into(), } } diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs index 2c6ab7c..e22b498 100644 --- a/crates/weksa/tests/stores.rs +++ b/crates/weksa/tests/stores.rs @@ -143,6 +143,17 @@ fn corpus_keys_are_canonical() { } // An alias of a stored key is not a corpus item id at all. assert!(weksa::CorpusItemId::parse("item:nibu:01").is_err()); + // The last free number is handed out; past u32::MAX the slug has no free + // key, and asking neither wraps nor panics. + store + .add(item("item:b:4294967294", "one before the last"), &ents(&d)) + .unwrap(); + assert_eq!(store.next_key("b").unwrap(), "item:b:4294967295"); + let full = "a".repeat(48); + assert_eq!( + refusal(store.next_key(&full)), + malformed(&full, "slug has no free key") + ); } #[test] @@ -263,6 +274,322 @@ fn corpus_admission_rules() { ); } +#[test] +fn s6_reads_text_under_compatibility_case_folding() { + let d = dir(); + let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + // Each pair is one text to the key: fullwidth, ligature, superscript and + // circled-digit compatibility forms and full case folds (straße, STRASSE). + let pairs = [ + ("Brokkr", "\u{ff22}\u{ff52}\u{ff4f}\u{ff4b}\u{ff4b}\u{ff52}"), + ("fine", "\u{fb01}ne"), + ("room 2", "room \u{b2}"), + ("gate 1", "gate \u{2460}"), + ("stra\u{df}e", "STRASSE"), + ( + "\u{3a3}\u{3af}\u{3c3}\u{3c5}\u{3c6}\u{3bf}\u{3c2}", + "\u{3c3}\u{3af}\u{3c3}\u{3c5}\u{3c6}\u{3bf}\u{3c3}", + ), + // The first NFKC step: a compatibility letter folds only after it is + // mapped (math-bold S has no case fold of its own; black-letter H is + // not a case variant of h until NFKC reads it as H). + ( + "sigrun", + "\u{1d412}\u{1d422}\u{1d420}\u{1d42b}\u{1d42e}\u{1d427}", + ), + ("h", "\u{210c}"), + // The second NFKC step: a full case fold can leave a decomposed or + // reordered sequence that only a second NFKC composes to the same key + // (dialytika tonos, caron under line, macron below). + ("\u{390}", "\u{3aa}\u{301}"), + ("\u{3b0}", "\u{3ab}\u{301}"), + ("\u{1e96}\u{30c}", "\u{21e}\u{331}"), + ]; + for (n, (a, b)) in pairs.into_iter().enumerate() { + let (first, second) = ( + format!("item:s6:{}", 2 * n + 1), + format!("item:s6:{}", 2 * n + 2), + ); + store.add(item(&first, a), &ents(&d)).unwrap(); + assert_eq!( + refusal(store.add(item(&second, b), &ents(&d))), + StoreRefusal::S6TextHeld { + key: second.clone(), + held_by: first.clone() + }, + "{a:?} vs {b:?}" + ); + assert!(store.get(&second).unwrap().is_none()); + } + // A decomposed text never reaches the key: the door refuses it as not NFC. + assert_eq!( + refusal(store.add(item("item:s6:90", "Bro\u{308}kkr"), &ents(&d))), + StoreRefusal::NotNfc { + key: "item:s6:90".into() + } + ); + // Cyrillic a is not Latin a: homoglyphs across scripts stay apart. + store.add(item("item:s6:91", "cat"), &ents(&d)).unwrap(); + store + .add(item("item:s6:92", "c\u{430}t"), &ents(&d)) + .unwrap(); +} + +#[test] +fn s6_keeps_tone_marks_and_viramas_apart() { + let d = dir(); + let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + // Minimal pairs that differ only by a combining mark: Thai mai tho and mai + // ek on one word, Thai tone on a short vowel, a Devanagari virama. + let pairs = [ + ("\u{e19}\u{e49}\u{e33}", "\u{e19}\u{e33}"), + ("\u{e44}\u{e21}\u{e49}", "\u{e44}\u{e21}\u{e48}"), + ("\u{915}\u{94d}\u{92f}", "\u{915}\u{92f}"), + ]; + for (n, (a, b)) in pairs.into_iter().enumerate() { + store + .add(item(&format!("item:mk:{}", 2 * n + 1), a), &ents(&d)) + .unwrap_or_else(|e| panic!("{a:?}: {e}")); + store + .add(item(&format!("item:mk:{}", 2 * n + 2), b), &ents(&d)) + .unwrap_or_else(|e| panic!("{b:?} held by {a:?}: {e}")); + } + // A mark alone is still no text. + for (n, text) in ["\u{e49}", "\u{94d}"].into_iter().enumerate() { + let key = format!("item:mk:{}", 90 + n); + assert_eq!( + refusal(store.add(item(&key, text), &ents(&d))), + malformed(&key, "text has no alphanumeric character") + ); + } +} + +#[test] +fn an_attestation_holds_something_readable() { + let d = dir(); + let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + let s = |x: &str| x.to_string(); + type Build = fn(String) -> Source; + let builders: [(Build, &'static str); 9] = [ + ( + |v| Source::CastAuthored { + persona: v, + fixture: "f".into(), + author: "a".into(), + }, + "a cast-authored source names its persona", + ), + ( + |v| Source::CastAuthored { + persona: "p".into(), + fixture: v, + author: "a".into(), + }, + "a cast-authored source names its fixture", + ), + ( + |v| Source::CastAuthored { + persona: "p".into(), + fixture: "f".into(), + author: v, + }, + "a cast-authored source names its author", + ), + ( + |v| Source::OperatorApproved { note: v }, + "an operator-approved source carries its note", + ), + ( + |v| Source::PublicDomain { + work: v, + author: "a".into(), + edition_url: "u".into(), + }, + "a public-domain source names its work", + ), + ( + |v| Source::PublicDomain { + work: "w".into(), + author: v, + edition_url: "u".into(), + }, + "a public-domain source names its author", + ), + ( + |v| Source::PublicDomain { + work: "w".into(), + author: "a".into(), + edition_url: v, + }, + "a public-domain source names its edition url", + ), + ( + |v| Source::ModelWritten { + family: ModelFamily::Unrecorded, + generator: Some(v), + prompt_digest: None, + }, + "a model-written source names a non-empty generator", + ), + ( + |v| Source::ModelWritten { + family: ModelFamily::Unrecorded, + generator: None, + prompt_digest: Some(v), + }, + "a model-written source carries a non-empty prompt digest", + ), + ]; + let unreadable = [ + s(" "), + s("\t"), + s("\n"), + s("\u{200b}"), + s("\u{200d}"), + s("\0"), + s("--"), + s("\u{e49}"), + s("ok\u{200b}ok"), + s("ok\u{200d}ok"), + s("ok\u{2028}ok"), + s("ok\nok"), + s("ok\0ok"), + "x".repeat(201), + "x".repeat(2 * 1024 * 1024), + ]; + let mut n = 0; + for (build, why) in &builders { + for bad in &unreadable { + n += 1; + let key = format!("item:att:{n}"); + let refused = item_in(&key, &format!("line {n}"), "scene-1", build(bad.clone())); + assert_eq!( + refusal(store.add(refused, &ents(&d))), + malformed(&key, why), + "{why}: {:?}", + bad.chars().take(12).collect::() + ); + } + } + assert!(store.all().unwrap().is_empty()); + // 200 bytes and plain words with spaces attest; 201 does not. + for (m, (build, _)) in builders.iter().enumerate() { + for (k, good) in ["x".repeat(200), s("two words, 2 more")] + .into_iter() + .enumerate() + { + let key = format!("item:att-ok:{}", 2 * m + k + 1); + store + .add( + item_in(&key, &format!("fine {m} {k}"), "scene-1", build(good)), + &ents(&d), + ) + .unwrap_or_else(|e| panic!("{key}: {e}")); + } + } +} + +#[test] +fn a_text_with_no_alphanumeric_character_is_refused() { + let d = dir(); + let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + for (n, text) in ["...", "?!", " ", "\u{2014}\u{2014}", "\u{ff01}"] + .into_iter() + .enumerate() + { + let key = format!("item:p:{}", n + 1); + assert_eq!( + refusal(store.add(item(&key, text), &ents(&d))), + malformed(&key, "text has no alphanumeric character"), + "{text:?}" + ); + } + assert!(store.all().unwrap().is_empty()); + store.add(item("item:p:1", "a..."), &ents(&d)).unwrap(); +} + +#[test] +fn an_item_source_attests_with_non_empty_strings() { + let d = dir(); + let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + let s = |x: &str| x.to_string(); + let cast = |persona: &str, fixture: &str, author: &str| Source::CastAuthored { + persona: s(persona), + fixture: s(fixture), + author: s(author), + }; + let model = |generator: Option<&str>, digest: Option<&str>| Source::ModelWritten { + family: ModelFamily::Unrecorded, + generator: generator.map(s), + prompt_digest: digest.map(s), + }; + let public = |work: &str, author: &str, url: &str| Source::PublicDomain { + work: s(work), + author: s(author), + edition_url: s(url), + }; + let cases: [(Source, &'static str); 9] = [ + ( + cast("", "f", "a"), + "a cast-authored source names its persona", + ), + ( + cast("p", "", "a"), + "a cast-authored source names its fixture", + ), + ( + cast("p", "f", ""), + "a cast-authored source names its author", + ), + ( + Source::OperatorApproved { note: s("") }, + "an operator-approved source carries its note", + ), + ( + public("", "a", "u"), + "a public-domain source names its work", + ), + ( + public("w", "", "u"), + "a public-domain source names its author", + ), + ( + public("w", "a", ""), + "a public-domain source names its edition url", + ), + ( + model(Some(""), None), + "a model-written source names a non-empty generator", + ), + ( + model(None, Some("")), + "a model-written source carries a non-empty prompt digest", + ), + ]; + for (n, (source, why)) in cases.into_iter().enumerate() { + let key = format!("item:prov:{}", n + 1); + let refused = item_in(&key, &format!("line {n}"), "scene-1", source); + assert_eq!(refusal(store.add(refused, &ents(&d))), malformed(&key, why)); + } + assert!(store.all().unwrap().is_empty()); + // The same sources with every string present are admitted. + for (n, source) in [ + cast("p", "f", "a"), + Source::OperatorApproved { note: s("n") }, + Source::OperatorWritten, + public("w", "a", "u"), + model(Some("g"), Some("d")), + model(None, None), + ] + .into_iter() + .enumerate() + { + let key = format!("item:prov:{}", n + 1); + let item = item_in(&key, &format!("line {n}"), "scene-1", source); + assert_eq!(store.add(item, &ents(&d)).unwrap(), AddOutcome::Added); + } +} + #[test] fn s5_supersede_keeps_the_group() { let d = dir(); @@ -842,6 +1169,23 @@ fn the_committed_stores_load_and_hold_clean_data() { "no item claims a recorded human author" ); assert_eq!(items.len(), 3); + // Every committed entity attests, with non-empty strings, where it came + // from, and a cast entity's fixture is a file of this repository. + let entities = EntityStore::open(&root.join("data/entity/entity.cc")) + .unwrap() + .all() + .unwrap(); + assert!(!entities.is_empty()); + for e in &entities { + match &e.origin { + weksa::entity::EntityOrigin::Cast { persona, fixture } => { + assert!(!persona.is_empty(), "{}", e.id); + assert!(root.join(fixture).is_file(), "{}: {fixture}", e.id); + } + weksa::entity::EntityOrigin::Live { persona } => assert!(!persona.is_empty()), + weksa::entity::EntityOrigin::PublicDomain { work } => assert!(!work.is_empty()), + } + } } #[test] diff --git a/data/entity/entity.cc b/data/entity/entity.cc index f561201..8f9a198 100644 --- a/data/entity/entity.cc +++ b/data/entity/entity.cc @@ -1 +1 @@ -cultcache.store.v1weksa.entityweksa.entityweksa.entity.v1weksa.entityL{"schemaName":"weksa.entity","schemaVersion":"weksa.entity.v1","members":[]}weksa.entityentity:brokkrweksa.entity2026-10-29T14:59:18Z-entity:brokkren-USBrokkrCastbrokkrentity:nibuweksa.entity2026-10-29T14:59:18Z'entity:nibuen-USNibuCastnibu \ No newline at end of file +cultcache.store.v1weksa.entityweksa.entityweksa.entity.v1weksa.entityL{"schemaName":"weksa.entity","schemaVersion":"weksa.entity.v1","members":[]}weksa.entityentity:brokkrweksa.entity2026-10-29T16:43:28Znentity:brokkren-USBrokkrCastbrokkr?examples/persona-lowering/brokkr-edda-voice/lowered-output.yamlentity:nibuweksa.entity2026-10-29T16:43:28Zeentity:nibuen-USNibuCastnibu