entity-store (2c): referents cite entity ids; typed entity store with bounded NFC forms per language; corpus name spans on grapheme boundaries; required CorpusItemId; packet schema v3 (squash of weksa-lowering/entity-store 29d671e..ea2c96c; Soul s1, s2)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
29d671e338
commit
20865cac59
29 changed files with 1999 additions and 340 deletions
23
Cargo.lock
generated
23
Cargo.lock
generated
|
|
@ -659,12 +659,33 @@ dependencies = [
|
|||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec"
|
||||
version = "1.13.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-normalization"
|
||||
version = "0.1.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "75cd3fb03d5288b3ba9a1ee103ac7e9cbb59f0b03d64f8ddfcc04dd42758a85d"
|
||||
dependencies = [
|
||||
"tinyvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-segmentation"
|
||||
version = "1.13.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
|
||||
|
||||
[[package]]
|
||||
name = "utf8parse"
|
||||
version = "0.2.2"
|
||||
|
|
@ -738,6 +759,8 @@ dependencies = [
|
|||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"unicode-normalization",
|
||||
"unicode-segmentation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
|
|||
|
|
@ -18,11 +18,11 @@ measured.
|
|||
|
||||
- [Interlingua Standard](./docs/interlingua-standard.md): the packet contract.
|
||||
- `crates/weksa`: the packet, inventory and corpus types, `validate`, the
|
||||
derived split and the `weksa` CLI (`weksa validate`, `corpus`, `inventory`,
|
||||
derived split and the `weksa` CLI (`weksa validate`, `corpus`, `entity`, `inventory`,
|
||||
`split`, `schema`); published schemas in `schemas/cultnet/`.
|
||||
- `data/corpus/corpus.cc` and `data/interlingua/inventory.cc`: the English
|
||||
corpus and the inventory, read with `weksa corpus dump` and
|
||||
`weksa inventory dump`.
|
||||
- `data/corpus/corpus.cc`, `data/entity/entity.cc` and `data/interlingua/inventory.cc`: the English
|
||||
corpus, its entities and the inventory, read with `weksa corpus dump`,
|
||||
`weksa entity dump` and `weksa inventory dump`.
|
||||
- [Lowering target](./docs/targets/interlingua-lowering.md) and
|
||||
[its map](./docs/targets/interlingua-lowering-map.md): the design, scoring
|
||||
and cut sequence.
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@ cultcache-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "64b86e0
|
|||
schemars = "1"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
unicode-normalization = "0.1"
|
||||
unicode-segmentation = "1"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
|
|
|
|||
|
|
@ -6,9 +6,12 @@ use cultcache_rs::DatabaseEntry;
|
|||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::form::{FormError, is_slug, string_form};
|
||||
use crate::packet::EntityId;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Turn {
|
||||
pub speaker: String,
|
||||
pub speaker: EntityId,
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
|
|
@ -62,7 +65,7 @@ pub struct CorpusItem {
|
|||
#[cultcache(key = 2)]
|
||||
pub language: String,
|
||||
#[cultcache(key = 3)]
|
||||
pub speaker: String,
|
||||
pub speaker: EntityId,
|
||||
/// The unit the split hashes: a scene, a fixture, a work.
|
||||
#[cultcache(key = 4)]
|
||||
pub group: String,
|
||||
|
|
@ -77,8 +80,48 @@ pub struct CorpusItem {
|
|||
pub supersedes: Option<String>,
|
||||
#[cultcache(key = 9)]
|
||||
pub withdrawn: bool,
|
||||
/// Which entities the text names, and where. Fixed with the text at intake.
|
||||
#[cultcache(key = 10)]
|
||||
pub names: Vec<NameSpan>,
|
||||
}
|
||||
|
||||
/// A byte range `[start, end)` of an item's text that names an entity. The
|
||||
/// store admits it only when both ends sit on extended grapheme cluster
|
||||
/// boundaries (UAX #29) of the NFC text and the range's bytes equal one of the
|
||||
/// entity's forms in the item's language.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct NameSpan {
|
||||
pub entity: EntityId,
|
||||
pub start: u32,
|
||||
pub end: u32,
|
||||
}
|
||||
|
||||
/// The key of a corpus item, `item:<slug>:<n>` as `key_slug` reads it. A packet
|
||||
/// names the item it was minted for by this type, so the key cannot be free text.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
#[serde(try_from = "String", into = "String")]
|
||||
pub struct CorpusItemId(String);
|
||||
|
||||
impl CorpusItemId {
|
||||
pub fn parse(s: &str) -> Result<Self, FormError> {
|
||||
key_slug(s)
|
||||
.map(|_| Self(s.to_string()))
|
||||
.ok_or_else(|| FormError::new("CorpusItemId", s))
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for CorpusItemId {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
string_form!(CorpusItemId, { "type": "string", "pattern": "^item:[a-z0-9]([a-z0-9-]{0,46}[a-z0-9])?:[1-9][0-9]*$" });
|
||||
|
||||
/// `item:<slug>:<n>`.
|
||||
pub fn item_key(slug: &str, n: u32) -> String {
|
||||
format!("item:{slug}:{n}")
|
||||
|
|
@ -92,11 +135,5 @@ pub fn key_slug(key: &str) -> Option<&str> {
|
|||
let (slug, n) = rest.rsplit_once(':')?;
|
||||
let canonical_n =
|
||||
n.bytes().all(|b| b.is_ascii_digit()) && !n.starts_with('0') && n.parse::<u32>().is_ok();
|
||||
let b = slug.as_bytes();
|
||||
let slug_ok = (1..=48).contains(&b.len())
|
||||
&& b.iter()
|
||||
.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || *c == b'-')
|
||||
&& b[0] != b'-'
|
||||
&& b[b.len() - 1] != b'-';
|
||||
(canonical_n && slug_ok).then_some(slug)
|
||||
(canonical_n && is_slug(slug)).then_some(slug)
|
||||
}
|
||||
|
|
|
|||
51
crates/weksa/src/entity.rs
Normal file
51
crates/weksa/src/entity.rs
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
//! The entity store's documents. An entity is a named thing a packet may cite by
|
||||
//! id. Its forms per language are data, citation form first, then inflected or
|
||||
//! particle-bound forms; no rule here reads what a form means, only its shape.
|
||||
|
||||
use cultcache_rs::DatabaseEntry;
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::packet::EntityId;
|
||||
|
||||
/// An entity's forms in one language, as the corpus item language tag spells it.
|
||||
/// The citation form comes first.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct LanguageForms {
|
||||
pub language: String,
|
||||
/// 1 to 64 bytes each, no control character, no edge whitespace, none repeated.
|
||||
pub forms: Vec<String>,
|
||||
}
|
||||
|
||||
/// Where an entity was minted.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub enum EntityOrigin {
|
||||
/// From a cast member's fixture.
|
||||
Cast { persona: String },
|
||||
/// From a public-domain work.
|
||||
PublicDomain { work: String },
|
||||
/// Minted live by a persona before it names the entity.
|
||||
Live,
|
||||
}
|
||||
|
||||
/// Key: the entity id. Forms only grow: see `EntityStore::put`.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, JsonSchema, DatabaseEntry)]
|
||||
#[cultcache(type = "weksa.entity")]
|
||||
pub struct Entity {
|
||||
#[cultcache(key = 0)]
|
||||
pub id: EntityId,
|
||||
#[cultcache(key = 1)]
|
||||
pub forms: Vec<LanguageForms>,
|
||||
#[cultcache(key = 2)]
|
||||
pub origin: EntityOrigin,
|
||||
}
|
||||
|
||||
impl Entity {
|
||||
/// The forms stored for `language`, if any.
|
||||
pub fn forms_in(&self, language: &str) -> Option<&[String]> {
|
||||
self.forms
|
||||
.iter()
|
||||
.find(|f| f.language == language)
|
||||
.map(|f| f.forms.as_slice())
|
||||
}
|
||||
}
|
||||
|
|
@ -60,3 +60,29 @@ macro_rules! string_form {
|
|||
};
|
||||
}
|
||||
pub(crate) use string_form;
|
||||
|
||||
/// A BCP 47-form language tag: 1 to 35 bytes of ASCII alphanumerics in
|
||||
/// hyphen-separated subtags of 1 to 8 bytes; the first is 2 to 8 letters, or the
|
||||
/// single letter `x` of a private-use tag. Form only: no registry is consulted.
|
||||
pub(crate) fn is_language_tag(s: &str) -> bool {
|
||||
if s.len() > 35 {
|
||||
return false;
|
||||
}
|
||||
let mut subtags = s.split('-');
|
||||
let first = subtags.next().unwrap_or("");
|
||||
let first_ok = first.bytes().all(|b| b.is_ascii_alphabetic())
|
||||
&& ((2..=8).contains(&first.len()) || first == "x");
|
||||
first_ok
|
||||
&& subtags
|
||||
.all(|t| (1..=8).contains(&t.len()) && t.bytes().all(|b| b.is_ascii_alphanumeric()))
|
||||
}
|
||||
|
||||
/// A slug: `[a-z0-9]` and inner `-`, 1 to 48 bytes, no edge `-`.
|
||||
pub(crate) fn is_slug(s: &str) -> bool {
|
||||
let b = s.as_bytes();
|
||||
(1..=48).contains(&b.len())
|
||||
&& b.iter()
|
||||
.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || *c == b'-')
|
||||
&& b[0] != b'-'
|
||||
&& b[b.len() - 1] != b'-'
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
//! train/validation/test split, and the stores that admit them.
|
||||
|
||||
pub mod corpus;
|
||||
pub mod entity;
|
||||
pub mod form;
|
||||
pub mod inventory;
|
||||
pub mod packet;
|
||||
|
|
@ -11,15 +12,19 @@ pub mod split;
|
|||
pub mod store;
|
||||
pub mod validate;
|
||||
|
||||
pub use corpus::{CorpusItem, Source, Turn};
|
||||
pub use corpus::{CorpusItem, CorpusItemId, NameSpan, Source, Turn};
|
||||
pub use entity::{Entity, EntityOrigin, LanguageForms};
|
||||
pub use form::FormError;
|
||||
pub use inventory::{
|
||||
Change, Family, InvId, Inventory, InventoryEntry, InventoryRevision, OptionLabel, OptionSet,
|
||||
Slot, ValueKind,
|
||||
};
|
||||
pub use packet::{
|
||||
EncoderId, Grade, NodeId, Packet, PacketId, PacketKind, ProperName, SCHEMA_VERSION,
|
||||
EncoderId, EntityId, Grade, NodeId, Packet, PacketId, PacketKind, SCHEMA_VERSION,
|
||||
};
|
||||
pub use split::{Split, SplitScheme};
|
||||
pub use store::{AddOutcome, CorpusStore, InventoryStore, PacketStore, StoreError, StoreRefusal};
|
||||
pub use store::{
|
||||
AddOutcome, CorpusStore, EntityBound, EntityStore, InventoryStore, PacketStore, StoreError,
|
||||
StoreRefusal,
|
||||
};
|
||||
pub use validate::{LoweringInput, Refusal, validate};
|
||||
|
|
|
|||
|
|
@ -5,9 +5,14 @@ use std::path::PathBuf;
|
|||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use clap::{Args, Parser, Subcommand, ValueEnum};
|
||||
use weksa::corpus::{CorpusItem, ModelFamily, Source, Turn};
|
||||
use unicode_normalization::UnicodeNormalization;
|
||||
use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn};
|
||||
use weksa::entity::{Entity, EntityOrigin, LanguageForms};
|
||||
use weksa::inventory::seed;
|
||||
use weksa::{CorpusStore, InventoryStore, PacketStore, SplitScheme, StoreError, validate};
|
||||
use weksa::{
|
||||
CorpusStore, EntityId, EntityStore, InventoryStore, PacketStore, SplitScheme, StoreError,
|
||||
validate,
|
||||
};
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(
|
||||
|
|
@ -21,6 +26,9 @@ struct Cli {
|
|||
/// The inventory store.
|
||||
#[arg(long, global = true, default_value = "data/interlingua/inventory.cc")]
|
||||
inventory: PathBuf,
|
||||
/// The entity store.
|
||||
#[arg(long, global = true, default_value = "data/entity/entity.cc")]
|
||||
entities: PathBuf,
|
||||
#[command(subcommand)]
|
||||
command: Command,
|
||||
}
|
||||
|
|
@ -33,6 +41,8 @@ enum Command {
|
|||
Corpus(CorpusCommand),
|
||||
#[command(subcommand)]
|
||||
Inventory(InventoryCommand),
|
||||
#[command(subcommand)]
|
||||
Entity(EntityCommand),
|
||||
/// Print the split of a corpus item.
|
||||
Split { item_key: String },
|
||||
/// Write the published JSON Schemas.
|
||||
|
|
@ -60,6 +70,74 @@ enum InventoryCommand {
|
|||
Seed,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum EntityCommand {
|
||||
/// Admit an entity, or append forms to a stored one.
|
||||
Put(PutArgs),
|
||||
/// Text projection of the entity store for review; not a store.
|
||||
Dump,
|
||||
}
|
||||
|
||||
#[derive(Args)]
|
||||
struct PutArgs {
|
||||
/// `entity:<slug>`.
|
||||
id: String,
|
||||
/// `language=form`, repeated; a language's forms keep the order given,
|
||||
/// citation form first. Forms are NFC-normalized here.
|
||||
#[arg(long = "form", required = true)]
|
||||
forms: Vec<String>,
|
||||
#[arg(long, value_enum)]
|
||||
origin: OriginKind,
|
||||
/// The persona of a `cast` origin; refused with any other origin.
|
||||
#[arg(long)]
|
||||
persona: Option<String>,
|
||||
/// The work of a `public-domain` origin; refused with any other origin.
|
||||
#[arg(long)]
|
||||
work: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, ValueEnum)]
|
||||
enum OriginKind {
|
||||
Cast,
|
||||
PublicDomain,
|
||||
Live,
|
||||
}
|
||||
|
||||
impl PutArgs {
|
||||
fn entity(self) -> Result<Entity> {
|
||||
let mut forms: Vec<LanguageForms> = Vec::new();
|
||||
for f in &self.forms {
|
||||
let (language, form) = f
|
||||
.split_once('=')
|
||||
.with_context(|| format!("--form {f:?} is not language=form"))?;
|
||||
match forms.iter_mut().find(|lf| lf.language == language) {
|
||||
Some(lf) => lf.forms.push(form.nfc().collect()),
|
||||
None => forms.push(LanguageForms {
|
||||
language: language.to_string(),
|
||||
forms: vec![form.nfc().collect()],
|
||||
}),
|
||||
}
|
||||
}
|
||||
let origin = match (self.origin, self.persona, self.work) {
|
||||
(OriginKind::Cast, persona, None) => EntityOrigin::Cast {
|
||||
persona: need(persona, "persona")?,
|
||||
},
|
||||
(OriginKind::PublicDomain, None, work) => EntityOrigin::PublicDomain {
|
||||
work: need(work, "work")?,
|
||||
},
|
||||
(OriginKind::Live, None, None) => EntityOrigin::Live,
|
||||
_ => bail!(
|
||||
"--persona is only for --origin cast and --work only for --origin public-domain"
|
||||
),
|
||||
};
|
||||
Ok(Entity {
|
||||
id: self.id.parse()?,
|
||||
forms,
|
||||
origin,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, ValueEnum)]
|
||||
enum SourceKind {
|
||||
CastAuthored,
|
||||
|
|
@ -98,12 +176,17 @@ impl From<ModelFamilyArg> for ModelFamily {
|
|||
struct AddArgs {
|
||||
#[arg(long)]
|
||||
slug: String,
|
||||
/// NFC-normalized here; `--name` byte offsets index the normalized text.
|
||||
#[arg(long)]
|
||||
text: String,
|
||||
#[arg(long, default_value = "en-US")]
|
||||
language: String,
|
||||
/// An entity id.
|
||||
#[arg(long)]
|
||||
speaker: String,
|
||||
speaker: EntityId,
|
||||
/// `start..end=entity:<slug>`, repeated: a byte range of the text naming an entity.
|
||||
#[arg(long = "name")]
|
||||
names: Vec<String>,
|
||||
#[arg(long)]
|
||||
group: String,
|
||||
/// `speaker=text`, repeated, oldest first.
|
||||
|
|
@ -177,15 +260,29 @@ impl AddArgs {
|
|||
.split_once('=')
|
||||
.with_context(|| format!("--prior-turn {t:?} is not speaker=text"))?;
|
||||
Ok(Turn {
|
||||
speaker: speaker.to_string(),
|
||||
text: text.to_string(),
|
||||
speaker: speaker.parse()?,
|
||||
text: text.nfc().collect(),
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
let names = self
|
||||
.names
|
||||
.iter()
|
||||
.map(|n| {
|
||||
let bad = || format!("--name {n:?} is not start..end=entity:<slug>");
|
||||
let (range, entity) = n.split_once('=').with_context(bad)?;
|
||||
let (start, end) = range.split_once("..").with_context(bad)?;
|
||||
Ok(NameSpan {
|
||||
entity: entity.parse()?,
|
||||
start: start.parse().with_context(bad)?,
|
||||
end: end.parse().with_context(bad)?,
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
Ok(CorpusItem {
|
||||
id,
|
||||
source: self.source()?,
|
||||
text: self.text,
|
||||
text: self.text.nfc().collect(),
|
||||
language: self.language,
|
||||
speaker: self.speaker,
|
||||
group: self.group,
|
||||
|
|
@ -193,6 +290,7 @@ impl AddArgs {
|
|||
admitted_on: self.admitted_on,
|
||||
supersedes: self.supersedes,
|
||||
withdrawn: false,
|
||||
names,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -241,7 +339,10 @@ fn main() -> Result<()> {
|
|||
Command::Corpus(CorpusCommand::Add(args)) => {
|
||||
let mut store = CorpusStore::open(&cli.corpus)?;
|
||||
let key = store.next_key(&args.slug)?;
|
||||
let outcome = store.add((*args).item(key.clone())?).map_err(refused)?;
|
||||
let entities = EntityStore::open(&cli.entities)?;
|
||||
let outcome = store
|
||||
.add((*args).item(key.clone())?, &entities)
|
||||
.map_err(refused)?;
|
||||
println!("{outcome:?} {key}");
|
||||
}
|
||||
Command::Corpus(CorpusCommand::Dump) => {
|
||||
|
|
@ -249,8 +350,8 @@ fn main() -> Result<()> {
|
|||
let split = split_text(&i);
|
||||
let withdrawn = if i.withdrawn { " withdrawn" } else { "" };
|
||||
println!(
|
||||
"{} [{} {} group={} {split}{withdrawn}]\n {:?}\n source: {:?}",
|
||||
i.id, i.language, i.speaker, i.group, i.text, i.source
|
||||
"{} [{} {} group={} {split}{withdrawn}]\n {:?}\n names: {:?}\n source: {:?}",
|
||||
i.id, i.language, i.speaker, i.group, i.text, i.names, i.source
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -278,6 +379,19 @@ fn main() -> Result<()> {
|
|||
);
|
||||
}
|
||||
}
|
||||
Command::Entity(EntityCommand::Put(args)) => {
|
||||
let entity = args.entity()?;
|
||||
let id = entity.id.clone();
|
||||
let outcome = EntityStore::open(&cli.entities)?
|
||||
.put(entity)
|
||||
.map_err(refused)?;
|
||||
println!("{outcome:?} {id}");
|
||||
}
|
||||
Command::Entity(EntityCommand::Dump) => {
|
||||
for e in EntityStore::open(&cli.entities)?.all()? {
|
||||
println!("{} {:?}\n forms: {:?}", e.id, e.origin, e.forms);
|
||||
}
|
||||
}
|
||||
Command::Inventory(InventoryCommand::Seed) => {
|
||||
let (rev, entries) = seed();
|
||||
let count = entries.len();
|
||||
|
|
|
|||
|
|
@ -1,9 +1,8 @@
|
|||
//! The interlingua packet. Every id is an opaque form type: a node id `n<k>`, a
|
||||
//! packet id `packet:<hex16>`, an encoder id `encoder:<hex16>`, an inventory id
|
||||
//! `<family>:<n>`. The free text is a bounded [`ProperName`] and the trace, which
|
||||
//! is stripped before lowering. A malformed id or name cannot be built, and
|
||||
//! cannot be read back from a store, so a packet cannot carry target-language
|
||||
//! wording through an id slot.
|
||||
//! `<family>:<n>`. The only free text is the trace, which is stripped before
|
||||
//! lowering. A malformed id cannot be built, and cannot be read back from a
|
||||
//! store, so a packet cannot carry target-language wording through an id slot.
|
||||
|
||||
use std::fmt;
|
||||
|
||||
|
|
@ -11,12 +10,13 @@ use cultcache_rs::DatabaseEntry;
|
|||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::form::{FormError, string_form};
|
||||
use crate::corpus::CorpusItemId;
|
||||
use crate::form::{FormError, is_slug, string_form};
|
||||
pub use crate::inventory::InvId;
|
||||
use crate::inventory::OptionLabel;
|
||||
|
||||
/// Bumped by a commit that changes these types; the published schema moves with it.
|
||||
pub const SCHEMA_VERSION: u32 = 1;
|
||||
pub const SCHEMA_VERSION: u32 = 3;
|
||||
|
||||
/// Id of a referent or predication within one packet: `n<k>`, `k` a canonical
|
||||
/// decimal 1..=9999.
|
||||
|
|
@ -109,6 +109,33 @@ impl fmt::Display for EncoderId {
|
|||
|
||||
string_form!(EncoderId, { "type": "string", "pattern": "^encoder:[0-9a-f]{16}$" });
|
||||
|
||||
/// Id of an entity: `entity:<slug>`, a slug as an item key's (`[a-z0-9]` and
|
||||
/// inner `-`, 1 to 48 bytes, no edge `-`).
|
||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
#[serde(try_from = "String", into = "String")]
|
||||
pub struct EntityId(String);
|
||||
|
||||
impl EntityId {
|
||||
pub fn parse(s: &str) -> Result<Self, FormError> {
|
||||
match s.strip_prefix("entity:") {
|
||||
Some(slug) if is_slug(slug) => Ok(Self(s.to_string())),
|
||||
_ => Err(FormError::new("EntityId", s)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for EntityId {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
string_form!(EntityId, { "type": "string", "pattern": "^entity:[a-z0-9]([a-z0-9-]{0,46}[a-z0-9])?$" });
|
||||
|
||||
/// Permille, 0..=1000.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
#[serde(try_from = "u16", into = "u16")]
|
||||
|
|
@ -152,60 +179,6 @@ impl JsonSchema for Grade {
|
|||
}
|
||||
}
|
||||
|
||||
/// A proper name: 1 to 4 tokens joined by single spaces, 64 bytes at most. A
|
||||
/// token is 1 to 24 letters, apostrophes (`'` or U+2019) and hyphens, beginning
|
||||
/// and ending with a letter. The form bounds capacity only: names are text the
|
||||
/// scorer masks, and no door judges them for leakage.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
#[serde(try_from = "String", into = "String")]
|
||||
pub struct ProperName(String);
|
||||
|
||||
impl ProperName {
|
||||
pub const MAX_BYTES: usize = 64;
|
||||
pub const MAX_TOKENS: usize = 4;
|
||||
pub const MAX_TOKEN_CHARS: usize = 24;
|
||||
|
||||
pub fn parse(s: &str) -> Result<Self, FormError> {
|
||||
let tokens: Vec<&str> = s.split(' ').collect();
|
||||
if s.len() <= Self::MAX_BYTES
|
||||
&& tokens.len() <= Self::MAX_TOKENS
|
||||
&& tokens.iter().all(|t| Self::token_ok(t))
|
||||
{
|
||||
Ok(Self(s.to_string()))
|
||||
} else {
|
||||
Err(FormError::new("ProperName", s))
|
||||
}
|
||||
}
|
||||
|
||||
fn token_ok(t: &str) -> bool {
|
||||
let (Some(first), Some(last)) = (t.chars().next(), t.chars().next_back()) else {
|
||||
return false;
|
||||
};
|
||||
t.chars().count() <= Self::MAX_TOKEN_CHARS
|
||||
&& first.is_alphabetic()
|
||||
&& last.is_alphabetic()
|
||||
&& t.chars()
|
||||
.all(|c| c.is_alphabetic() || matches!(c, '\'' | '\u{2019}' | '-'))
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ProperName {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
string_form!(ProperName, {
|
||||
"type": "string",
|
||||
"maxLength": 64,
|
||||
"pattern": "^\\p{L}([\\p{L}'\u{2019}-]{0,22}\\p{L})?( \\p{L}([\\p{L}'\u{2019}-]{0,22}\\p{L})?){0,3}$",
|
||||
"description": "1 to 4 space-joined tokens of letters, apostrophes and hyphens."
|
||||
});
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, JsonSchema, DatabaseEntry)]
|
||||
#[cultcache(type = "weksa.interlingua_packet")]
|
||||
pub struct Packet {
|
||||
|
|
@ -265,8 +238,9 @@ pub enum ProvenanceSource {
|
|||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Provenance {
|
||||
pub source: ProvenanceSource,
|
||||
/// Key of the corpus item this packet was minted for.
|
||||
pub corpus_item: Option<String>,
|
||||
/// The corpus item this packet was minted for. Required: the packet store
|
||||
/// admits only packets that name their item.
|
||||
pub corpus_item: CorpusItemId,
|
||||
pub encoder: EncoderId,
|
||||
pub confidence: Grade,
|
||||
}
|
||||
|
|
@ -310,9 +284,9 @@ pub struct Referent {
|
|||
pub kind: InvId,
|
||||
/// Concept id.
|
||||
pub concept: InvId,
|
||||
/// A proper name, the one place a referent carries wording. It is
|
||||
/// form-bounded text; the scorer masks it.
|
||||
pub name: Option<ProperName>,
|
||||
/// The entity this referent names, when it has one. The entity store holds
|
||||
/// its wording; the packet holds only the id.
|
||||
pub entity: Option<EntityId>,
|
||||
pub features: Vec<FeatureValue>,
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
//! documents; nothing in this crate reads JSON.
|
||||
|
||||
use crate::corpus::CorpusItem;
|
||||
use crate::entity::Entity;
|
||||
use crate::inventory::{InventoryEntry, InventoryRevision};
|
||||
use crate::packet::Packet;
|
||||
use cultcache_rs::DatabaseEntry;
|
||||
|
|
@ -19,5 +20,6 @@ pub fn published() -> Vec<(String, String)> {
|
|||
published_for::<InventoryEntry>(),
|
||||
published_for::<InventoryRevision>(),
|
||||
published_for::<CorpusItem>(),
|
||||
published_for::<Entity>(),
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,10 +7,14 @@ use std::fmt;
|
|||
use std::path::Path;
|
||||
|
||||
use cultcache_rs::{CultCache, SingleFileMessagePackBackingStore};
|
||||
use unicode_normalization::is_nfc;
|
||||
use unicode_segmentation::UnicodeSegmentation;
|
||||
|
||||
use crate::corpus::{CorpusItem, item_key, key_slug};
|
||||
use crate::corpus::{CorpusItem, CorpusItemId, item_key, key_slug};
|
||||
use crate::entity::Entity;
|
||||
use crate::form::is_language_tag;
|
||||
use crate::inventory::{Change, InvId, Inventory, InventoryEntry, InventoryRevision, revision_key};
|
||||
use crate::packet::Packet;
|
||||
use crate::packet::{EntityId, Packet};
|
||||
use crate::validate::{Refusal, validate};
|
||||
|
||||
/// A broken store rule, named.
|
||||
|
|
@ -40,10 +44,61 @@ pub enum StoreRefusal {
|
|||
UnknownCorpusItem { key: String },
|
||||
/// A packet key already holds a different packet.
|
||||
PacketImmutable { key: String },
|
||||
/// An entity id the entity store does not hold: a speaker, a prior-turn
|
||||
/// speaker, a name span's entity or a referent's entity.
|
||||
UnknownEntity { entity: EntityId },
|
||||
/// A name span with `start >= end`, `end` past the text, or an end off a
|
||||
/// character boundary.
|
||||
NameSpanRange { key: String, start: u32, end: u32 },
|
||||
/// A name span that starts or ends inside an extended grapheme cluster
|
||||
/// (UAX #29) of the item's text.
|
||||
NameSpanSplitsCluster { key: String, start: u32, end: u32 },
|
||||
/// An item text or prior-turn text, or an entity form, that is not NFC.
|
||||
/// Callers normalize at intake; the stores refuse what they were handed.
|
||||
NotNfc { key: String },
|
||||
/// An entity beyond a capacity bound.
|
||||
EntityBounds { id: EntityId, bound: EntityBound },
|
||||
/// Two name spans of one item that overlap.
|
||||
NameSpanOverlap { key: String },
|
||||
/// A name span whose bytes equal none of its entity's forms in the item's
|
||||
/// language (an entity with no forms in that language included).
|
||||
NameSpanNotAForm {
|
||||
key: String,
|
||||
entity: EntityId,
|
||||
start: u32,
|
||||
end: u32,
|
||||
},
|
||||
/// A `put` that drops, reorders or edits a stored language, form or origin.
|
||||
EntityNotAppendOnly { id: EntityId },
|
||||
/// A corpus packet's referent cites an entity that is neither named on its
|
||||
/// item nor the item's speaker or a prior-turn speaker.
|
||||
EntityNotOnItem { entity: EntityId, item: String },
|
||||
/// Any other malformed document or change.
|
||||
Malformed { key: String, why: &'static str },
|
||||
}
|
||||
|
||||
/// The capacity bound an entity broke.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum EntityBound {
|
||||
/// A language that is not a BCP 47-form tag (1 to 35 bytes).
|
||||
LanguageTag,
|
||||
/// More than 8 languages.
|
||||
Languages,
|
||||
/// More than 8 forms in one language.
|
||||
Forms,
|
||||
/// A form of more than 64 bytes.
|
||||
FormBytes,
|
||||
/// A form of more than 4 words.
|
||||
FormWords,
|
||||
}
|
||||
|
||||
impl EntityBound {
|
||||
pub const MAX_LANGUAGES: usize = 8;
|
||||
pub const MAX_FORMS: usize = 8;
|
||||
pub const MAX_FORM_BYTES: usize = 64;
|
||||
pub const MAX_FORM_WORDS: usize = 4;
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum StoreError {
|
||||
Refused(StoreRefusal),
|
||||
|
|
@ -141,23 +196,25 @@ impl CorpusStore {
|
|||
Ok(item_key(slug, highest + 1))
|
||||
}
|
||||
|
||||
/// The item `key` names, or `UnknownCorpusItem` when the key is not
|
||||
/// `item:<slug>:<n>` or holds nothing.
|
||||
fn resolve(&self, key: &str) -> Result<CorpusItem, StoreError> {
|
||||
let unknown = || StoreRefusal::UnknownCorpusItem {
|
||||
key: key.to_string(),
|
||||
};
|
||||
if key_slug(key).is_none() {
|
||||
return Err(unknown().into());
|
||||
}
|
||||
self.get(key)?.ok_or_else(|| unknown().into())
|
||||
/// The item `id` names, or `UnknownCorpusItem` when it holds nothing.
|
||||
fn resolve(&self, id: &CorpusItemId) -> Result<CorpusItem, StoreError> {
|
||||
self.get(id.as_str())?.ok_or_else(|| {
|
||||
StoreRefusal::UnknownCorpusItem {
|
||||
key: id.to_string(),
|
||||
}
|
||||
.into()
|
||||
})
|
||||
}
|
||||
|
||||
/// S1: an item key holds its whole item for good. S6: a text held under
|
||||
/// one key (live or withdrawn) is not re-added under another unless the new
|
||||
/// item supersedes that key. S5: a correction stays in its predecessor's
|
||||
/// group, so it cannot move across the split.
|
||||
pub fn add(&mut self, item: CorpusItem) -> Result<AddOutcome, StoreError> {
|
||||
pub fn add(
|
||||
&mut self,
|
||||
item: CorpusItem,
|
||||
entities: &EntityStore,
|
||||
) -> Result<AddOutcome, StoreError> {
|
||||
if key_slug(&item.id).is_none() {
|
||||
return Err(malformed(&item.id, "item key is not item:<slug>:<n>"));
|
||||
}
|
||||
|
|
@ -173,6 +230,13 @@ impl CorpusStore {
|
|||
}
|
||||
return Err(StoreRefusal::S1ItemImmutable { key: item.id }.into());
|
||||
}
|
||||
if !is_language_tag(&item.language) {
|
||||
return Err(malformed(&item.id, "language is not a BCP 47-form tag"));
|
||||
}
|
||||
if !is_nfc(&item.text) || item.prior_turns.iter().any(|t| !is_nfc(&t.text)) {
|
||||
return Err(StoreRefusal::NotNfc { key: item.id }.into());
|
||||
}
|
||||
check_entities(&item, entities)?;
|
||||
let text = normalized(&item.text);
|
||||
for held in self.all()? {
|
||||
if normalized(&held.text) == text && item.supersedes.as_deref() != Some(&held.id) {
|
||||
|
|
@ -216,6 +280,76 @@ impl CorpusStore {
|
|||
}
|
||||
}
|
||||
|
||||
/// Every entity an item cites exists, and every name span is a well-formed,
|
||||
/// non-overlapping byte range of the text, starting and ending on extended
|
||||
/// grapheme cluster boundaries, equal to one of its entity's forms in the
|
||||
/// item's language. Id lookup, UAX #29 boundaries and byte equality: no reading
|
||||
/// of meaning, so every script is held to the same check.
|
||||
fn check_entities(item: &CorpusItem, entities: &EntityStore) -> Result<(), StoreError> {
|
||||
let known = |id: &EntityId| -> Result<Entity, StoreError> {
|
||||
entities
|
||||
.get(id)?
|
||||
.ok_or_else(|| StoreRefusal::UnknownEntity { entity: id.clone() }.into())
|
||||
};
|
||||
known(&item.speaker)?;
|
||||
for turn in &item.prior_turns {
|
||||
known(&turn.speaker)?;
|
||||
}
|
||||
let mut ranges = Vec::with_capacity(item.names.len());
|
||||
let cuts: BTreeSet<usize> = item
|
||||
.text
|
||||
.grapheme_indices(true)
|
||||
.map(|(i, _)| i)
|
||||
.chain([item.text.len()])
|
||||
.collect();
|
||||
for span in &item.names {
|
||||
let entity = known(&span.entity)?;
|
||||
let (start, end) = (span.start as usize, span.end as usize);
|
||||
let Some(bytes) = item
|
||||
.text
|
||||
.get(start..end)
|
||||
.filter(|_| start < end)
|
||||
.map(str::as_bytes)
|
||||
else {
|
||||
return Err(StoreRefusal::NameSpanRange {
|
||||
key: item.id.clone(),
|
||||
start: span.start,
|
||||
end: span.end,
|
||||
}
|
||||
.into());
|
||||
};
|
||||
if !cuts.contains(&start) || !cuts.contains(&end) {
|
||||
return Err(StoreRefusal::NameSpanSplitsCluster {
|
||||
key: item.id.clone(),
|
||||
start: span.start,
|
||||
end: span.end,
|
||||
}
|
||||
.into());
|
||||
}
|
||||
let is_form = entity
|
||||
.forms_in(&item.language)
|
||||
.is_some_and(|forms| forms.iter().any(|f| f.as_bytes() == bytes));
|
||||
if !is_form {
|
||||
return Err(StoreRefusal::NameSpanNotAForm {
|
||||
key: item.id.clone(),
|
||||
entity: span.entity.clone(),
|
||||
start: span.start,
|
||||
end: span.end,
|
||||
}
|
||||
.into());
|
||||
}
|
||||
ranges.push((span.start, span.end));
|
||||
}
|
||||
ranges.sort_unstable();
|
||||
if ranges.windows(2).any(|w| w[1].0 < w[0].1) {
|
||||
return Err(StoreRefusal::NameSpanOverlap {
|
||||
key: item.id.clone(),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Lowercased alphanumerics only, so case, spacing and punctuation variants of
|
||||
/// a text compare equal.
|
||||
fn normalized(text: &str) -> String {
|
||||
|
|
@ -225,6 +359,112 @@ fn normalized(text: &str) -> String {
|
|||
.collect()
|
||||
}
|
||||
|
||||
pub struct EntityStore {
|
||||
cache: CultCache,
|
||||
}
|
||||
|
||||
impl EntityStore {
|
||||
pub fn open(path: &Path) -> anyhow::Result<Self> {
|
||||
Ok(Self {
|
||||
cache: open_cache(path, |c| c.register_entry_type::<Entity>())?,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn all(&self) -> anyhow::Result<Vec<Entity>> {
|
||||
let mut entities = self.cache.get_all::<Entity>()?;
|
||||
entities.sort_by(|a, b| a.id.cmp(&b.id));
|
||||
Ok(entities)
|
||||
}
|
||||
|
||||
pub fn get(&self, id: &EntityId) -> anyhow::Result<Option<Entity>> {
|
||||
self.cache.get::<Entity>(id.as_str())
|
||||
}
|
||||
|
||||
/// Admits a new entity, an identical replay, or a replacement that keeps
|
||||
/// every stored language and form in order and origin as it was, and only
|
||||
/// appends: forms grow, never shrink, reorder or change. An entity has 1 to
|
||||
/// 8 languages, each a BCP 47-form tag named once, with 1 to 8 NFC forms of
|
||||
/// 1 to 64 bytes and at most 4 words, no control character, no edge
|
||||
/// whitespace, none repeated.
|
||||
pub fn put(&mut self, entity: Entity) -> Result<AddOutcome, StoreError> {
|
||||
let key = entity.id.to_string();
|
||||
check_entity_shape(&entity)?;
|
||||
if let Some(old) = self.get(&entity.id)? {
|
||||
if old == entity {
|
||||
return Ok(AddOutcome::Unchanged);
|
||||
}
|
||||
let grows = old.origin == entity.origin
|
||||
&& old.forms.len() <= entity.forms.len()
|
||||
&& old.forms.iter().zip(&entity.forms).all(|(o, n)| {
|
||||
o.language == n.language
|
||||
&& o.forms.len() <= n.forms.len()
|
||||
&& n.forms.starts_with(&o.forms)
|
||||
});
|
||||
if !grows {
|
||||
return Err(StoreRefusal::EntityNotAppendOnly { id: entity.id }.into());
|
||||
}
|
||||
}
|
||||
self.cache.put(key, &entity)?;
|
||||
Ok(AddOutcome::Added)
|
||||
}
|
||||
}
|
||||
|
||||
fn check_entity_shape(entity: &Entity) -> Result<(), StoreError> {
|
||||
let bad = |why| malformed(entity.id.as_str(), why);
|
||||
let bound = |bound| -> StoreError {
|
||||
StoreRefusal::EntityBounds {
|
||||
id: entity.id.clone(),
|
||||
bound,
|
||||
}
|
||||
.into()
|
||||
};
|
||||
if entity.forms.is_empty() {
|
||||
return Err(bad("an entity has forms in at least one language"));
|
||||
}
|
||||
if entity.forms.len() > EntityBound::MAX_LANGUAGES {
|
||||
return Err(bound(EntityBound::Languages));
|
||||
}
|
||||
let mut languages = BTreeSet::new();
|
||||
for lf in &entity.forms {
|
||||
if !is_language_tag(&lf.language) {
|
||||
return Err(bound(EntityBound::LanguageTag));
|
||||
}
|
||||
if !languages.insert(&lf.language) {
|
||||
return Err(bad("a language is named once"));
|
||||
}
|
||||
if lf.forms.is_empty() {
|
||||
return Err(bad("a language has at least one form"));
|
||||
}
|
||||
if lf.forms.len() > EntityBound::MAX_FORMS {
|
||||
return Err(bound(EntityBound::Forms));
|
||||
}
|
||||
let mut seen = BTreeSet::new();
|
||||
for f in &lf.forms {
|
||||
if f.len() > EntityBound::MAX_FORM_BYTES {
|
||||
return Err(bound(EntityBound::FormBytes));
|
||||
}
|
||||
if f.split_whitespace().count() > EntityBound::MAX_FORM_WORDS {
|
||||
return Err(bound(EntityBound::FormWords));
|
||||
}
|
||||
if !is_nfc(f) {
|
||||
return Err(StoreRefusal::NotNfc {
|
||||
key: entity.id.to_string(),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
if f.is_empty() || f.chars().any(char::is_control) || f.trim() != f.as_str() {
|
||||
return Err(bad(
|
||||
"a form is not empty and has no control character or edge whitespace",
|
||||
));
|
||||
}
|
||||
if !seen.insert(f) {
|
||||
return Err(bad("a form is not repeated within its language"));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub struct InventoryStore {
|
||||
cache: CultCache,
|
||||
}
|
||||
|
|
@ -360,16 +600,36 @@ impl PacketStore {
|
|||
|
||||
/// The only door: a packet exists only if `validate` passes against the
|
||||
/// inventory the store holds at the packet's revision, its `corpus_item`
|
||||
/// (when set) resolves, and a key holds its packet for good.
|
||||
/// resolves, every referent's entity is in the entity store and is named on
|
||||
/// that item or is its speaker or a prior-turn speaker (an id-set check). A
|
||||
/// key holds its packet for good. A packet that names no corpus item has no
|
||||
/// door here.
|
||||
pub fn admit(
|
||||
&mut self,
|
||||
packet: Packet,
|
||||
inventory: &InventoryStore,
|
||||
corpus: &CorpusStore,
|
||||
entities: &EntityStore,
|
||||
) -> Result<(), StoreError> {
|
||||
validate(&packet, &inventory.inventory()?).map_err(StoreError::Invalid)?;
|
||||
if let Some(item_key) = &packet.provenance.corpus_item {
|
||||
corpus.resolve(item_key)?;
|
||||
let item = corpus.resolve(&packet.provenance.corpus_item)?;
|
||||
for entity in packet.referents.iter().filter_map(|r| r.entity.as_ref()) {
|
||||
if entities.get(entity)?.is_none() {
|
||||
return Err(StoreRefusal::UnknownEntity {
|
||||
entity: entity.clone(),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
let on_item = item.names.iter().any(|n| &n.entity == entity)
|
||||
|| &item.speaker == entity
|
||||
|| item.prior_turns.iter().any(|t| &t.speaker == entity);
|
||||
if !on_item {
|
||||
return Err(StoreRefusal::EntityNotOnItem {
|
||||
entity: entity.clone(),
|
||||
item: item.id.clone(),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
}
|
||||
let key = packet.id.to_string();
|
||||
if let Some(existing) = self.get(&key)? {
|
||||
|
|
|
|||
|
|
@ -4,9 +4,11 @@
|
|||
//! or a grade is its type's parse, not a rule, so V9 is retired and not
|
||||
//! renumbered.
|
||||
//!
|
||||
//! `validate` owns form and capacity and makes no leak decision. A packet's
|
||||
//! free text is form-bounded and every list is bounded, so a packet has no
|
||||
//! unbounded channel. What bounds each list (V11, V12): a feature list holds at
|
||||
//! `validate` owns form and capacity and makes no leak decision. A packet's one
|
||||
//! free text is its trace (`notes` and `rejected_readings`): `validate` does not
|
||||
//! bound or read it, and [`LoweringInput`] drops it, so no lowerer sees it.
|
||||
//! Every other slot is an id type, so the lists are the bounded part. What bounds
|
||||
//! each list (V11, V12): a feature list holds at
|
||||
//! most one value per feature id and `extensions` at most one per extension id,
|
||||
//! so each is at most the inventory's ids of that kind; `participants`,
|
||||
//! `audience` and `preserve` hold each node id at most once and every id names
|
||||
|
|
@ -14,7 +16,7 @@
|
|||
//! hold each role id at most once, so at most the inventory's roles; and the
|
||||
//! nodes themselves are at most `Packet::MAX_NODES` (V12). Whether a score may
|
||||
//! rise because corpus text reached a packet is owned by the golf loop's
|
||||
//! context builder and the scorer (ruling `leak-owned-by-flow-and-scorer`),
|
||||
//! context builder and the scorer (ruling `leak-owned-by-flow-and-scorer-3`),
|
||||
//! never by a text rule here, in `admit_revision` or in `PacketStore::admit`.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
|
|
|
|||
|
|
@ -5,12 +5,42 @@ use weksa::{InventoryStore, PacketStore};
|
|||
fn weksa(dir: &std::path::Path, args: &[&str]) -> Output {
|
||||
Command::new(env!("CARGO_BIN_EXE_weksa"))
|
||||
.current_dir(dir)
|
||||
.args(["--corpus", "./c.cc", "--inventory", "./i.cc"])
|
||||
.args([
|
||||
"--corpus",
|
||||
"./c.cc",
|
||||
"--inventory",
|
||||
"./i.cc",
|
||||
"--entities",
|
||||
"./e.cc",
|
||||
])
|
||||
.args(args)
|
||||
.output()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// Admits the cast entities the tests cite.
|
||||
fn cast_entities(dir: &std::path::Path, slugs: &[&str]) {
|
||||
for slug in slugs {
|
||||
let id = format!("entity:{slug}");
|
||||
let form = format!("en-US={}", slug.to_uppercase());
|
||||
let put = weksa(
|
||||
dir,
|
||||
&[
|
||||
"entity",
|
||||
"put",
|
||||
&id,
|
||||
"--form",
|
||||
&form,
|
||||
"--origin",
|
||||
"cast",
|
||||
"--persona",
|
||||
slug,
|
||||
],
|
||||
);
|
||||
assert!(put.status.success(), "{}", out(&put));
|
||||
}
|
||||
}
|
||||
|
||||
fn out(o: &Output) -> String {
|
||||
String::from_utf8_lossy(&o.stdout).into_owned() + &String::from_utf8_lossy(&o.stderr)
|
||||
}
|
||||
|
|
@ -23,7 +53,7 @@ const ADD: [&str; 14] = [
|
|||
"--text",
|
||||
"Mind the hatch.",
|
||||
"--speaker",
|
||||
"nibu",
|
||||
"entity:nibu",
|
||||
"--group",
|
||||
"scene-1",
|
||||
"--admitted-on",
|
||||
|
|
@ -35,6 +65,7 @@ const ADD: [&str; 14] = [
|
|||
#[test]
|
||||
fn seed_add_dump_split_withdraw() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
cast_entities(d.path(), &["nibu"]);
|
||||
let seeded = weksa(d.path(), &["inventory", "seed"]);
|
||||
assert!(seeded.status.success(), "{}", out(&seeded));
|
||||
assert!(out(&seeded).contains("seeded rev:1"));
|
||||
|
|
@ -105,13 +136,14 @@ fn seed_add_dump_split_withdraw() {
|
|||
#[test]
|
||||
fn model_written_and_public_domain_sources_name_their_provenance() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
cast_entities(d.path(), &["s", "player"]);
|
||||
let base = [
|
||||
"corpus",
|
||||
"add",
|
||||
"--slug",
|
||||
"x",
|
||||
"--speaker",
|
||||
"s",
|
||||
"entity:s",
|
||||
"--group",
|
||||
"g",
|
||||
"--admitted-on",
|
||||
|
|
@ -186,7 +218,7 @@ fn model_written_and_public_domain_sources_name_their_provenance() {
|
|||
"--source",
|
||||
"operator-written",
|
||||
"--prior-turn",
|
||||
"player=hello",
|
||||
"entity:player=hello",
|
||||
]);
|
||||
assert!(turn.status.success(), "{}", out(&turn));
|
||||
}
|
||||
|
|
@ -200,7 +232,7 @@ fn validate_reports_each_packet() {
|
|||
let packet = common::good();
|
||||
PacketStore::open(&packets)
|
||||
.unwrap()
|
||||
.admit(packet.clone(), &w.inventory, &w.corpus)
|
||||
.admit(packet.clone(), &w.inventory, &w.corpus, &w.entities)
|
||||
.unwrap();
|
||||
|
||||
let validate = |inventory: &std::path::Path, extra: &[&str]| {
|
||||
|
|
@ -250,9 +282,188 @@ fn schema_writes_the_published_files() {
|
|||
names,
|
||||
[
|
||||
"weksa.corpus_item.schema.json",
|
||||
"weksa.entity.schema.json",
|
||||
"weksa.interlingua_packet.schema.json",
|
||||
"weksa.inventory_entry.schema.json",
|
||||
"weksa.inventory_revision.schema.json"
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entity_put_and_name_spans() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
let put = |args: &[&str]| {
|
||||
let mut all = vec!["entity", "put", "entity:kay", "--origin", "cast"];
|
||||
all.extend_from_slice(args);
|
||||
weksa(d.path(), &all)
|
||||
};
|
||||
let no_persona = put(&["--form", "en-US=Kay"]);
|
||||
assert!(
|
||||
!no_persona.status.success() && out(&no_persona).contains("--persona"),
|
||||
"{}",
|
||||
out(&no_persona)
|
||||
);
|
||||
let first = put(&["--form", "en-US=Kay", "--persona", "kay"]);
|
||||
assert!(
|
||||
first.status.success() && out(&first).contains("Added entity:kay"),
|
||||
"{}",
|
||||
out(&first)
|
||||
);
|
||||
// Forms append; a language's forms keep the order given.
|
||||
let grown = put(&[
|
||||
"--form",
|
||||
"en-US=Kay",
|
||||
"--form",
|
||||
"en-US=Kay's",
|
||||
"--persona",
|
||||
"kay",
|
||||
]);
|
||||
assert!(grown.status.success(), "{}", out(&grown));
|
||||
let shrunk = put(&["--form", "en-US=Kay", "--persona", "kay"]);
|
||||
assert!(
|
||||
!shrunk.status.success() && out(&shrunk).contains("EntityNotAppendOnly"),
|
||||
"{}",
|
||||
out(&shrunk)
|
||||
);
|
||||
let dump = out(&weksa(d.path(), &["entity", "dump"]));
|
||||
assert!(
|
||||
dump.contains("entity:kay") && dump.contains("Kay's"),
|
||||
"{dump}"
|
||||
);
|
||||
|
||||
cast_entities(d.path(), &["nibu"]);
|
||||
let add = |range: &str, text: &str| {
|
||||
weksa(
|
||||
d.path(),
|
||||
&[
|
||||
"corpus",
|
||||
"add",
|
||||
"--slug",
|
||||
"kay",
|
||||
"--text",
|
||||
text,
|
||||
"--speaker",
|
||||
"entity:nibu",
|
||||
"--group",
|
||||
"g",
|
||||
"--admitted-on",
|
||||
"2026-10-29",
|
||||
"--source",
|
||||
"operator-written",
|
||||
"--name",
|
||||
range,
|
||||
],
|
||||
)
|
||||
};
|
||||
let named = add("0..3=entity:kay", "Kay left.");
|
||||
assert!(named.status.success(), "{}", out(&named));
|
||||
let unequal = add("0..3=entity:kay", "Kai left.");
|
||||
assert!(
|
||||
!unequal.status.success() && out(&unequal).contains("NameSpanNotAForm"),
|
||||
"{}",
|
||||
out(&unequal)
|
||||
);
|
||||
let malformed = add("0-3", "Kay again.");
|
||||
assert!(
|
||||
!malformed.status.success() && out(&malformed).contains("start..end=entity"),
|
||||
"{}",
|
||||
out(&malformed)
|
||||
);
|
||||
let dump = out(&weksa(d.path(), &["corpus", "dump"]));
|
||||
assert!(
|
||||
dump.contains("names:") && dump.contains("entity:kay"),
|
||||
"{dump}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entity_origin_flags_must_match_origin() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
let put = |args: &[&str]| {
|
||||
let mut all = vec!["entity", "put", "entity:kay", "--form", "en-US=Kay"];
|
||||
all.extend_from_slice(args);
|
||||
weksa(d.path(), &all)
|
||||
};
|
||||
for mismatched in [
|
||||
&["--origin", "live", "--persona", "kay"][..],
|
||||
&["--origin", "live", "--work", "edda"],
|
||||
&["--origin", "cast", "--persona", "kay", "--work", "edda"],
|
||||
&[
|
||||
"--origin",
|
||||
"public-domain",
|
||||
"--persona",
|
||||
"kay",
|
||||
"--work",
|
||||
"edda",
|
||||
],
|
||||
&["--origin", "public-domain", "--persona", "kay"],
|
||||
] {
|
||||
let o = put(mismatched);
|
||||
assert!(!o.status.success(), "{mismatched:?}: {}", out(&o));
|
||||
}
|
||||
assert!(!out(&weksa(d.path(), &["entity", "dump"])).contains("entity:kay"));
|
||||
assert!(put(&["--origin", "live"]).status.success());
|
||||
let work = weksa(
|
||||
d.path(),
|
||||
&[
|
||||
"entity",
|
||||
"put",
|
||||
"entity:brokkr",
|
||||
"--form",
|
||||
"en-US=Brokkr",
|
||||
"--origin",
|
||||
"public-domain",
|
||||
"--work",
|
||||
"edda",
|
||||
],
|
||||
);
|
||||
assert!(work.status.success(), "{}", out(&work));
|
||||
assert!(out(&weksa(d.path(), &["entity", "dump"])).contains("PublicDomain"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn intake_normalizes_to_nfc() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
// A decomposed form and decomposed text meet as the same composed bytes.
|
||||
let put = weksa(
|
||||
d.path(),
|
||||
&[
|
||||
"entity",
|
||||
"put",
|
||||
"entity:rene",
|
||||
"--form",
|
||||
"fr=Rene\u{301}",
|
||||
"--origin",
|
||||
"live",
|
||||
],
|
||||
);
|
||||
assert!(put.status.success(), "{}", out(&put));
|
||||
cast_entities(d.path(), &["nibu"]);
|
||||
let add = weksa(
|
||||
d.path(),
|
||||
&[
|
||||
"corpus",
|
||||
"add",
|
||||
"--slug",
|
||||
"rene",
|
||||
"--text",
|
||||
"Rene\u{301} part.",
|
||||
"--language",
|
||||
"fr",
|
||||
"--speaker",
|
||||
"entity:nibu",
|
||||
"--group",
|
||||
"g",
|
||||
"--admitted-on",
|
||||
"2026-10-29",
|
||||
"--source",
|
||||
"operator-written",
|
||||
"--name",
|
||||
"0..5=entity:rene",
|
||||
],
|
||||
);
|
||||
assert!(add.status.success(), "{}", out(&add));
|
||||
let dump = out(&weksa(d.path(), &["corpus", "dump"]));
|
||||
assert!(dump.contains("Ren\u{e9} part."), "{dump}");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,8 +7,9 @@ use weksa::corpus::ModelFamily;
|
|||
use weksa::inventory::{Change, seed};
|
||||
use weksa::packet::*;
|
||||
use weksa::{
|
||||
CorpusItem, CorpusStore, InvId, Inventory, InventoryEntry, InventoryRevision, InventoryStore,
|
||||
OptionLabel, Slot, Source, SplitScheme, ValueKind,
|
||||
CorpusItem, CorpusItemId, CorpusStore, Entity, EntityOrigin, EntityStore, InvId, Inventory,
|
||||
InventoryEntry, InventoryRevision, InventoryStore, LanguageForms, NameSpan, OptionLabel, Slot,
|
||||
Source, SplitScheme, ValueKind,
|
||||
};
|
||||
|
||||
pub const PERSON: &str = "referent_kind:1";
|
||||
|
|
@ -73,12 +74,45 @@ pub fn model() -> Source {
|
|||
}
|
||||
}
|
||||
|
||||
/// Every speaker the tests use, as entities with one en-US form.
|
||||
const SPEAKERS: [&str; 6] = ["nibu", "brokkr", "player", "s", "someone", "hero"];
|
||||
|
||||
pub fn eid(slug: &str) -> EntityId {
|
||||
EntityId::parse(&format!("entity:{slug}")).unwrap()
|
||||
}
|
||||
|
||||
/// A cast entity with `forms` in `language`, citation form first.
|
||||
pub fn ent(slug: &str, language: &str, forms: &[&str]) -> Entity {
|
||||
Entity {
|
||||
id: eid(slug),
|
||||
forms: vec![LanguageForms {
|
||||
language: language.into(),
|
||||
forms: forms.iter().map(|f| f.to_string()).collect(),
|
||||
}],
|
||||
origin: EntityOrigin::Cast {
|
||||
persona: slug.into(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// The entity store at `entities.cc` under `dir`, holding every test speaker
|
||||
/// (en-US form: the slug capitalized). Reopening replays identically.
|
||||
pub fn entities_in(dir: &std::path::Path) -> EntityStore {
|
||||
let mut store = EntityStore::open(&dir.join("entities.cc")).unwrap();
|
||||
for slug in SPEAKERS {
|
||||
let form = slug[..1].to_uppercase() + &slug[1..];
|
||||
store.put(ent(slug, "en-US", &[&form])).unwrap();
|
||||
}
|
||||
store
|
||||
}
|
||||
|
||||
pub fn item_in(key: &str, text: &str, group: &str, source: Source) -> CorpusItem {
|
||||
CorpusItem {
|
||||
id: key.into(),
|
||||
text: text.into(),
|
||||
language: "en-US".into(),
|
||||
speaker: "nibu".into(),
|
||||
speaker: eid("nibu"),
|
||||
names: vec![],
|
||||
group: group.into(),
|
||||
prior_turns: vec![],
|
||||
source,
|
||||
|
|
@ -132,6 +166,7 @@ pub struct World {
|
|||
pub dir: TempDir,
|
||||
pub inventory: InventoryStore,
|
||||
pub corpus: CorpusStore,
|
||||
pub entities: EntityStore,
|
||||
}
|
||||
|
||||
impl World {
|
||||
|
|
@ -143,15 +178,15 @@ impl World {
|
|||
pub fn world() -> World {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut corpus = CorpusStore::open(&dir.path().join("c.cc")).unwrap();
|
||||
let entities = entities_in(dir.path());
|
||||
let train = group_in_bucket(10);
|
||||
corpus
|
||||
.add(item_in(
|
||||
"item:nibu:1",
|
||||
"Nibu says mind the hatch.",
|
||||
&train,
|
||||
cast(),
|
||||
))
|
||||
.unwrap();
|
||||
let mut first = item_in("item:nibu:1", "Nibu says mind the hatch.", &train, cast());
|
||||
first.names = vec![NameSpan {
|
||||
entity: eid("nibu"),
|
||||
start: 0,
|
||||
end: 4,
|
||||
}];
|
||||
corpus.add(first, &entities).unwrap();
|
||||
let mut inventory = InventoryStore::open(&dir.path().join("i.cc")).unwrap();
|
||||
let (rev1, entries) = seed();
|
||||
inventory.admit_revision(rev1, entries).unwrap();
|
||||
|
|
@ -192,6 +227,7 @@ pub fn world() -> World {
|
|||
dir,
|
||||
inventory,
|
||||
corpus,
|
||||
entities,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -209,7 +245,7 @@ pub fn good() -> Packet {
|
|||
kind: PacketKind::DialogueLine,
|
||||
provenance: Provenance {
|
||||
source: ProvenanceSource::Authored,
|
||||
corpus_item: Some("item:nibu:1".into()),
|
||||
corpus_item: CorpusItemId::parse("item:nibu:1").unwrap(),
|
||||
encoder: EncoderId::parse(ENCODER_ID).unwrap(),
|
||||
confidence: grade(900),
|
||||
},
|
||||
|
|
@ -231,7 +267,7 @@ pub fn good() -> Packet {
|
|||
id: nd("n1"),
|
||||
kind: iv(PERSON),
|
||||
concept: iv(CONCEPT_A),
|
||||
name: Some(ProperName::parse("Nibu").unwrap()),
|
||||
entity: Some(eid("nibu")),
|
||||
features: vec![],
|
||||
}],
|
||||
predications: vec![Predication {
|
||||
|
|
|
|||
419
crates/weksa/tests/entities.rs
Normal file
419
crates/weksa/tests/entities.rs
Normal file
|
|
@ -0,0 +1,419 @@
|
|||
mod common;
|
||||
use common::*;
|
||||
use weksa::corpus::{CorpusItem, NameSpan, Turn};
|
||||
use weksa::packet::{Packet, Referent};
|
||||
use weksa::{
|
||||
AddOutcome, CorpusItemId, CorpusStore, Entity, EntityId, EntityOrigin, EntityStore, PacketId,
|
||||
PacketStore, Refusal, SCHEMA_VERSION, StoreError, StoreRefusal, validate,
|
||||
};
|
||||
|
||||
fn refusal(r: Result<impl std::fmt::Debug, StoreError>) -> StoreRefusal {
|
||||
match r {
|
||||
Err(StoreError::Refused(r)) => r,
|
||||
other => panic!("expected a store refusal, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn malformed(r: Result<impl std::fmt::Debug, StoreError>) -> &'static str {
|
||||
match refusal(r) {
|
||||
StoreRefusal::Malformed { why, .. } => why,
|
||||
other => panic!("expected Malformed, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn span(slug: &str, start: u32, end: u32) -> NameSpan {
|
||||
NameSpan {
|
||||
entity: eid(slug),
|
||||
start,
|
||||
end,
|
||||
}
|
||||
}
|
||||
|
||||
/// An item in `language` whose text and spans the test chooses.
|
||||
fn named(n: u32, language: &str, text: &str, names: Vec<NameSpan>) -> CorpusItem {
|
||||
CorpusItem {
|
||||
language: language.into(),
|
||||
names,
|
||||
..item_in(&format!("item:t:{n}"), text, "g", cast())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entity_id_form() {
|
||||
for ok in ["entity:brokkr", "entity:a-1", "entity:7", "entity:a-b-c"] {
|
||||
assert_eq!(EntityId::parse(ok).unwrap().as_str(), ok);
|
||||
}
|
||||
let long_ok = format!("entity:{}", "a".repeat(48));
|
||||
assert!(EntityId::parse(&long_ok).is_ok());
|
||||
let too_long = format!("entity:{}", "a".repeat(49));
|
||||
for bad in [
|
||||
"Entity:x",
|
||||
"entity:-a",
|
||||
"entity:a-",
|
||||
"entity:",
|
||||
"entity:a_b",
|
||||
"entity:A",
|
||||
"entity:a b",
|
||||
"entity:a:b",
|
||||
"brokkr",
|
||||
"entity:\u{5c71}",
|
||||
&too_long,
|
||||
] {
|
||||
assert!(EntityId::parse(bad).is_err(), "{bad:?}");
|
||||
assert!(
|
||||
serde_json::from_value::<EntityId>(bad.into()).is_err(),
|
||||
"{bad:?} must not read back from a store"
|
||||
);
|
||||
}
|
||||
// A referent reads its entity through the same parse.
|
||||
let mut referent = serde_json::to_value(good().referents[0].clone()).unwrap();
|
||||
assert!(serde_json::from_value::<Referent>(referent.clone()).is_ok());
|
||||
referent["entity"] = "Nibu".into();
|
||||
assert!(serde_json::from_value::<Referent>(referent).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entity_forms_only_grow() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
let mut store = EntityStore::open(&d.path().join("e.cc")).unwrap();
|
||||
let kay = ent("kay", "en-US", &["Kay"]);
|
||||
assert_eq!(store.put(kay.clone()).unwrap(), AddOutcome::Added);
|
||||
assert_eq!(store.put(kay.clone()).unwrap(), AddOutcome::Unchanged);
|
||||
|
||||
// A form appends; a language appends.
|
||||
let mut grown = kay.clone();
|
||||
grown.forms[0].forms.push("Kay's".into());
|
||||
assert_eq!(store.put(grown.clone()).unwrap(), AddOutcome::Added);
|
||||
let mut more = grown.clone();
|
||||
more.forms.push(weksa::LanguageForms {
|
||||
language: "tr".into(),
|
||||
forms: vec!["Kay".into()],
|
||||
});
|
||||
assert_eq!(store.put(more.clone()).unwrap(), AddOutcome::Added);
|
||||
// The store read back from disk holds the growth.
|
||||
let mut store = EntityStore::open(&d.path().join("e.cc")).unwrap();
|
||||
assert_eq!(store.get(&eid("kay")).unwrap(), Some(more.clone()));
|
||||
|
||||
// Dropping, reordering, editing and re-originating are refused, each leaving
|
||||
// the stored entity as it was.
|
||||
let not_append = StoreRefusal::EntityNotAppendOnly { id: eid("kay") };
|
||||
let edits: [fn(&mut Entity); 6] = [
|
||||
|e| e.forms[0].forms.reverse(),
|
||||
|e| e.forms[0].forms[0] = "Kai".into(),
|
||||
|e| {
|
||||
e.forms.pop();
|
||||
},
|
||||
|e| e.forms.reverse(),
|
||||
|e| e.forms[1].language = "fi".into(),
|
||||
|e| e.origin = EntityOrigin::Live,
|
||||
];
|
||||
assert_eq!(
|
||||
refusal(store.put(ent("kay", "en-US", &["Kay"]))),
|
||||
not_append
|
||||
);
|
||||
for edit in edits {
|
||||
let mut v = more.clone();
|
||||
edit(&mut v);
|
||||
assert_eq!(refusal(store.put(v.clone())), not_append, "{v:?}");
|
||||
}
|
||||
assert_eq!(store.get(&eid("kay")).unwrap(), Some(more));
|
||||
|
||||
// Shape: no forms, a language twice, an empty language, an empty language
|
||||
// entry, an empty, padded, control-bearing, oversize or repeated form.
|
||||
let shape = |forms: Vec<weksa::LanguageForms>| weksa::Entity {
|
||||
forms,
|
||||
..ent("shape", "en-US", &["X"])
|
||||
};
|
||||
let lf = |language: &str, forms: &[&str]| weksa::LanguageForms {
|
||||
language: language.into(),
|
||||
forms: forms.iter().map(|f| f.to_string()).collect(),
|
||||
};
|
||||
for bad in [
|
||||
vec![],
|
||||
vec![lf("en-US", &["A"]), lf("en-US", &["B"])],
|
||||
vec![lf("en-US", &[])],
|
||||
vec![lf("en-US", &[""])],
|
||||
vec![lf("en-US", &[" A"])],
|
||||
vec![lf("en-US", &["A\u{a0}"])],
|
||||
vec![lf("en-US", &["A\nB"])],
|
||||
vec![lf("en-US", &["A", "A"])],
|
||||
] {
|
||||
malformed(store.put(shape(bad.clone())));
|
||||
assert!(
|
||||
store.get(&eid("shape")).unwrap().is_none(),
|
||||
"{bad:?} was stored"
|
||||
);
|
||||
}
|
||||
// An empty language is not a tag.
|
||||
let e = shape(vec![lf("", &["A"])]);
|
||||
assert!(matches!(
|
||||
refusal(store.put(e)),
|
||||
StoreRefusal::EntityBounds {
|
||||
bound: weksa::store::EntityBound::LanguageTag,
|
||||
..
|
||||
}
|
||||
));
|
||||
// 64 bytes is the edge; a form may repeat across languages.
|
||||
let edge = "a".repeat(64);
|
||||
store
|
||||
.put(shape(vec![lf("en-US", &[&edge]), lf("tr", &[&edge])]))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corpus_spans_checked() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
let mut ents = entities_in(d.path());
|
||||
ents.put(ent("kay", "en-US", &["Kay"])).unwrap();
|
||||
ents.put(ent("ab", "en-US", &["abab", "bab"])).unwrap();
|
||||
let mut corpus = CorpusStore::open(&d.path().join("c.cc")).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
corpus
|
||||
.add(
|
||||
named(1, "en-US", "Kay left.", vec![span("kay", 0, 3)]),
|
||||
&ents
|
||||
)
|
||||
.unwrap(),
|
||||
AddOutcome::Added
|
||||
);
|
||||
|
||||
let range = |n: u32, start: u32, end: u32| StoreRefusal::NameSpanRange {
|
||||
key: format!("item:t:{n}"),
|
||||
start,
|
||||
end,
|
||||
};
|
||||
let not_a_form = |n: u32, slug: &str, start: u32, end: u32| StoreRefusal::NameSpanNotAForm {
|
||||
key: format!("item:t:{n}"),
|
||||
entity: eid(slug),
|
||||
start,
|
||||
end,
|
||||
};
|
||||
let mut refused = |item: CorpusItem| refusal(corpus.add(item, &ents));
|
||||
// Empty, backwards, past the end: a range refusal.
|
||||
for (start, end) in [(3, 3), (4, 3), (0, 99)] {
|
||||
let item = named(2, "en-US", "Kay left.", vec![span("kay", start, end)]);
|
||||
assert_eq!(refused(item), range(2, start, end));
|
||||
}
|
||||
// Inside a multibyte character: start and end off the boundary.
|
||||
let text = "\u{e9}Kay";
|
||||
assert_eq!(text.len(), 5);
|
||||
let item = named(2, "en-US", text, vec![span("kay", 1, 4)]);
|
||||
assert_eq!(refused(item), range(2, 1, 4));
|
||||
let item = named(2, "en-US", "Kay\u{e9}", vec![span("kay", 0, 4)]);
|
||||
assert_eq!(refused(item), range(2, 0, 4));
|
||||
// An unknown entity.
|
||||
let item = named(2, "en-US", "Kay left.", vec![span("ghost", 0, 3)]);
|
||||
assert_eq!(
|
||||
refused(item),
|
||||
StoreRefusal::UnknownEntity {
|
||||
entity: eid("ghost")
|
||||
}
|
||||
);
|
||||
// Not byte-equal: other text, other case, a missing language.
|
||||
let item = named(2, "en-US", "Kai left.", vec![span("kay", 0, 3)]);
|
||||
assert_eq!(refused(item), not_a_form(2, "kay", 0, 3));
|
||||
let item = named(2, "en-US", "kay left.", vec![span("kay", 0, 3)]);
|
||||
assert_eq!(refused(item), not_a_form(2, "kay", 0, 3));
|
||||
let item = named(2, "tr", "Kay left.", vec![span("kay", 0, 3)]);
|
||||
assert_eq!(refused(item), not_a_form(2, "kay", 0, 3));
|
||||
// Overlap: both spans are forms, they share bytes.
|
||||
let item = named(2, "en-US", "abab", vec![span("ab", 0, 4), span("ab", 1, 4)]);
|
||||
assert_eq!(
|
||||
refused(item),
|
||||
StoreRefusal::NameSpanOverlap {
|
||||
key: "item:t:2".into()
|
||||
}
|
||||
);
|
||||
// Adjacent spans do not overlap, whatever order they are listed in.
|
||||
let adjacent = named(
|
||||
90,
|
||||
"en-US",
|
||||
"KayKay",
|
||||
vec![span("kay", 3, 6), span("kay", 0, 3)],
|
||||
);
|
||||
assert_eq!(corpus.add(adjacent, &ents).unwrap(), AddOutcome::Added);
|
||||
|
||||
// Speakers: unknown, and unknown in a prior turn.
|
||||
let ghost = EntityId::parse("entity:ghost").unwrap();
|
||||
let unknown = StoreRefusal::UnknownEntity {
|
||||
entity: ghost.clone(),
|
||||
};
|
||||
let mut item = named(91, "en-US", "Hello there.", vec![]);
|
||||
item.speaker = ghost.clone();
|
||||
assert_eq!(refusal(corpus.add(item, &ents)), unknown);
|
||||
let mut item = named(92, "en-US", "Hello again.", vec![]);
|
||||
item.prior_turns = vec![
|
||||
Turn {
|
||||
speaker: eid("nibu"),
|
||||
text: "hi".into(),
|
||||
},
|
||||
Turn {
|
||||
speaker: ghost,
|
||||
text: "hi".into(),
|
||||
},
|
||||
];
|
||||
assert_eq!(refusal(corpus.add(item, &ents)), unknown);
|
||||
let mut item = named(93, "en-US", "Hello at last.", vec![]);
|
||||
item.prior_turns = vec![Turn {
|
||||
speaker: eid("nibu"),
|
||||
text: "hi".into(),
|
||||
}];
|
||||
assert_eq!(corpus.add(item, &ents).unwrap(), AddOutcome::Added);
|
||||
// Nothing refused was stored.
|
||||
let keys: Vec<String> = corpus.all().unwrap().into_iter().map(|i| i.id).collect();
|
||||
assert_eq!(keys, ["item:t:1", "item:t:90", "item:t:93"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corpus_spans_every_script() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
let mut ents = entities_in(d.path());
|
||||
ents.put(ent(
|
||||
"minsu",
|
||||
"ko",
|
||||
&["\u{BBFC}\u{C218}", "\u{BBFC}\u{C218}\u{AC00}"],
|
||||
))
|
||||
.unwrap();
|
||||
ents.put(ent("brokkr-tr", "tr", &["Brokkr", "Brokkr'\u{131}n"]))
|
||||
.unwrap();
|
||||
ents.put(ent(
|
||||
"somchai",
|
||||
"th",
|
||||
&["\u{E2A}\u{E21}\u{E0A}\u{E32}\u{E22}"],
|
||||
))
|
||||
.unwrap();
|
||||
let mut corpus = CorpusStore::open(&d.path().join("c.cc")).unwrap();
|
||||
|
||||
// The particle-bound Korean form: byte equality with the stored form.
|
||||
let ko = "\u{BBFC}\u{C218}\u{AC00} \u{C654}\u{B2E4}";
|
||||
let with_particle = "\u{BBFC}\u{C218}\u{AC00}".len();
|
||||
assert_eq!(with_particle, 9);
|
||||
let item = named(1, "ko", ko, vec![span("minsu", 0, with_particle as u32)]);
|
||||
assert_eq!(corpus.add(item, &ents).unwrap(), AddOutcome::Added);
|
||||
// The bare stem is a stored form too; a span cutting a syllable is not.
|
||||
let item = named(
|
||||
2,
|
||||
"ko",
|
||||
"\u{BBFC}\u{C218} \u{C654}\u{B2E4}",
|
||||
vec![span("minsu", 0, 6)],
|
||||
);
|
||||
assert_eq!(corpus.add(item, &ents).unwrap(), AddOutcome::Added);
|
||||
let item = named(3, "ko", ko, vec![span("minsu", 0, 7)]);
|
||||
assert!(matches!(
|
||||
refusal(corpus.add(item, &ents)),
|
||||
StoreRefusal::NameSpanRange { .. }
|
||||
));
|
||||
// Turkish: the apostrophe suffix.
|
||||
let tr = "Brokkr'\u{131}n geldi.";
|
||||
let len = "Brokkr'\u{131}n".len() as u32;
|
||||
let item = named(4, "tr", tr, vec![span("brokkr-tr", 0, len)]);
|
||||
assert_eq!(corpus.add(item, &ents).unwrap(), AddOutcome::Added);
|
||||
// Thai: no spaces, no segmentation, a span is a byte range.
|
||||
let th = "\u{E2A}\u{E21}\u{E0A}\u{E32}\u{E22}\u{E21}\u{E32}\u{E41}\u{E25}\u{E49}\u{E27}";
|
||||
let len = "\u{E2A}\u{E21}\u{E0A}\u{E32}\u{E22}".len() as u32;
|
||||
let item = named(5, "th", th, vec![span("somchai", 0, len)]);
|
||||
assert_eq!(corpus.add(item, &ents).unwrap(), AddOutcome::Added);
|
||||
// The same bytes under another language tag are refused.
|
||||
let item = named(6, "ko", th, vec![span("somchai", 0, len)]);
|
||||
assert!(matches!(
|
||||
refusal(corpus.add(item, &ents)),
|
||||
StoreRefusal::NameSpanNotAForm { .. }
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn packet_entities_resolve() {
|
||||
let mut w = world();
|
||||
w.entities.put(ent("kay", "en-US", &["Kay"])).unwrap();
|
||||
let mut talk = named(2, "en-US", "Kay and Hero talk.", vec![span("kay", 0, 3)]);
|
||||
talk.id = "item:nibu:2".into();
|
||||
talk.prior_turns = vec![Turn {
|
||||
speaker: eid("hero"),
|
||||
text: "hi".into(),
|
||||
}];
|
||||
w.corpus.add(talk, &w.entities).unwrap();
|
||||
|
||||
let mut packets = PacketStore::open(&w.dir.path().join("p.cc")).unwrap();
|
||||
let packet = |n: u8, item: &str, entity: &str| {
|
||||
let mut p = good();
|
||||
p.id = PacketId::parse(&format!("packet:{n:016x}")).unwrap();
|
||||
p.provenance.corpus_item = CorpusItemId::parse(item).unwrap();
|
||||
p.referents[0].entity = Some(eid(entity));
|
||||
p
|
||||
};
|
||||
let mut admit = |p: Packet| packets.admit(p, &w.inventory, &w.corpus, &w.entities);
|
||||
|
||||
// An entity the store lacks.
|
||||
let unknown = StoreRefusal::UnknownEntity {
|
||||
entity: eid("ghost"),
|
||||
};
|
||||
assert_eq!(refusal(admit(packet(1, "item:nibu:1", "ghost"))), unknown);
|
||||
assert_eq!(refusal(admit(packet(2, "item:nibu:2", "ghost"))), unknown);
|
||||
// A stored entity that is not on the item.
|
||||
let off_item = |item: &str, entity: &str| StoreRefusal::EntityNotOnItem {
|
||||
entity: eid(entity),
|
||||
item: item.into(),
|
||||
};
|
||||
assert_eq!(
|
||||
refusal(admit(packet(3, "item:nibu:2", "brokkr"))),
|
||||
off_item("item:nibu:2", "brokkr")
|
||||
);
|
||||
// nibu:1 names nibu (and is spoken by nibu); kay is only on nibu:2.
|
||||
assert_eq!(
|
||||
refusal(admit(packet(4, "item:nibu:1", "kay"))),
|
||||
off_item("item:nibu:1", "kay")
|
||||
);
|
||||
// On the item: a name entity, the speaker, a prior-turn speaker.
|
||||
admit(packet(5, "item:nibu:2", "kay")).unwrap();
|
||||
admit(packet(6, "item:nibu:2", "nibu")).unwrap();
|
||||
admit(packet(7, "item:nibu:2", "hero")).unwrap();
|
||||
// No packet escapes the item check: a stored entity that no item names is
|
||||
// refused on every item, and a packet cannot omit its item.
|
||||
assert_eq!(
|
||||
refusal(admit(packet(8, "item:nibu:1", "brokkr"))),
|
||||
off_item("item:nibu:1", "brokkr")
|
||||
);
|
||||
// A referent with no entity needs none; only a packet that cites checks.
|
||||
let mut anonymous = packet(9, "item:nibu:2", "kay");
|
||||
anonymous.referents[0].entity = None;
|
||||
admit(anonymous).unwrap();
|
||||
// Refused packets were not stored.
|
||||
let stored: Vec<String> = PacketStore::open(&w.dir.path().join("p.cc"))
|
||||
.unwrap()
|
||||
.all()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|p| p.id.to_string())
|
||||
.collect();
|
||||
assert_eq!(stored.len(), 4, "{stored:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn packet_schema_v2() {
|
||||
assert_eq!(SCHEMA_VERSION, 3);
|
||||
let mut p = good();
|
||||
p.schema_version = 2;
|
||||
let Err(refused) = validate(&p, &inventory()) else {
|
||||
panic!("a v1 packet must not validate")
|
||||
};
|
||||
assert_eq!(refused, vec![Refusal::V1SchemaVersion { found: 2 }]);
|
||||
|
||||
let published: std::collections::BTreeMap<_, _> =
|
||||
weksa::schema::published().into_iter().collect();
|
||||
let packet: serde_json::Value =
|
||||
serde_json::from_str(&published["weksa.interlingua_packet.schema.json"]).unwrap();
|
||||
let referent = &packet["$defs"]["Referent"]["properties"];
|
||||
assert!(referent.get("entity").is_some(), "{referent}");
|
||||
assert!(referent.get("name").is_none(), "{referent}");
|
||||
let defs = packet["$defs"].as_object().unwrap();
|
||||
assert!(!defs.keys().any(|k| k.ends_with("Name")), "{defs:?}");
|
||||
assert_eq!(
|
||||
packet["$defs"]["EntityId"]["pattern"],
|
||||
"^entity:[a-z0-9]([a-z0-9-]{0,46}[a-z0-9])?$"
|
||||
);
|
||||
let item: serde_json::Value =
|
||||
serde_json::from_str(&published["weksa.corpus_item.schema.json"]).unwrap();
|
||||
assert!(item["properties"].get("names").is_some());
|
||||
assert!(published.contains_key("weksa.entity.schema.json"));
|
||||
}
|
||||
348
crates/weksa/tests/entity_door.rs
Normal file
348
crates/weksa/tests/entity_door.rs
Normal file
|
|
@ -0,0 +1,348 @@
|
|||
//! Behavioural pins from Soul's entity-store s1 probes: the entity door does not
|
||||
//! trust the writer (ruling entity-door-bounds).
|
||||
mod common;
|
||||
use common::*;
|
||||
use unicode_normalization::UnicodeNormalization;
|
||||
use weksa::corpus::{CorpusItem, NameSpan};
|
||||
use weksa::entity::{Entity, EntityOrigin, LanguageForms};
|
||||
use weksa::store::EntityBound;
|
||||
use weksa::{EntityStore, StoreError, StoreRefusal};
|
||||
|
||||
fn refusal(r: Result<impl std::fmt::Debug, StoreError>) -> StoreRefusal {
|
||||
match r {
|
||||
Err(StoreError::Refused(r)) => r,
|
||||
other => panic!("expected a store refusal, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn item(
|
||||
n: impl std::fmt::Display,
|
||||
lang: &str,
|
||||
text: &str,
|
||||
names: Vec<NameSpan>,
|
||||
group: &str,
|
||||
) -> CorpusItem {
|
||||
CorpusItem {
|
||||
language: lang.into(),
|
||||
names,
|
||||
..item_in(&format!("item:probe:{n}"), text, group, cast())
|
||||
}
|
||||
}
|
||||
|
||||
fn span(slug: &str, start: usize, end: usize) -> NameSpan {
|
||||
NameSpan {
|
||||
entity: eid(slug),
|
||||
start: start as u32,
|
||||
end: end as u32,
|
||||
}
|
||||
}
|
||||
|
||||
fn bounds(entity: &Entity, bound: EntityBound) -> StoreRefusal {
|
||||
StoreRefusal::EntityBounds {
|
||||
id: entity.id.clone(),
|
||||
bound,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn span_never_splits_a_grapheme_cluster() {
|
||||
let mut w = world();
|
||||
let g = group_in_bucket(10);
|
||||
let splits = |n: u32, start: usize, end: usize| StoreRefusal::NameSpanSplitsCluster {
|
||||
key: format!("item:probe:{n}"),
|
||||
start: start as u32,
|
||||
end: end as u32,
|
||||
};
|
||||
|
||||
// Devanagari: the word is a base, a spacing vowel sign and a base; the form
|
||||
// "र" ends inside the cluster of the first two.
|
||||
w.entities.put(ent("ra", "hi", &["र"])).unwrap();
|
||||
let text = "राम आया";
|
||||
assert_eq!(
|
||||
refusal(
|
||||
w.corpus
|
||||
.add(item(1, "hi", text, vec![span("ra", 0, 3)], &g), &w.entities)
|
||||
),
|
||||
splits(1, 0, 3)
|
||||
);
|
||||
// The whole words are legal names, the second starting after a space.
|
||||
w.entities.put(ent("ram", "hi", &["राम", "आया"])).unwrap();
|
||||
let both = vec![span("ram", 0, 9), span("ram", 10, text.len())];
|
||||
w.corpus
|
||||
.add(item(2, "hi", text, both, &g), &w.entities)
|
||||
.unwrap();
|
||||
|
||||
// A start inside a cluster is refused too.
|
||||
w.entities.put(ent("aa", "hi", &["ा"])).unwrap();
|
||||
assert_eq!(
|
||||
refusal(w.corpus.add(
|
||||
item(3, "hi", "राम x", vec![span("aa", 3, 6)], &g),
|
||||
&w.entities
|
||||
)),
|
||||
splits(3, 3, 6)
|
||||
);
|
||||
|
||||
// Latin base + combining mark with no precomposed form, so NFC keeps two
|
||||
// code points: the form "q" leaves the accent outside.
|
||||
let q = "q\u{301} arrive.";
|
||||
w.entities.put(ent("q", "fr", &["q"])).unwrap();
|
||||
assert_eq!(
|
||||
refusal(
|
||||
w.corpus
|
||||
.add(item(4, "fr", q, vec![span("q", 0, 1)], &g), &w.entities)
|
||||
),
|
||||
splits(4, 0, 1)
|
||||
);
|
||||
w.entities.put(ent("qa", "fr", &["q\u{301}"])).unwrap();
|
||||
w.corpus
|
||||
.add(item(5, "fr", q, vec![span("qa", 0, 3)], &g), &w.entities)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_and_forms_are_stored_nfc() {
|
||||
let mut w = world();
|
||||
let g = group_in_bucket(10);
|
||||
let not_nfc = |key: &str| StoreRefusal::NotNfc { key: key.into() };
|
||||
// Decomposed Hangul (conjoining jamo) text is refused, not segmented.
|
||||
let nfd = "\u{1106}\u{1175}\u{11AB}\u{1109}\u{116E} \u{110B}\u{116A}\u{11BB}\u{1103}\u{1161}";
|
||||
assert!(!unicode_normalization::is_nfc(nfd));
|
||||
assert_eq!(
|
||||
refusal(w.corpus.add(item(1, "ko", nfd, vec![], &g), &w.entities)),
|
||||
not_nfc("item:probe:1")
|
||||
);
|
||||
// The same words composed are admitted, and a span on a syllable pair is exact.
|
||||
let nfc: String = nfd.nfc().collect();
|
||||
let name: String = "\u{1106}\u{1175}\u{11AB}\u{1109}\u{116E}".nfc().collect();
|
||||
w.entities.put(ent("minsu", "ko", &[&name])).unwrap();
|
||||
w.corpus
|
||||
.add(
|
||||
item(2, "ko", &nfc, vec![span("minsu", 0, name.len())], &g),
|
||||
&w.entities,
|
||||
)
|
||||
.unwrap();
|
||||
// Latin NFD text, NFD forms and NFD prior turns are refused.
|
||||
assert_eq!(
|
||||
refusal(w.corpus.add(
|
||||
item(3, "fr", "Rene\u{301} arrive.", vec![], &g),
|
||||
&w.entities
|
||||
)),
|
||||
not_nfc("item:probe:3")
|
||||
);
|
||||
assert_eq!(
|
||||
refusal(w.entities.put(ent("rene", "fr", &["Rene\u{301}"]))),
|
||||
not_nfc("entity:rene")
|
||||
);
|
||||
let mut turns = item(4, "fr", "Salut.", vec![], &g);
|
||||
turns.prior_turns = vec![weksa::Turn {
|
||||
speaker: eid("nibu"),
|
||||
text: "Rene\u{301}?".into(),
|
||||
}];
|
||||
assert_eq!(
|
||||
refusal(w.corpus.add(turns, &w.entities)),
|
||||
not_nfc("item:probe:4")
|
||||
);
|
||||
// NFC text and an NFC form of the same name agree byte for byte.
|
||||
w.entities.put(ent("rene", "fr", &["Ren\u{e9}"])).unwrap();
|
||||
w.corpus
|
||||
.add(
|
||||
item(5, "fr", "Ren\u{e9} part.", vec![span("rene", 0, 5)], &g),
|
||||
&w.entities,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entity_capacity_is_bounded() {
|
||||
let mut w = world();
|
||||
let lf = |language: &str, forms: Vec<String>| LanguageForms {
|
||||
language: language.into(),
|
||||
forms,
|
||||
};
|
||||
let one = |f: &str| vec![f.to_string()];
|
||||
let make = |forms: Vec<LanguageForms>| Entity {
|
||||
id: eid("big"),
|
||||
forms,
|
||||
origin: EntityOrigin::Live,
|
||||
};
|
||||
|
||||
// A whole line is not a name: 7 words.
|
||||
let line = "I will never leave you, not ever.";
|
||||
let e = make(vec![lf("en-US", one(line))]);
|
||||
assert_eq!(
|
||||
refusal(w.entities.put(e.clone())),
|
||||
bounds(&e, EntityBound::FormWords)
|
||||
);
|
||||
// Four words is the edge; five is over. 64 bytes is the edge; 65 is over.
|
||||
w.entities
|
||||
.put(Entity {
|
||||
id: eid("four"),
|
||||
..make(vec![lf("en-US", one("Mary of the Isles"))])
|
||||
})
|
||||
.unwrap();
|
||||
let five = make(vec![lf("en-US", one("Mary of the Isles too"))]);
|
||||
assert_eq!(
|
||||
refusal(w.entities.put(five.clone())),
|
||||
bounds(&five, EntityBound::FormWords)
|
||||
);
|
||||
let wide = make(vec![lf("en-US", one(&"a".repeat(65)))]);
|
||||
assert_eq!(
|
||||
refusal(w.entities.put(wide.clone())),
|
||||
bounds(&wide, EntityBound::FormBytes)
|
||||
);
|
||||
w.entities
|
||||
.put(Entity {
|
||||
id: eid("edge"),
|
||||
..make(vec![lf("en-US", one(&"a".repeat(64)))])
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
// 8 forms per language is the edge.
|
||||
let forms = |n: usize| (0..n).map(|i| format!("form{i}")).collect::<Vec<_>>();
|
||||
w.entities
|
||||
.put(Entity {
|
||||
id: eid("eight"),
|
||||
..make(vec![lf("en-US", forms(8))])
|
||||
})
|
||||
.unwrap();
|
||||
let nine = make(vec![lf("en-US", forms(9))]);
|
||||
assert_eq!(
|
||||
refusal(w.entities.put(nine.clone())),
|
||||
bounds(&nine, EntityBound::Forms)
|
||||
);
|
||||
|
||||
// 8 languages is the edge.
|
||||
let langs = |n: usize| {
|
||||
(0..n)
|
||||
.map(|i| lf(&format!("x-l{i}"), one("Form")))
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
w.entities
|
||||
.put(Entity {
|
||||
id: eid("langs"),
|
||||
..make(langs(8))
|
||||
})
|
||||
.unwrap();
|
||||
let many = make(langs(9));
|
||||
assert_eq!(
|
||||
refusal(w.entities.put(many.clone())),
|
||||
bounds(&many, EntityBound::Languages)
|
||||
);
|
||||
|
||||
// Soul's probe: 200 junk languages of 5000 long forms each.
|
||||
let long: Vec<String> = (0..5000)
|
||||
.map(|i| format!("form number {i} of a long free text channel"))
|
||||
.collect();
|
||||
let probe = make(
|
||||
(0..200)
|
||||
.map(|l| lf(&format!("x-{l}\n\u{7}"), long.clone()))
|
||||
.collect(),
|
||||
);
|
||||
assert_eq!(
|
||||
refusal(w.entities.put(probe.clone())),
|
||||
bounds(&probe, EntityBound::Languages)
|
||||
);
|
||||
assert!(w.entities.get(&eid("big")).unwrap().is_none());
|
||||
|
||||
// Language tags are BCP 47 in form.
|
||||
for bad in [
|
||||
"",
|
||||
"e",
|
||||
"en_US",
|
||||
"x-0\n\u{7}",
|
||||
"-en",
|
||||
"en-",
|
||||
"en--US",
|
||||
"en-US-abcdefghi",
|
||||
"1n",
|
||||
"\u{65e5}\u{672c}",
|
||||
"en-US-aaaaaaaa-bbbbbbbb-cccccccc-dddddddd-eeeeeeee",
|
||||
"en-aaaaaaaa-aaaaaaaa-aaaaaaaa-bbbbbb",
|
||||
] {
|
||||
let e = make(vec![lf(bad, one("Form"))]);
|
||||
assert_eq!(
|
||||
refusal(w.entities.put(e.clone())),
|
||||
bounds(&e, EntityBound::LanguageTag),
|
||||
"{bad:?}"
|
||||
);
|
||||
}
|
||||
for (i, ok) in [
|
||||
"en",
|
||||
"en-US",
|
||||
"zh-Hant-TW",
|
||||
"x-klingon",
|
||||
"de-CH-1996",
|
||||
"en-aaaaaaaa-aaaaaaaa-aaaaaaaa-bbbbb",
|
||||
]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
w.entities
|
||||
.put(Entity {
|
||||
id: eid(&format!("tag-{i}")),
|
||||
..make(vec![lf(ok, one("Form"))])
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
assert!(w.entities.get(&eid("big")).unwrap().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn item_language_is_a_tag() {
|
||||
let mut w = world();
|
||||
let g = group_in_bucket(10);
|
||||
for (n, bad) in ["", "en_US", "x-0\n\u{7}"].into_iter().enumerate() {
|
||||
assert!(matches!(
|
||||
refusal(
|
||||
w.corpus
|
||||
.add(item(n + 1, bad, "Nibu left.", vec![], &g), &w.entities)
|
||||
),
|
||||
StoreRefusal::Malformed { .. }
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entities_are_append_only_and_replay_needs_them() {
|
||||
let mut w = world();
|
||||
let base = ent("z", "en-US", &["Zed"]);
|
||||
w.entities.put(base.clone()).unwrap();
|
||||
let not_append = StoreRefusal::EntityNotAppendOnly { id: eid("z") };
|
||||
|
||||
let mut prepend = base.clone();
|
||||
prepend.forms.insert(
|
||||
0,
|
||||
LanguageForms {
|
||||
language: "fr".into(),
|
||||
forms: vec!["Z\u{e8}de".into()],
|
||||
},
|
||||
);
|
||||
assert_eq!(refusal(w.entities.put(prepend)), not_append);
|
||||
let mut persona = base.clone();
|
||||
persona.origin = EntityOrigin::Cast {
|
||||
persona: "someone-else".into(),
|
||||
};
|
||||
assert_eq!(refusal(w.entities.put(persona)), not_append);
|
||||
let mut duplicate = base.clone();
|
||||
duplicate.forms.push(LanguageForms {
|
||||
language: "en-US".into(),
|
||||
forms: vec!["Zeddy".into()],
|
||||
});
|
||||
assert!(matches!(
|
||||
refusal(w.entities.put(duplicate)),
|
||||
StoreRefusal::Malformed { .. }
|
||||
));
|
||||
|
||||
// A corpus replay against an entity store that lacks the speaker is refused.
|
||||
let empty = EntityStore::open(&w.dir.path().join("empty.cc")).unwrap();
|
||||
let again = CorpusItem {
|
||||
id: "item:nibu:2".into(),
|
||||
..item(1, "en-US", "Nibu says hello.", vec![], &group_in_bucket(10))
|
||||
};
|
||||
assert_eq!(
|
||||
refusal(w.corpus.add(again, &empty)),
|
||||
StoreRefusal::UnknownEntity {
|
||||
entity: eid("nibu")
|
||||
}
|
||||
);
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
mod common;
|
||||
use common::*;
|
||||
use weksa::packet::{EncoderId, Grade, NodeId, PacketId, ProperName, Referent};
|
||||
use weksa::packet::{EncoderId, Grade, NodeId, PacketId, Referent};
|
||||
use weksa::{Family, InvId, OptionLabel, OptionSet};
|
||||
|
||||
/// Soul's probe: a whole corpus line, and the snake_case form that would
|
||||
|
|
@ -77,9 +77,6 @@ fn a_malformed_value_does_not_deserialize() {
|
|||
referent["id"] = SNAKE.into();
|
||||
assert!(serde_json::from_value::<Referent>(referent.clone()).is_err());
|
||||
referent["id"] = "n1".into();
|
||||
referent["name"] = LINE.into();
|
||||
assert!(serde_json::from_value::<Referent>(referent.clone()).is_err());
|
||||
referent["name"] = "Nibu".into();
|
||||
referent["concept"] = "concept:01".into();
|
||||
assert!(serde_json::from_value::<Referent>(referent).is_err());
|
||||
assert!(serde_json::from_str::<Grade>("1001").is_err());
|
||||
|
|
@ -87,64 +84,6 @@ fn a_malformed_value_does_not_deserialize() {
|
|||
assert!(serde_json::from_str::<OptionSet>("[]").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn proper_name_bounds() {
|
||||
for ok in [
|
||||
"Nibu",
|
||||
"Brokkr Ivaldi",
|
||||
"O'Neil",
|
||||
"O\u{2019}Neil",
|
||||
"Jean-Luc",
|
||||
"A B C D",
|
||||
] {
|
||||
assert_eq!(ProperName::parse(ok).unwrap().as_str(), ok);
|
||||
}
|
||||
// A caseless script is a name; so is a capital beyond ASCII.
|
||||
assert!(ProperName::parse("\u{5c71}\u{7530}").is_ok());
|
||||
assert!(ProperName::parse("\u{c9}owyn").is_ok());
|
||||
for bad in [
|
||||
LINE,
|
||||
"A B C D E",
|
||||
"Nibu.",
|
||||
"Nibu, Ivaldi",
|
||||
"Nibu Ivaldi",
|
||||
" Nibu",
|
||||
"Nibu ",
|
||||
"",
|
||||
"Nibu2",
|
||||
"'Nibu",
|
||||
"-Nibu",
|
||||
"Nibu-",
|
||||
"Nibu'",
|
||||
"Nibu_Ivaldi",
|
||||
] {
|
||||
assert!(ProperName::parse(bad).is_err(), "{bad:?}");
|
||||
}
|
||||
// 24 chars a token, 4 tokens and 64 bytes are the bounds.
|
||||
let token24 = format!("A{}", "b".repeat(23));
|
||||
assert!(ProperName::parse(&token24).is_ok());
|
||||
assert!(ProperName::parse(&format!("{token24}b")).is_err());
|
||||
let sized = |lens: [usize; 4]| {
|
||||
lens.iter()
|
||||
.map(|n| format!("A{}", "b".repeat(n - 1)))
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
};
|
||||
let at_limit = sized([15, 15, 15, 16]);
|
||||
assert_eq!(at_limit.len(), 64);
|
||||
assert!(ProperName::parse(&at_limit).is_ok());
|
||||
let over = sized([15, 15, 15, 17]);
|
||||
assert_eq!(over.len(), 65);
|
||||
assert!(ProperName::parse(&over).is_err());
|
||||
// The byte bound counts bytes, not characters.
|
||||
let wide = format!(
|
||||
"{0} {0} {0}",
|
||||
"\u{e9}".repeat(11).replace('\u{e9}', "\u{c9}")
|
||||
);
|
||||
assert!(wide.chars().count() < 64 && wide.len() > 64);
|
||||
assert!(ProperName::parse(&wide).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inv_id_canonical() {
|
||||
for bad in [
|
||||
|
|
@ -211,38 +150,3 @@ fn grade_bound_is_the_type() {
|
|||
assert_eq!(grade["minimum"], 0, "{grade}");
|
||||
assert_eq!(schema["$defs"]["NodeId"]["pattern"], "^n[1-9][0-9]{0,3}$");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn proper_name_bound_in_chars() {
|
||||
// The token bound counts characters: 24 two-byte letters are 48 bytes and parse.
|
||||
let cyrillic = |n: usize| "\u{411}".repeat(n);
|
||||
assert_eq!(cyrillic(24).len(), 48);
|
||||
assert!(ProperName::parse(&cyrillic(24)).is_ok());
|
||||
assert!(ProperName::parse(&cyrillic(25)).is_err());
|
||||
// The 64-byte bound still counts bytes: two 48-byte tokens are 97 bytes.
|
||||
assert!(ProperName::parse(&format!("{0} {0}", cyrillic(24))).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn proper_name_is_form_only() {
|
||||
// A name is bounded text: no case rule, so lowercase, hyphenated and
|
||||
// caseless-script tokens parse.
|
||||
for ok in [
|
||||
"nibu",
|
||||
"Nibu ivaldi",
|
||||
"take-your-hand-off-that",
|
||||
"Take-your-hand-off-that",
|
||||
"\u{3131}-control-unless-you",
|
||||
"\u{5c71}\u{7530}",
|
||||
] {
|
||||
assert_eq!(ProperName::parse(ok).unwrap().as_str(), ok);
|
||||
}
|
||||
// The bounds still hold: 5 tokens, 65 bytes, a sentence's punctuation.
|
||||
assert!(ProperName::parse("a b c d e").is_err());
|
||||
assert!(ProperName::parse(&"a".repeat(25)).is_err());
|
||||
let token = |n: usize| "a".repeat(n);
|
||||
let over = [token(15), token(15), token(15), token(17)].join(" ");
|
||||
assert_eq!(over.len(), 65);
|
||||
assert!(ProperName::parse(&over).is_err());
|
||||
assert!(ProperName::parse("take your hand, now.").is_err());
|
||||
}
|
||||
|
|
|
|||
38
crates/weksa/tests/seeds.rs
Normal file
38
crates/weksa/tests/seeds.rs
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
//! The committed seed stores replay through today's doors, so a rule that
|
||||
//! tightens cannot leave a seed the stores would now refuse.
|
||||
mod common;
|
||||
use std::path::Path;
|
||||
use weksa::{CorpusStore, EntityStore};
|
||||
|
||||
#[test]
|
||||
fn committed_seeds_replay_through_the_doors() {
|
||||
let data = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data");
|
||||
let seeded_entities = EntityStore::open(&data.join("entity/entity.cc")).unwrap();
|
||||
let seeded_corpus = CorpusStore::open(&data.join("corpus/corpus.cc")).unwrap();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut entities = EntityStore::open(&dir.path().join("e.cc")).unwrap();
|
||||
let all = seeded_entities.all().unwrap();
|
||||
assert!(!all.is_empty());
|
||||
for e in all {
|
||||
entities.put(e).unwrap();
|
||||
}
|
||||
let mut corpus = CorpusStore::open(&dir.path().join("c.cc")).unwrap();
|
||||
let items = seeded_corpus.all().unwrap();
|
||||
assert!(!items.is_empty());
|
||||
for item in items {
|
||||
let withdrawn = item.withdrawn;
|
||||
let id = item.id.clone();
|
||||
corpus
|
||||
.add(
|
||||
weksa::CorpusItem {
|
||||
withdrawn: false,
|
||||
..item
|
||||
},
|
||||
&entities,
|
||||
)
|
||||
.unwrap();
|
||||
if withdrawn {
|
||||
corpus.withdraw(&id).unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -7,7 +7,8 @@ fn item(group: &str, source: Source) -> CorpusItem {
|
|||
id: "item:t:1".into(),
|
||||
text: "t".into(),
|
||||
language: "en-US".into(),
|
||||
speaker: "s".into(),
|
||||
speaker: weksa::EntityId::parse("entity:s").unwrap(),
|
||||
names: vec![],
|
||||
group: group.into(),
|
||||
prior_turns: vec![],
|
||||
source,
|
||||
|
|
@ -54,7 +55,7 @@ fn p1_split_deterministic() {
|
|||
}
|
||||
// The split reads the group and nothing else: speaker, text and id do not move it.
|
||||
let mut other = item("scene-9", cast());
|
||||
other.speaker = "someone".into();
|
||||
other.speaker = weksa::EntityId::parse("entity:someone").unwrap();
|
||||
other.text = "different".into();
|
||||
other.id = "item:other:7".into();
|
||||
assert_eq!(SplitScheme::V1.split(&other), a);
|
||||
|
|
|
|||
|
|
@ -4,14 +4,14 @@ use tempfile::TempDir;
|
|||
use weksa::corpus::{CorpusItem, ModelFamily, Source, Turn};
|
||||
use weksa::inventory::{Change, seed};
|
||||
use weksa::{
|
||||
AddOutcome, CorpusStore, Family, InventoryEntry, InventoryRevision, InventoryStore,
|
||||
PacketStore, Refusal, Slot, SplitScheme, StoreError, StoreRefusal, ValueKind,
|
||||
AddOutcome, CorpusStore, EntityStore, Family, InventoryEntry, InventoryRevision,
|
||||
InventoryStore, PacketStore, Refusal, Slot, SplitScheme, StoreError, StoreRefusal, ValueKind,
|
||||
};
|
||||
|
||||
fn item(key: &str, text: &str) -> CorpusItem {
|
||||
CorpusItem {
|
||||
prior_turns: vec![Turn {
|
||||
speaker: "player".into(),
|
||||
speaker: eid("player"),
|
||||
text: "hello".into(),
|
||||
}],
|
||||
..item_in(key, text, "scene-1", cast())
|
||||
|
|
@ -25,6 +25,10 @@ fn refusal(r: Result<impl std::fmt::Debug, StoreError>) -> StoreRefusal {
|
|||
}
|
||||
}
|
||||
|
||||
fn ents(d: &TempDir) -> EntityStore {
|
||||
entities_in(d.path())
|
||||
}
|
||||
|
||||
fn dir() -> TempDir {
|
||||
tempfile::tempdir().unwrap()
|
||||
}
|
||||
|
|
@ -46,13 +50,17 @@ fn s1_item_immutable() {
|
|||
let path = d.path().join("c.cc");
|
||||
let mut store = CorpusStore::open(&path).unwrap();
|
||||
assert_eq!(
|
||||
store.add(item("item:nibu:1", "Mind the hatch.")).unwrap(),
|
||||
store
|
||||
.add(item("item:nibu:1", "Mind the hatch."), &ents(&d))
|
||||
.unwrap(),
|
||||
AddOutcome::Added
|
||||
);
|
||||
// An equal item is idempotent, even from a reopened store.
|
||||
let mut store = CorpusStore::open(&path).unwrap();
|
||||
assert_eq!(
|
||||
store.add(item("item:nibu:1", "Mind the hatch.")).unwrap(),
|
||||
store
|
||||
.add(item("item:nibu:1", "Mind the hatch."), &ents(&d))
|
||||
.unwrap(),
|
||||
AddOutcome::Unchanged
|
||||
);
|
||||
let refused = || StoreRefusal::S1ItemImmutable {
|
||||
|
|
@ -71,7 +79,7 @@ fn s1_item_immutable() {
|
|||
v.language = "en-GB".into();
|
||||
variants.push(v);
|
||||
let mut v = base.clone();
|
||||
v.speaker = "brokkr".into();
|
||||
v.speaker = eid("brokkr");
|
||||
variants.push(v);
|
||||
let mut v = base.clone();
|
||||
v.group = "scene-2".into();
|
||||
|
|
@ -86,12 +94,12 @@ fn s1_item_immutable() {
|
|||
v.supersedes = Some("item:nibu:9".into());
|
||||
variants.push(v);
|
||||
for v in variants {
|
||||
assert_eq!(refusal(store.add(v.clone())), refused(), "{v:?}");
|
||||
assert_eq!(refusal(store.add(v.clone(), &ents(&d))), refused(), "{v:?}");
|
||||
}
|
||||
assert_eq!(store.get("item:nibu:1").unwrap(), Some(base.clone()));
|
||||
// A withdrawn item is a different item: re-adding it live is refused too.
|
||||
store.withdraw("item:nibu:1").unwrap();
|
||||
assert_eq!(refusal(store.add(base)), refused());
|
||||
assert_eq!(refusal(store.add(base, &ents(&d))), refused());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -115,7 +123,7 @@ fn corpus_keys_are_canonical() {
|
|||
];
|
||||
for (i, key) in not_canonical.into_iter().enumerate() {
|
||||
assert_eq!(
|
||||
refusal(store.add(item(key, &format!("line {i}")))),
|
||||
refusal(store.add(item(key, &format!("line {i}")), &ents(&d))),
|
||||
malformed(key, "item key is not item:<slug>:<n>"),
|
||||
"{key}"
|
||||
);
|
||||
|
|
@ -126,29 +134,24 @@ fn corpus_keys_are_canonical() {
|
|||
let canonical = ["item:a:1", "item:a-b:10", "item:7:2", "item:ab:9", &longest];
|
||||
for (i, key) in canonical.into_iter().enumerate() {
|
||||
assert_eq!(
|
||||
store.add(item(key, &format!("fine {i}"))).unwrap(),
|
||||
store
|
||||
.add(item(key, &format!("fine {i}")), &ents(&d))
|
||||
.unwrap(),
|
||||
AddOutcome::Added,
|
||||
"{key}"
|
||||
);
|
||||
}
|
||||
// An alias of a stored key resolves to nothing.
|
||||
let w = world();
|
||||
let mut packets = PacketStore::open(&w.dir.path().join("p.cc")).unwrap();
|
||||
let mut p = good();
|
||||
p.provenance.corpus_item = Some("item:nibu:01".into());
|
||||
assert_eq!(
|
||||
refusal(packets.admit(p, &w.inventory, &w.corpus)),
|
||||
StoreRefusal::UnknownCorpusItem {
|
||||
key: "item:nibu:01".into()
|
||||
}
|
||||
);
|
||||
// An alias of a stored key is not a corpus item id at all.
|
||||
assert!(weksa::CorpusItemId::parse("item:nibu:01").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn s6_text_held_by_another_key() {
|
||||
let d = dir();
|
||||
let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap();
|
||||
store.add(item("item:nibu:1", "Mind the hatch.")).unwrap();
|
||||
store
|
||||
.add(item("item:nibu:1", "Mind the hatch."), &ents(&d))
|
||||
.unwrap();
|
||||
let elsewhere = |key: &str, text: &str| CorpusItem {
|
||||
group: "scene-2".into(),
|
||||
..item(key, text)
|
||||
|
|
@ -159,11 +162,11 @@ fn s6_text_held_by_another_key() {
|
|||
};
|
||||
// The same text in another group, and in the same group, is refused.
|
||||
assert_eq!(
|
||||
refusal(store.add(elsewhere("item:nibu:2", "Mind the hatch."))),
|
||||
refusal(store.add(elsewhere("item:nibu:2", "Mind the hatch."), &ents(&d))),
|
||||
held("item:nibu:2", "item:nibu:1")
|
||||
);
|
||||
assert_eq!(
|
||||
refusal(store.add(item("item:nibu:2", "Mind the hatch."))),
|
||||
refusal(store.add(item("item:nibu:2", "Mind the hatch."), &ents(&d))),
|
||||
held("item:nibu:2", "item:nibu:1")
|
||||
);
|
||||
// Case, spacing and punctuation variants are the same text.
|
||||
|
|
@ -174,38 +177,51 @@ fn s6_text_held_by_another_key() {
|
|||
"Mind the hatch.\n",
|
||||
] {
|
||||
assert_eq!(
|
||||
refusal(store.add(elsewhere("item:nibu:2", text))),
|
||||
refusal(store.add(elsewhere("item:nibu:2", text), &ents(&d))),
|
||||
held("item:nibu:2", "item:nibu:1"),
|
||||
"{text:?}"
|
||||
);
|
||||
}
|
||||
store
|
||||
.add(item("item:nibu:3", "\u{c4}rger \u{fc}ber \u{d6}l"))
|
||||
.add(
|
||||
item("item:nibu:3", "\u{c4}rger \u{fc}ber \u{d6}l"),
|
||||
&ents(&d),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
refusal(store.add(elsewhere("item:nibu:4", "\u{e4}rger \u{dc}BER \u{f6}l"))),
|
||||
refusal(store.add(
|
||||
elsewhere("item:nibu:4", "\u{e4}rger \u{dc}BER \u{f6}l"),
|
||||
&ents(&d)
|
||||
)),
|
||||
held("item:nibu:4", "item:nibu:3")
|
||||
);
|
||||
// Text that differs in a letter or digit is a different text.
|
||||
store.add(item("item:nibu:5", "Mind the hatch 2")).unwrap();
|
||||
store.add(item("item:nibu:6", "Mind the hatches.")).unwrap();
|
||||
store
|
||||
.add(item("item:nibu:5", "Mind the hatch 2"), &ents(&d))
|
||||
.unwrap();
|
||||
store
|
||||
.add(item("item:nibu:6", "Mind the hatches."), &ents(&d))
|
||||
.unwrap();
|
||||
// A withdrawn holder still holds.
|
||||
store.withdraw("item:nibu:1").unwrap();
|
||||
assert_eq!(
|
||||
refusal(store.add(elsewhere("item:nibu:2", "mind the hatch"))),
|
||||
refusal(store.add(elsewhere("item:nibu:2", "mind the hatch"), &ents(&d))),
|
||||
held("item:nibu:2", "item:nibu:1")
|
||||
);
|
||||
// Superseding a different key does not excuse a copy of another item.
|
||||
let mut copy = item("item:nibu:8", "Mind the hatches!");
|
||||
copy.supersedes = Some("item:nibu:5".into());
|
||||
assert_eq!(refusal(store.add(copy)), held("item:nibu:8", "item:nibu:6"));
|
||||
assert_eq!(
|
||||
refusal(store.add(copy, &ents(&d))),
|
||||
held("item:nibu:8", "item:nibu:6")
|
||||
);
|
||||
// A superseder of the holder, in the holder's group, is Added.
|
||||
let mut fix = item("item:nibu:2", "mind the hatch");
|
||||
fix.supersedes = Some("item:nibu:1".into());
|
||||
assert_eq!(store.add(fix).unwrap(), AddOutcome::Added);
|
||||
assert_eq!(store.add(fix, &ents(&d)).unwrap(), AddOutcome::Added);
|
||||
// The superseded holder's text stays held against a third key.
|
||||
assert_eq!(
|
||||
refusal(store.add(elsewhere("item:nibu:7", "Mind the hatch."))),
|
||||
refusal(store.add(elsewhere("item:nibu:7", "Mind the hatch."), &ents(&d))),
|
||||
held("item:nibu:7", "item:nibu:1")
|
||||
);
|
||||
}
|
||||
|
|
@ -215,32 +231,32 @@ fn corpus_admission_rules() {
|
|||
let d = dir();
|
||||
let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap();
|
||||
assert_eq!(
|
||||
refusal(store.add(item("nibu:1", "x"))),
|
||||
refusal(store.add(item("nibu:1", "x"), &ents(&d))),
|
||||
malformed("nibu:1", "item key is not item:<slug>:<n>")
|
||||
);
|
||||
assert_eq!(
|
||||
refusal(store.add(item("item:nibu:x", "x"))),
|
||||
refusal(store.add(item("item:nibu:x", "x"), &ents(&d))),
|
||||
malformed("item:nibu:x", "item key is not item:<slug>:<n>")
|
||||
);
|
||||
let mut born_withdrawn = item("item:nibu:1", "x");
|
||||
born_withdrawn.withdrawn = true;
|
||||
assert_eq!(
|
||||
refusal(store.add(born_withdrawn)),
|
||||
refusal(store.add(born_withdrawn, &ents(&d))),
|
||||
malformed("item:nibu:1", "an item is admitted live, then withdrawn")
|
||||
);
|
||||
let mut orphan = item("item:nibu:1", "x");
|
||||
orphan.supersedes = Some("item:nibu:9".into());
|
||||
assert_eq!(
|
||||
refusal(store.add(orphan)),
|
||||
refusal(store.add(orphan, &ents(&d))),
|
||||
malformed("item:nibu:1", "supersedes an item that does not exist")
|
||||
);
|
||||
// Nothing was written by the refusals.
|
||||
assert!(store.all().unwrap().is_empty());
|
||||
// A correction is a new item that supersedes the old one.
|
||||
store.add(item("item:nibu:1", "old")).unwrap();
|
||||
store.add(item("item:nibu:1", "old"), &ents(&d)).unwrap();
|
||||
let mut fix = item("item:nibu:2", "new");
|
||||
fix.supersedes = Some("item:nibu:1".into());
|
||||
assert_eq!(store.add(fix).unwrap(), AddOutcome::Added);
|
||||
assert_eq!(store.add(fix, &ents(&d)).unwrap(), AddOutcome::Added);
|
||||
assert_eq!(
|
||||
refusal(store.withdraw("item:nibu:7")),
|
||||
malformed("item:nibu:7", "no such item")
|
||||
|
|
@ -251,12 +267,12 @@ fn corpus_admission_rules() {
|
|||
fn s5_supersede_keeps_the_group() {
|
||||
let d = dir();
|
||||
let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap();
|
||||
store.add(item("item:nibu:1", "old")).unwrap();
|
||||
store.add(item("item:nibu:1", "old"), &ents(&d)).unwrap();
|
||||
let mut elsewhere = item("item:nibu:2", "new");
|
||||
elsewhere.supersedes = Some("item:nibu:1".into());
|
||||
elsewhere.group = "scene-2".into();
|
||||
assert_eq!(
|
||||
refusal(store.add(elsewhere)),
|
||||
refusal(store.add(elsewhere, &ents(&d))),
|
||||
StoreRefusal::S5SupersedeGroup {
|
||||
key: "item:nibu:2".into(),
|
||||
expected: "scene-1".into(),
|
||||
|
|
@ -266,7 +282,7 @@ fn s5_supersede_keeps_the_group() {
|
|||
assert!(store.get("item:nibu:2").unwrap().is_none());
|
||||
let mut same = item("item:nibu:2", "new");
|
||||
same.supersedes = Some("item:nibu:1".into());
|
||||
assert_eq!(store.add(same).unwrap(), AddOutcome::Added);
|
||||
assert_eq!(store.add(same, &ents(&d)).unwrap(), AddOutcome::Added);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -274,7 +290,9 @@ fn withdrawing_keeps_the_text_and_the_number() {
|
|||
let d = dir();
|
||||
let path = d.path().join("c.cc");
|
||||
let mut store = CorpusStore::open(&path).unwrap();
|
||||
store.add(item("item:nibu:1", "keep me")).unwrap();
|
||||
store
|
||||
.add(item("item:nibu:1", "keep me"), &ents(&d))
|
||||
.unwrap();
|
||||
assert_eq!(store.next_key("nibu").unwrap(), "item:nibu:2");
|
||||
assert_eq!(store.next_key("other").unwrap(), "item:other:1");
|
||||
store.withdraw("item:nibu:1").unwrap();
|
||||
|
|
@ -604,7 +622,7 @@ fn packet_admission_goes_through_the_validator() {
|
|||
let mut store = PacketStore::open(&w.dir.path().join("p.cc")).unwrap();
|
||||
let mut bad = good();
|
||||
bad.schema_version = 5;
|
||||
match store.admit(bad, &w.inventory, &w.corpus) {
|
||||
match store.admit(bad, &w.inventory, &w.corpus, &w.entities) {
|
||||
Err(e @ StoreError::Invalid(_)) => {
|
||||
assert!(e.to_string().contains("V1"), "{e}");
|
||||
let StoreError::Invalid(r) = e else {
|
||||
|
|
@ -615,13 +633,17 @@ fn packet_admission_goes_through_the_validator() {
|
|||
other => panic!("{other:?}"),
|
||||
}
|
||||
assert!(store.all().unwrap().is_empty());
|
||||
store.admit(good(), &w.inventory, &w.corpus).unwrap();
|
||||
store
|
||||
.admit(good(), &w.inventory, &w.corpus, &w.entities)
|
||||
.unwrap();
|
||||
// The same packet is idempotent; a different one under the same key is refused.
|
||||
store.admit(good(), &w.inventory, &w.corpus).unwrap();
|
||||
store
|
||||
.admit(good(), &w.inventory, &w.corpus, &w.entities)
|
||||
.unwrap();
|
||||
let mut other = good();
|
||||
other.provenance.confidence = grade(1);
|
||||
assert_eq!(
|
||||
refusal(store.admit(other, &w.inventory, &w.corpus)),
|
||||
refusal(store.admit(other, &w.inventory, &w.corpus, &w.entities)),
|
||||
StoreRefusal::PacketImmutable {
|
||||
key: PACKET_ID.into()
|
||||
}
|
||||
|
|
@ -637,17 +659,15 @@ fn admission_reads_the_store_inventory() {
|
|||
let d = dir();
|
||||
let mut corpus = empty_corpus(&d);
|
||||
corpus
|
||||
.add(item_in(
|
||||
"item:nibu:1",
|
||||
"Nibu says mind the hatch.",
|
||||
"g",
|
||||
cast(),
|
||||
))
|
||||
.add(
|
||||
item_in("item:nibu:1", "Nibu says mind the hatch.", "g", cast()),
|
||||
&ents(&d),
|
||||
)
|
||||
.unwrap();
|
||||
let mut inventory = seeded(&d.path().join("i.cc"));
|
||||
let mut packets = PacketStore::open(&d.path().join("p.cc")).unwrap();
|
||||
assert_eq!(
|
||||
match packets.admit(good(), &inventory, &corpus) {
|
||||
match packets.admit(good(), &inventory, &corpus, &ents(&d)) {
|
||||
Err(StoreError::Invalid(r)) => r,
|
||||
other => panic!("{other:?}"),
|
||||
},
|
||||
|
|
@ -667,26 +687,36 @@ fn admission_reads_the_store_inventory() {
|
|||
ext,
|
||||
],
|
||||
);
|
||||
packets.admit(good(), &inventory, &corpus).unwrap();
|
||||
packets
|
||||
.admit(good(), &inventory, &corpus, &ents(&d))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corpus_item_must_resolve() {
|
||||
let w = world();
|
||||
let mut packets = PacketStore::open(&w.dir.path().join("p.cc")).unwrap();
|
||||
for key in ["item:nibu:9", "banana"] {
|
||||
let mut p = good();
|
||||
p.provenance.corpus_item = Some(key.into());
|
||||
assert_eq!(
|
||||
refusal(packets.admit(p, &w.inventory, &w.corpus)),
|
||||
StoreRefusal::UnknownCorpusItem { key: key.into() }
|
||||
);
|
||||
}
|
||||
assert!(packets.all().unwrap().is_empty());
|
||||
// A packet minted for no item needs none.
|
||||
let mut p = good();
|
||||
p.provenance.corpus_item = None;
|
||||
packets.admit(p, &w.inventory, &w.corpus).unwrap();
|
||||
p.provenance.corpus_item = weksa::CorpusItemId::parse("item:nibu:9").unwrap();
|
||||
assert_eq!(
|
||||
refusal(packets.admit(p, &w.inventory, &w.corpus, &w.entities)),
|
||||
StoreRefusal::UnknownCorpusItem {
|
||||
key: "item:nibu:9".into()
|
||||
}
|
||||
);
|
||||
assert!(packets.all().unwrap().is_empty());
|
||||
// Free text is not an item id, in Rust or on the wire; a packet with no item
|
||||
// cannot be built or read back.
|
||||
for bad in ["banana", "", "the reference sentence goes right here"] {
|
||||
assert!(weksa::CorpusItemId::parse(bad).is_err(), "{bad:?}");
|
||||
}
|
||||
let mut wire = serde_json::to_value(good().provenance).unwrap();
|
||||
for bad in [serde_json::Value::Null, "banana".into()] {
|
||||
wire["corpus_item"] = bad;
|
||||
assert!(serde_json::from_value::<weksa::packet::Provenance>(wire.clone()).is_err());
|
||||
}
|
||||
wire.as_object_mut().unwrap().remove("corpus_item");
|
||||
assert!(serde_json::from_value::<weksa::packet::Provenance>(wire).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -698,7 +728,7 @@ fn store_roundtrip() {
|
|||
let p = good();
|
||||
PacketStore::open(&packets)
|
||||
.unwrap()
|
||||
.admit(p.clone(), &w.inventory, &w.corpus)
|
||||
.admit(p.clone(), &w.inventory, &w.corpus, &w.entities)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
PacketStore::open(&packets).unwrap().get(PACKET_ID).unwrap(),
|
||||
|
|
@ -732,7 +762,10 @@ fn store_roundtrip() {
|
|||
|
||||
let corpus = d.path().join("c.cc");
|
||||
let it = item("item:nibu:1", "round trip");
|
||||
CorpusStore::open(&corpus).unwrap().add(it.clone()).unwrap();
|
||||
CorpusStore::open(&corpus)
|
||||
.unwrap()
|
||||
.add(it.clone(), &ents(&d))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
CorpusStore::open(&corpus)
|
||||
.unwrap()
|
||||
|
|
@ -810,3 +843,29 @@ fn the_committed_stores_load_and_hold_clean_data() {
|
|||
);
|
||||
assert_eq!(items.len(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn seed_reseeded() {
|
||||
let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
|
||||
let entities = EntityStore::open(&root.join("data/entity/entity.cc")).unwrap();
|
||||
let ids: Vec<String> = entities
|
||||
.all()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|e| e.id.to_string())
|
||||
.collect();
|
||||
assert_eq!(ids, ["entity:brokkr", "entity:nibu"]);
|
||||
// The committed corpus replays through the door with its entities: every
|
||||
// speaker resolves and every span is a stored form.
|
||||
let items = CorpusStore::open(&root.join("data/corpus/corpus.cc"))
|
||||
.unwrap()
|
||||
.all()
|
||||
.unwrap();
|
||||
assert_eq!(items.len(), 3);
|
||||
let d = dir();
|
||||
let mut replay = CorpusStore::open(&d.path().join("c.cc")).unwrap();
|
||||
for i in items {
|
||||
assert!(entities.get(&i.speaker).unwrap().is_some(), "{}", i.id);
|
||||
replay.add(i, &entities).unwrap();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -360,7 +360,6 @@ fn lowering_input_carries_no_trace_id_or_provenance() {
|
|||
p.trace.rejected_readings = vec!["SECRET-READING".into()];
|
||||
let input = validate(&p, &inventory()).unwrap();
|
||||
let seen = format!("{input:?}");
|
||||
assert!(seen.contains("Nibu"), "the input carries the name: {seen}");
|
||||
for hidden in [PACKET_ID, ENCODER_ID, "item:nibu:1", "SECRET"] {
|
||||
assert!(!seen.contains(hidden), "{hidden} reached the input: {seen}");
|
||||
}
|
||||
|
|
@ -396,7 +395,7 @@ fn referent(id: &str) -> Referent {
|
|||
id: nd(id),
|
||||
kind: iv(PERSON),
|
||||
concept: iv(CONCEPT_A),
|
||||
name: None,
|
||||
entity: None,
|
||||
features: vec![],
|
||||
}
|
||||
}
|
||||
|
|
@ -507,7 +506,6 @@ fn two_referents() -> Packet {
|
|||
let mut p = good();
|
||||
let mut r = p.referents[0].clone();
|
||||
r.id = nd("n3");
|
||||
r.name = None;
|
||||
p.referents.push(r);
|
||||
p
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
“²cultcache.store.v1‘—±weksa.corpus_item±weksa.corpus_item´weksa.corpus_item.v1±weksa.corpus_itemÙV{"schemaName":"weksa.corpus_item","schemaVersion":"weksa.corpus_item.v1","members":[]}‘±weksa.corpus_item<EFBFBD>“”item:brokkr:1±weksa.corpus_item´2026-10-29T13:16:54ZÄ šitem:brokkr:1Ù,The command has no receipt. It is not truth.¥en-US¦brokkr±brokkr-edda-voice<EFBFBD><EFBFBD>¬ModelWritten“¦OpenAi¾codex-local-source-aware-draftÀª2026-10-29À”«item:nibu:1±weksa.corpus_item´2026-10-29T13:16:54ZÄÁš«item:nibu:1ÙnTake your hand off that control unless you want your nervous system to become my least useful circuit diagram.¥en-US¤nibu®nibu-proof-001<EFBFBD><EFBFBD>¬ModelWritten“ªUnrecordedÀÀª2026-10-29À”«item:nibu:2±weksa.corpus_item´2026-10-29T13:16:54ZÄäš«item:nibu:2Ù‘Do not touch that control. It is damaged; one touch may put you directly in harm's way. I am preventing an avoidable injury, not asking politely.¥en-US¤nibu®nibu-proof-001<EFBFBD><EFBFBD>¬ModelWritten“ªUnrecordedÀÀª2026-10-29ÀÂ
|
||||
“²cultcache.store.v1‘—±weksa.corpus_item±weksa.corpus_item´weksa.corpus_item.v1±weksa.corpus_itemÙV{"schemaName":"weksa.corpus_item","schemaVersion":"weksa.corpus_item.v1","members":[]}‘±weksa.corpus_item<EFBFBD>“”item:brokkr:1±weksa.corpus_item´2026-10-29T14:59:18ZĨ›item:brokkr:1Ù,The command has no receipt. It is not truth.¥en-USentity:brokkr±brokkr-edda-voice<EFBFBD><EFBFBD>¬ModelWritten“¦OpenAi¾codex-local-source-aware-draftÀª2026-10-29ÀÂ<EFBFBD>”«item:nibu:1±weksa.corpus_item´2026-10-29T14:59:18ZÄÉ›«item:nibu:1ÙnTake your hand off that control unless you want your nervous system to become my least useful circuit diagram.¥en-US«entity:nibu®nibu-proof-001<EFBFBD><EFBFBD>¬ModelWritten“ªUnrecordedÀÀª2026-10-29ÀÂ<EFBFBD>”«item:nibu:2±weksa.corpus_item´2026-10-29T14:59:18ZÄ웫item:nibu:2Ù‘Do not touch that control. It is damaged; one touch may put you directly in harm's way. I am preventing an avoidable injury, not asking politely.¥en-US«entity:nibu®nibu-proof-001<EFBFBD><EFBFBD>¬ModelWritten“ªUnrecordedÀÀª2026-10-29ÀÂ<EFBFBD>
|
||||
1
data/entity/entity.cc
Normal file
1
data/entity/entity.cc
Normal file
|
|
@ -0,0 +1 @@
|
|||
“²cultcache.store.v1‘—¬weksa.entity¬weksa.entity¯weksa.entity.v1¬weksa.entityÙL{"schemaName":"weksa.entity","schemaVersion":"weksa.entity.v1","members":[]}‘¬weksa.entity<EFBFBD>’”entity:brokkr¬weksa.entity´2026-10-29T14:59:18ZÄ-“entity:brokkr‘’¥en-US‘¦Brokkr<EFBFBD>¤Cast‘¦brokkr”«entity:nibu¬weksa.entity´2026-10-29T14:59:18ZÄ'“«entity:nibu‘’¥en-US‘¤Nibu<EFBFBD>¤Cast‘¤nibu
|
||||
|
|
@ -100,13 +100,19 @@ Every id is an opaque form type, parsed when it is built and when a store
|
|||
reads it: an inventory id is `<family>:<n>` with `n` a canonical decimal of at
|
||||
least 1 (`role:1`; `role:01` is not an id), a node id is `n<k>` with `k` in
|
||||
1..=9999, a packet id is `packet:<16 hex>` and an encoder id is
|
||||
`encoder:<16 hex>`. The free text a packet may hold is a referent's `name`,
|
||||
a `ProperName` of 1 to 4 space-joined tokens of letters, apostrophes and
|
||||
hyphens within 64 bytes, and the `trace`. A name is bounded text, not a judged
|
||||
one: no door compares packet text with the corpus. Leakage of scored text is
|
||||
owned outside the validator, by the loop's context builder (what each model
|
||||
call sees) and the scorer (names are masked before scoring); the validator
|
||||
owns form and capacity. The trace is stripped before lowering. Concept ids are opaque and carry a gloss in the inventory; concepts
|
||||
`encoder:<16 hex>`, an entity id is `entity:<slug>` (`[a-z0-9]` and inner
|
||||
`-`, 1 to 48 bytes) and a corpus item id is `item:<slug>:<n>`. The only free
|
||||
text a packet holds is the `trace` (`notes` and `rejected_readings`), which the
|
||||
validator neither bounds nor reads and which is stripped before lowering: a
|
||||
referent names its entity by id, and the entity store (`weksa.entity`) holds
|
||||
the entity's forms per language. The packet store refuses a referent whose
|
||||
entity the store lacks, a packet whose `corpus_item` the corpus lacks, and one
|
||||
whose entity is neither named on that item nor its speaker or a prior-turn
|
||||
speaker (a packet without a corpus item has no door in this store):
|
||||
an id-set check, not a text rule. Leakage of scored text is owned outside the
|
||||
validator, by the loop's context builder (what each model call sees) and the
|
||||
scorer; the validator owns form and capacity. The trace is stripped before
|
||||
lowering. Concept ids are opaque and carry a gloss in the inventory; concepts
|
||||
are open-class and grow, while the other families are closed-class.
|
||||
|
||||
```text
|
||||
|
|
@ -115,7 +121,7 @@ Packet
|
|||
provenance { source, corpus_item, encoder, confidence }
|
||||
context { setting, medium, participants, audience }
|
||||
discourse { speech_act, addressee, features }
|
||||
referents [ { id, kind, concept, name, features } ]
|
||||
referents [ { id, kind, concept, entity, features } ]
|
||||
predications [ { id, concept, frame, roles: [ { role, filler } ], features } ]
|
||||
relations [ { relation, from, to, features } ]
|
||||
constraints { preserve, deterministic, max_latency_ms }
|
||||
|
|
@ -165,8 +171,8 @@ reads at the packet's revision, never one the caller supplies.
|
|||
### `provenance`
|
||||
|
||||
`source` is `Authored`, `ModelDecomposition`, `GameTelemetry` or
|
||||
`ImportedCorpus`. `corpus_item` is the key of the corpus item the packet was
|
||||
minted for, `encoder` names what wrote it, and `confidence` is a grade.
|
||||
`ImportedCorpus`. `corpus_item` is the required `item:<slug>:<n>` key of the
|
||||
corpus item the packet was minted for, `encoder` names what wrote it, and `confidence` is a grade.
|
||||
|
||||
### `context`
|
||||
|
||||
|
|
@ -185,7 +191,7 @@ communicative pressure, not target-language register.
|
|||
Entities, groups, places, abstractions and the rest. `kind` is a referent-kind
|
||||
id (the inventory seeds `person`, `group`, `organism`, `place`, `artifact`,
|
||||
`institution`, `ecology`, `route`, `memory`, `event`, `abstract`), `concept` is
|
||||
a concept id, and `name` is a proper name when the referent has one.
|
||||
a concept id, and `entity` is the id of the named entity when the referent has one.
|
||||
|
||||
### `predications`
|
||||
|
||||
|
|
|
|||
|
|
@ -178,10 +178,12 @@ operator-attested annotations alone. Who annotates is question
|
|||
intake: cast entities from the cast fixtures, public-domain entities by the
|
||||
importer, and in live use by the persona before it names one. A corpus item
|
||||
carries its name spans (byte ranges of its text, each naming an entity and
|
||||
equal to one of that entity's forms in the item's language), and its speaker
|
||||
on extended grapheme cluster boundaries of its NFC text and equal to one of that
|
||||
entity's NFC forms in the item's language; an entity holds at most 8 languages
|
||||
of BCP 47-form tags, 8 forms each, a form at most 64 bytes and 4 words), and its speaker
|
||||
and prior-turn speakers are entity ids. The packet store refuses a packet
|
||||
whose referent cites an entity the store lacks, and a packet with a
|
||||
`corpus_item` that cites an entity neither annotated on that item nor one of
|
||||
`corpus_item` (required, typed `CorpusItemId`) that cites an entity neither annotated on that item nor one of
|
||||
its speakers: an id-set check, not a text rule. The scorer reads only the
|
||||
output and the reference and scores the whole of each. An honest name earns
|
||||
its credit ("Brokkr came back." against itself is 100), a wrong one loses it,
|
||||
|
|
|
|||
|
|
@ -17,6 +17,13 @@
|
|||
"language": {
|
||||
"type": "string"
|
||||
},
|
||||
"names": {
|
||||
"description": "Which entities the text names, and where. Fixed with the text at intake.",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/NameSpan"
|
||||
}
|
||||
},
|
||||
"prior_turns": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
|
|
@ -27,7 +34,7 @@
|
|||
"$ref": "#/$defs/Source"
|
||||
},
|
||||
"speaker": {
|
||||
"type": "string"
|
||||
"$ref": "#/$defs/EntityId"
|
||||
},
|
||||
"supersedes": {
|
||||
"description": "Key of the item this one corrects.",
|
||||
|
|
@ -52,9 +59,14 @@
|
|||
"prior_turns",
|
||||
"source",
|
||||
"admitted_on",
|
||||
"withdrawn"
|
||||
"withdrawn",
|
||||
"names"
|
||||
],
|
||||
"$defs": {
|
||||
"EntityId": {
|
||||
"type": "string",
|
||||
"pattern": "^entity:[a-z0-9]([a-z0-9-]{0,46}[a-z0-9])?$"
|
||||
},
|
||||
"ModelFamily": {
|
||||
"description": "The family of the model that wrote a line.",
|
||||
"oneOf": [
|
||||
|
|
@ -76,6 +88,30 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
"NameSpan": {
|
||||
"description": "A byte range `[start, end)` of an item's text that names an entity. The\nstore admits it only when both ends sit on extended grapheme cluster\nboundaries (UAX #29) of the NFC text and the range's bytes equal one of the\nentity's forms in the item's language.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"end": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0
|
||||
},
|
||||
"entity": {
|
||||
"$ref": "#/$defs/EntityId"
|
||||
},
|
||||
"start": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"entity",
|
||||
"start",
|
||||
"end"
|
||||
]
|
||||
},
|
||||
"Source": {
|
||||
"oneOf": [
|
||||
{
|
||||
|
|
@ -196,7 +232,7 @@
|
|||
"type": "object",
|
||||
"properties": {
|
||||
"speaker": {
|
||||
"type": "string"
|
||||
"$ref": "#/$defs/EntityId"
|
||||
},
|
||||
"text": {
|
||||
"type": "string"
|
||||
|
|
|
|||
103
schemas/cultnet/weksa.entity.schema.json
Normal file
103
schemas/cultnet/weksa.entity.schema.json
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "Entity",
|
||||
"description": "Key: the entity id. Forms only grow: see `EntityStore::put`.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"forms": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/LanguageForms"
|
||||
}
|
||||
},
|
||||
"id": {
|
||||
"$ref": "#/$defs/EntityId"
|
||||
},
|
||||
"origin": {
|
||||
"$ref": "#/$defs/EntityOrigin"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"forms",
|
||||
"origin"
|
||||
],
|
||||
"$defs": {
|
||||
"EntityId": {
|
||||
"type": "string",
|
||||
"pattern": "^entity:[a-z0-9]([a-z0-9-]{0,46}[a-z0-9])?$"
|
||||
},
|
||||
"EntityOrigin": {
|
||||
"description": "Where an entity was minted.",
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "From a cast member's fixture.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"Cast": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"persona": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"persona"
|
||||
]
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"Cast"
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "From a public-domain work.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"PublicDomain": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"work": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"work"
|
||||
]
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"PublicDomain"
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "Minted live by a persona before it names the entity.",
|
||||
"type": "string",
|
||||
"const": "Live"
|
||||
}
|
||||
]
|
||||
},
|
||||
"LanguageForms": {
|
||||
"description": "An entity's forms in one language, as the corpus item language tag spells it.\nThe citation form comes first.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"forms": {
|
||||
"description": "1 to 64 bytes each, no control character, no edge whitespace, none repeated.",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"language": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"language",
|
||||
"forms"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -143,6 +143,10 @@
|
|||
"audience"
|
||||
]
|
||||
},
|
||||
"CorpusItemId": {
|
||||
"type": "string",
|
||||
"pattern": "^item:[a-z0-9]([a-z0-9-]{0,46}[a-z0-9])?:[1-9][0-9]*$"
|
||||
},
|
||||
"Discourse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
|
@ -182,6 +186,10 @@
|
|||
"type": "string",
|
||||
"pattern": "^encoder:[0-9a-f]{16}$"
|
||||
},
|
||||
"EntityId": {
|
||||
"type": "string",
|
||||
"pattern": "^entity:[a-z0-9]([a-z0-9-]{0,46}[a-z0-9])?$"
|
||||
},
|
||||
"Extension": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
|
@ -323,12 +331,6 @@
|
|||
"features"
|
||||
]
|
||||
},
|
||||
"ProperName": {
|
||||
"description": "1 to 4 space-joined tokens of letters, apostrophes and hyphens.",
|
||||
"type": "string",
|
||||
"maxLength": 64,
|
||||
"pattern": "^\\p{L}([\\p{L}'’-]{0,22}\\p{L})?( \\p{L}([\\p{L}'’-]{0,22}\\p{L})?){0,3}$"
|
||||
},
|
||||
"Provenance": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
|
@ -336,11 +338,8 @@
|
|||
"$ref": "#/$defs/Grade"
|
||||
},
|
||||
"corpus_item": {
|
||||
"description": "Key of the corpus item this packet was minted for.",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
"description": "The corpus item this packet was minted for. Required: the packet store\nadmits only packets that name their item.",
|
||||
"$ref": "#/$defs/CorpusItemId"
|
||||
},
|
||||
"encoder": {
|
||||
"$ref": "#/$defs/EncoderId"
|
||||
|
|
@ -351,6 +350,7 @@
|
|||
},
|
||||
"required": [
|
||||
"source",
|
||||
"corpus_item",
|
||||
"encoder",
|
||||
"confidence"
|
||||
]
|
||||
|
|
@ -371,6 +371,17 @@
|
|||
"description": "Concept id.",
|
||||
"$ref": "#/$defs/InvId"
|
||||
},
|
||||
"entity": {
|
||||
"description": "The entity this referent names, when it has one. The entity store holds\nits wording; the packet holds only the id.",
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/EntityId"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"features": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
|
|
@ -383,17 +394,6 @@
|
|||
"kind": {
|
||||
"description": "Referent-kind id.",
|
||||
"$ref": "#/$defs/InvId"
|
||||
},
|
||||
"name": {
|
||||
"description": "A proper name, the one place a referent carries wording. It is\nform-bounded text; the scorer masks it.",
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/ProperName"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
|
|
|
|||
|
|
@ -36,8 +36,9 @@ phonology, diachrony, and renderer rules.
|
|||
- `extensions`: namespaced client or setting pressure.
|
||||
- `trace`: debug and authoring notes, stripped before lowering.
|
||||
|
||||
Every open slot is an inventory id or a node id. The only free text is a
|
||||
referent's proper `name` and the trace.
|
||||
Every open slot is an inventory id, a node id, an entity id or a corpus item id.
|
||||
The only free text is the trace, which the validator neither bounds nor reads
|
||||
and which is stripped before lowering.
|
||||
|
||||
## Extension Contract
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue