Compare commits
24 changed files with 100 additions and 2276 deletions
Binary file not shown.
10
AGENTS.md
10
AGENTS.md
|
|
@ -51,13 +51,3 @@ The live state architecture is ported from EpiphanyAgent in reduced form:
|
|||
Project truth does not belong in agent personality memory. If Weksa later grows
|
||||
role dossiers or heartbeat scheduling, those surfaces must remain separate from
|
||||
language-project truth.
|
||||
|
||||
## Building And Verifying
|
||||
|
||||
- `crates/weksa` depends on CultLib's `cultcache-rs` as a git dependency pinned by
|
||||
`rev`, fetched from GitHub. With the default `CARGO_HOME` that fetch fails on a
|
||||
Mind's host. Use the pre-fetched home:
|
||||
`CARGO_HOME=/var/lib/mind-weksa-self/cargo-home cargo test --workspace`.
|
||||
- `weksa packet proof <store>` admits the nibu proof packet under the key
|
||||
`packet:6e6962752d303031` (hex of `nibu-001`), not `nibu-proof-001`. Use that key
|
||||
with `weksa lower <store> <key>`.
|
||||
|
|
|
|||
|
|
@ -17,13 +17,10 @@ pub struct Turn {
|
|||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub enum Source {
|
||||
/// A line from a cast member's hand-authored fixture, with the human who
|
||||
/// wrote it. A fixture that records no author is `ModelWritten` with family
|
||||
/// `Unrecorded`, never this (`corpus-labels-tell-truth`).
|
||||
/// A line from a cast member's hand-authored fixture.
|
||||
CastAuthored {
|
||||
persona: String,
|
||||
fixture: String,
|
||||
author: String,
|
||||
},
|
||||
OperatorWritten,
|
||||
OperatorApproved {
|
||||
|
|
|
|||
|
|
@ -13,21 +13,19 @@ use crate::packet::EntityId;
|
|||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct LanguageForms {
|
||||
pub language: String,
|
||||
/// 1 to 64 bytes each, no control or invisible format character, no edge
|
||||
/// whitespace, none repeated.
|
||||
/// 1 to 64 bytes each, no control character, no edge whitespace, none repeated.
|
||||
pub forms: Vec<String>,
|
||||
}
|
||||
|
||||
/// Where an entity was minted.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub enum EntityOrigin {
|
||||
/// From a cast member's fixture: the persona and the fixture path the
|
||||
/// entity came from.
|
||||
Cast { persona: String, fixture: String },
|
||||
/// From a cast member's fixture.
|
||||
Cast { persona: String },
|
||||
/// From a public-domain work.
|
||||
PublicDomain { work: String },
|
||||
/// Minted live by the named persona before it names the entity.
|
||||
Live { persona: String },
|
||||
/// Minted live by a persona before it names the entity.
|
||||
Live,
|
||||
}
|
||||
|
||||
/// Key: the entity id. Forms only grow: see `EntityStore::put`.
|
||||
|
|
|
|||
|
|
@ -86,114 +86,3 @@ pub(crate) fn is_slug(s: &str) -> bool {
|
|||
&& b[0] != b'-'
|
||||
&& b[b.len() - 1] != b'-'
|
||||
}
|
||||
|
||||
/// A General_Category Cf code point, or the line and paragraph separators
|
||||
/// U+2028 and U+2029, other than the joiners ZWNJ U+200C and ZWJ U+200D. The
|
||||
/// ranges are Unicode 15.1 Cf; `tests/format_refused.txt` is the full list from
|
||||
/// Python's `unicodedata` that the table is swept against.
|
||||
pub(crate) fn is_format(c: char) -> bool {
|
||||
matches!(
|
||||
c,
|
||||
'\u{ad}'
|
||||
| '\u{600}'..='\u{605}'
|
||||
| '\u{61c}'
|
||||
| '\u{6dd}'
|
||||
| '\u{70f}'
|
||||
| '\u{890}'..='\u{891}'
|
||||
| '\u{8e2}'
|
||||
| '\u{180e}'
|
||||
| '\u{200b}'
|
||||
| '\u{200e}'..='\u{200f}'
|
||||
| '\u{2028}'..='\u{202e}'
|
||||
| '\u{2060}'..='\u{2064}'
|
||||
| '\u{2066}'..='\u{206f}'
|
||||
| '\u{feff}'
|
||||
| '\u{fff9}'..='\u{fffb}'
|
||||
| '\u{110bd}'
|
||||
| '\u{110cd}'
|
||||
| '\u{13430}'..='\u{1343f}'
|
||||
| '\u{1bca0}'..='\u{1bca3}'
|
||||
| '\u{1d173}'..='\u{1d17a}'
|
||||
| '\u{e0001}'
|
||||
| '\u{e0020}'..='\u{e007f}'
|
||||
)
|
||||
}
|
||||
|
||||
/// ZWNJ and ZWJ: Cf, but scripts and emoji sequences need them inside a form.
|
||||
pub(crate) fn is_joiner(c: char) -> bool {
|
||||
matches!(c, '\u{200c}' | '\u{200d}')
|
||||
}
|
||||
|
||||
/// A character a name or form may not hold: every `is_format` code point, and
|
||||
/// the invisible fillers that are Default_Ignorable without being Cf (Hangul
|
||||
/// U+115F, U+1160, U+3164, U+FFA0 and Khmer U+17B4, U+17B5). Joiners,
|
||||
/// variation selectors, CGJ and the Mongolian selectors are not refused.
|
||||
pub(crate) fn is_refused_format(c: char) -> bool {
|
||||
is_format(c)
|
||||
|| matches!(
|
||||
c,
|
||||
'\u{115f}' | '\u{1160}' | '\u{17b4}' | '\u{17b5}' | '\u{3164}' | '\u{ffa0}'
|
||||
)
|
||||
}
|
||||
|
||||
/// A joiner, a variation selector (U+FE00..U+FE0F, U+E0100..U+E01EF), CGJ
|
||||
/// U+034F or a Mongolian free variation selector (U+180B..U+180D, U+180F): it
|
||||
/// shapes a neighbour and shows nothing alone.
|
||||
pub(crate) fn is_joining_only(c: char) -> bool {
|
||||
is_joiner(c)
|
||||
|| matches!(
|
||||
c,
|
||||
'\u{34f}'
|
||||
| '\u{180b}'..='\u{180d}'
|
||||
| '\u{180f}'
|
||||
| '\u{fe00}'..='\u{fe0f}'
|
||||
| '\u{e0100}'..='\u{e01ef}'
|
||||
)
|
||||
}
|
||||
|
||||
/// An attestation string (author, persona, fixture, note, work, url, generator,
|
||||
/// digest): at most 200 bytes, at least one alphanumeric, no control, format or
|
||||
/// joiner character. Whitespace between words is fine; it attests something.
|
||||
pub(crate) fn is_attestation(s: &str) -> bool {
|
||||
s.len() <= 200
|
||||
&& s.chars().any(char::is_alphanumeric)
|
||||
&& !s
|
||||
.chars()
|
||||
.any(|c| c.is_control() || is_refused_format(c) || is_joiner(c))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The rule against every code point: `is_format` is exactly the listed
|
||||
/// Cf-minus-joiners-plus-U+2028/9 set, so each range has both of its
|
||||
/// edges and the gaps between ranges pinned.
|
||||
#[test]
|
||||
fn format_rule_matches_the_unicode_list_at_every_code_point() {
|
||||
let listed: std::collections::BTreeSet<u32> = include_str!("../tests/format_refused.txt")
|
||||
.lines()
|
||||
.filter(|l| !l.starts_with('#'))
|
||||
.map(|l| u32::from_str_radix(l, 16).unwrap())
|
||||
.collect();
|
||||
for code in 0..=0x10ffff_u32 {
|
||||
if let Some(c) = char::from_u32(code) {
|
||||
assert_eq!(is_format(c), listed.contains(&code), "U+{code:X}");
|
||||
}
|
||||
}
|
||||
assert!(!is_format('\u{200c}') && !is_format('\u{200d}'));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_attestation_is_bounded_readable_text() {
|
||||
assert!(is_attestation("a"));
|
||||
assert!(is_attestation(&"x".repeat(200)));
|
||||
assert!(!is_attestation(&"x".repeat(201)));
|
||||
assert!(is_attestation("two words, 2 more"));
|
||||
for bad in [
|
||||
"", " ", "\t", "\u{200b}", "\u{200d}", "\0", "--", "a\nb", "a\u{ad}b",
|
||||
] {
|
||||
assert!(!is_attestation(bad), "{bad:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -473,38 +473,3 @@ pub fn seed() -> (InventoryRevision, Vec<InventoryEntry>) {
|
|||
entries,
|
||||
)
|
||||
}
|
||||
|
||||
/// Revision 2: the concepts and frames the `nibu-proof-001` packet names, which
|
||||
/// the seed leaves open. Hand-seeded like [`seed`] until the golf loop's
|
||||
/// minting round owns concepts and frames.
|
||||
pub fn proof_revision() -> (InventoryRevision, Vec<InventoryEntry>) {
|
||||
let mut entries: Vec<InventoryEntry> = Vec::new();
|
||||
for (family, glosses) in [
|
||||
(
|
||||
Family::Concept,
|
||||
["endanger", "prevent", "player", "control", "ship_mind"].as_slice(),
|
||||
),
|
||||
(Family::Frame, ["hazard_warning", "intervention"].as_slice()),
|
||||
] {
|
||||
for (i, gloss) in glosses.iter().enumerate() {
|
||||
entries.push(InventoryEntry {
|
||||
id: family.id(NonZeroU32::MIN.saturating_add(i as u32)),
|
||||
gloss: gloss.to_string(),
|
||||
value_kind: None,
|
||||
applies_to: vec![],
|
||||
introduced_rev: 2,
|
||||
retired_rev: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
let changes = entries.iter().map(|e| Change::Add(e.id)).collect();
|
||||
(
|
||||
InventoryRevision {
|
||||
n: 2,
|
||||
parent: Some(1),
|
||||
changes,
|
||||
frozen: false,
|
||||
},
|
||||
entries,
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ pub mod corpus;
|
|||
pub mod entity;
|
||||
pub mod form;
|
||||
pub mod inventory;
|
||||
pub mod lower;
|
||||
pub mod packet;
|
||||
pub mod schema;
|
||||
pub mod score;
|
||||
|
|
@ -22,7 +21,6 @@ pub use inventory::{
|
|||
Change, Family, InvId, Inventory, InventoryEntry, InventoryRevision, OptionLabel, OptionSet,
|
||||
Slot, ValueKind,
|
||||
};
|
||||
pub use lower::{LowerError, lower};
|
||||
pub use packet::{
|
||||
EncoderId, EntityId, Grade, NodeId, Packet, PacketId, PacketKind, SCHEMA_VERSION, Stance,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,205 +0,0 @@
|
|||
//! The English lowerer: the validated lowering view of a packet to text, by a
|
||||
//! fixed table keyed by inventory ids. It renders predications, referents and the
|
||||
//! speech act, and nothing else: no style, no model, no corpus text. It takes a
|
||||
//! [`LoweringInput`], so it cannot see the trace or the corpus item. A packet
|
||||
//! carrying a feature, relation, extension, role fill or referent it does not
|
||||
//! render is refused, because rendering part of a meaning would assert a different
|
||||
//! one (`polarity: negative` dropped says the opposite). Context, kind and
|
||||
//! constraints steer delivery and are not rendered; the concept of a predication
|
||||
//! must resolve but the frame owns the verb.
|
||||
//!
|
||||
//! The table is a temporary owner. Each row speaks for an inventory entry and
|
||||
//! names the gloss it speaks for, so a row cannot outlive a change of what its id
|
||||
//! means. `language-ontology` moves the rows into ontology documents.
|
||||
|
||||
use std::fmt;
|
||||
|
||||
use crate::inventory::{Family, InvId, Inventory};
|
||||
use crate::packet::{NodeId, Predication};
|
||||
use crate::validate::LoweringInput;
|
||||
|
||||
/// `(inventory id, the gloss the row speaks for, surface)`.
|
||||
type Row = (&'static str, &'static str, &'static str);
|
||||
|
||||
/// Referent concepts, as the noun phrase after "the".
|
||||
const CONCEPTS: &[Row] = &[
|
||||
("concept:3", "player", "player"),
|
||||
("concept:4", "control", "control"),
|
||||
("concept:5", "ship_mind", "ship mind"),
|
||||
];
|
||||
|
||||
/// Frames: the clause a predication of that frame says. `{gloss}` is the filler
|
||||
/// of the role whose inventory gloss it is.
|
||||
const FRAMES: &[Row] = &[
|
||||
(
|
||||
"frame:1",
|
||||
"hazard_warning",
|
||||
"{agent} may endanger {patient} by touching {instrument}",
|
||||
),
|
||||
(
|
||||
"frame:2",
|
||||
"intervention",
|
||||
"{agent} will prevent {patient} to protect {beneficiary}",
|
||||
),
|
||||
];
|
||||
|
||||
/// Speech acts: the opener of the first sentence.
|
||||
const SPEECH_ACTS: &[Row] = &[
|
||||
("speech_act:1", "assert", ""),
|
||||
("speech_act:5", "warn", "Warning: "),
|
||||
];
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum LowerError {
|
||||
/// The packet carries a slot the lowerer does not render.
|
||||
Unrendered { what: &'static str },
|
||||
/// No table row for an id the packet uses.
|
||||
NoRow { id: InvId },
|
||||
/// The row's gloss is not the gloss the inventory gives its id at the
|
||||
/// packet's revision, or the id does not resolve there.
|
||||
GlossMismatch { id: InvId, row: &'static str },
|
||||
/// A frame template names a role the inventory has no live entry for.
|
||||
UnknownRole { frame: InvId, role: String },
|
||||
/// The predication fills no role its frame's template needs.
|
||||
MissingRole { predication: NodeId, role: String },
|
||||
}
|
||||
|
||||
impl fmt::Display for LowerError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{self:?}")
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for LowerError {}
|
||||
|
||||
fn row(
|
||||
table: &'static [Row],
|
||||
id: InvId,
|
||||
inv: &Inventory,
|
||||
rev: u32,
|
||||
) -> Result<&'static str, LowerError> {
|
||||
let (_, gloss, surface) = table
|
||||
.iter()
|
||||
.find(|(key, _, _)| *key == id.to_string())
|
||||
.ok_or(LowerError::NoRow { id })?;
|
||||
match inv.resolve(&id, rev) {
|
||||
Some(e) if e.gloss == *gloss => Ok(surface),
|
||||
_ => Err(LowerError::GlossMismatch { id, row: gloss }),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn lower(input: &LoweringInput, inv: &Inventory) -> Result<String, LowerError> {
|
||||
let rev = input.inventory_rev();
|
||||
let unrendered = |what| Err(LowerError::Unrendered { what });
|
||||
if input.predications().is_empty() {
|
||||
return unrendered("no predication");
|
||||
}
|
||||
if !input.discourse().features.is_empty()
|
||||
|| input.referents().iter().any(|r| !r.features.is_empty())
|
||||
|| input.predications().iter().any(|p| !p.features.is_empty())
|
||||
{
|
||||
return unrendered("features");
|
||||
}
|
||||
if !input.relations().is_empty() {
|
||||
return unrendered("relations");
|
||||
}
|
||||
if !input.extensions().is_empty() {
|
||||
return unrendered("extensions");
|
||||
}
|
||||
let opener = match input.discourse().speech_act {
|
||||
Some(act) => row(SPEECH_ACTS, act, inv, rev)?,
|
||||
None => "",
|
||||
};
|
||||
let mut sentences = Vec::new();
|
||||
for p in input.predications() {
|
||||
sentences.push(clause(input, inv, p)?);
|
||||
}
|
||||
let mentioned = |r: &NodeId| {
|
||||
input
|
||||
.predications()
|
||||
.iter()
|
||||
.any(|p| p.roles.iter().any(|f| &f.filler == r))
|
||||
};
|
||||
if input.referents().iter().any(|r| !mentioned(&r.id)) {
|
||||
return unrendered("a referent no role mentions");
|
||||
}
|
||||
let mut out = String::new();
|
||||
for (i, clause) in sentences.iter().enumerate() {
|
||||
if i > 0 {
|
||||
out.push(' ');
|
||||
}
|
||||
if i == 0 {
|
||||
out.push_str(opener);
|
||||
}
|
||||
let mut chars = clause.chars();
|
||||
if let Some(first) = chars.next() {
|
||||
if i == 0 && !opener.is_empty() {
|
||||
out.push(first);
|
||||
} else {
|
||||
out.extend(first.to_uppercase());
|
||||
}
|
||||
out.push_str(chars.as_str());
|
||||
}
|
||||
out.push('.');
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn clause(input: &LoweringInput, inv: &Inventory, p: &Predication) -> Result<String, LowerError> {
|
||||
let rev = input.inventory_rev();
|
||||
let template = row(FRAMES, p.frame, inv, rev)?;
|
||||
let mut said: Vec<&NodeId> = Vec::new();
|
||||
let mut out = String::new();
|
||||
let mut rest = template;
|
||||
while let Some(open) = rest.find('{') {
|
||||
out.push_str(&rest[..open]);
|
||||
let close = rest[open..].find('}').map_or(rest.len(), |c| open + c);
|
||||
let name = &rest[open + 1..close];
|
||||
rest = &rest[(close + 1).min(rest.len())..];
|
||||
let role = inv
|
||||
.entries()
|
||||
.find(|e| e.family() == Family::Role && e.gloss == name && e.live_at(rev))
|
||||
.ok_or_else(|| LowerError::UnknownRole {
|
||||
frame: p.frame,
|
||||
role: name.to_string(),
|
||||
})?;
|
||||
let fill =
|
||||
p.roles
|
||||
.iter()
|
||||
.find(|r| r.role == role.id)
|
||||
.ok_or_else(|| LowerError::MissingRole {
|
||||
predication: p.id.clone(),
|
||||
role: name.to_string(),
|
||||
})?;
|
||||
out.push_str(&filler(input, inv, &fill.filler, &said)?);
|
||||
said.push(&fill.filler);
|
||||
}
|
||||
out.push_str(rest);
|
||||
if p.roles.len() != said.len() {
|
||||
return Err(LowerError::Unrendered {
|
||||
what: "a role the frame does not name",
|
||||
});
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// A role filler as a phrase: the addressee is "you", a predication is "that",
|
||||
/// another referent is "the" and its concept word. The addressee said twice in
|
||||
/// one predication is reflexive; no other referent is, because the packet has no
|
||||
/// animacy to choose "themself" or "itself" by.
|
||||
fn filler(
|
||||
input: &LoweringInput,
|
||||
inv: &Inventory,
|
||||
node: &NodeId,
|
||||
said: &[&NodeId],
|
||||
) -> Result<String, LowerError> {
|
||||
let again = said.contains(&node);
|
||||
if let Some(r) = input.referents().iter().find(|r| &r.id == node) {
|
||||
if input.discourse().addressee.as_ref() == Some(node) {
|
||||
return Ok(if again { "yourself" } else { "you" }.to_string());
|
||||
}
|
||||
let word = row(CONCEPTS, r.concept, inv, input.inventory_rev())?;
|
||||
return Ok(format!("the {word}"));
|
||||
}
|
||||
Ok("that".to_string())
|
||||
}
|
||||
|
|
@ -8,12 +8,10 @@ use clap::{Args, Parser, Subcommand, ValueEnum};
|
|||
use unicode_normalization::UnicodeNormalization;
|
||||
use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn};
|
||||
use weksa::entity::{Entity, EntityOrigin, LanguageForms};
|
||||
use weksa::inventory::{proof_revision, seed};
|
||||
use weksa::packet::proof_packet;
|
||||
use weksa::score::ScoreInput;
|
||||
use weksa::inventory::seed;
|
||||
use weksa::{
|
||||
CorpusStore, EntityId, EntityStore, InventoryStore, OllamaEmbedder, PacketStore, Scorer,
|
||||
SplitScheme, StoreError, lower, validate,
|
||||
CorpusStore, EntityId, EntityStore, InventoryStore, PacketStore, SplitScheme, StoreError,
|
||||
validate,
|
||||
};
|
||||
|
||||
#[derive(Parser)]
|
||||
|
|
@ -45,12 +43,6 @@ enum Command {
|
|||
Inventory(InventoryCommand),
|
||||
#[command(subcommand)]
|
||||
Entity(EntityCommand),
|
||||
#[command(subcommand)]
|
||||
Packet(PacketCommand),
|
||||
/// Lower a stored packet to English text.
|
||||
Lower { store: PathBuf, key: String },
|
||||
/// Score output text (stdin, or --output) against corpus items, one embed call.
|
||||
Score(ScoreArgs),
|
||||
/// Print the split of a corpus item.
|
||||
Split { item_key: String },
|
||||
/// Write the published JSON Schemas.
|
||||
|
|
@ -60,26 +52,6 @@ enum Command {
|
|||
},
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum PacketCommand {
|
||||
/// Admit the nibu-proof-001 packet through `PacketStore::admit`.
|
||||
Proof { store: PathBuf },
|
||||
}
|
||||
|
||||
#[derive(Args)]
|
||||
struct ScoreArgs {
|
||||
/// Corpus item keys, the references.
|
||||
#[arg(required = true)]
|
||||
items: Vec<String>,
|
||||
/// The text to score; stdin when absent.
|
||||
#[arg(long)]
|
||||
output: Option<PathBuf>,
|
||||
#[arg(long, default_value = "http://127.0.0.1:11434")]
|
||||
ollama_url: String,
|
||||
#[arg(long, default_value = "qwen3-embedding:0.6b")]
|
||||
model: String,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum CorpusCommand {
|
||||
Add(Box<AddArgs>),
|
||||
|
|
@ -96,8 +68,6 @@ enum InventoryCommand {
|
|||
Dump,
|
||||
/// Admit revision 1, the standard's enumerated values and the nuance features.
|
||||
Seed,
|
||||
/// Admit revision 2, the concepts and frames the nibu-proof-001 packet names.
|
||||
Proof,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
|
|
@ -118,12 +88,9 @@ struct PutArgs {
|
|||
forms: Vec<String>,
|
||||
#[arg(long, value_enum)]
|
||||
origin: OriginKind,
|
||||
/// The persona of a `cast` or `live` origin; refused with any other origin.
|
||||
/// The persona of a `cast` origin; refused with any other origin.
|
||||
#[arg(long)]
|
||||
persona: Option<String>,
|
||||
/// The fixture path a `cast` entity came from; refused with any other origin.
|
||||
#[arg(long)]
|
||||
fixture: Option<String>,
|
||||
/// The work of a `public-domain` origin; refused with any other origin.
|
||||
#[arg(long)]
|
||||
work: Option<String>,
|
||||
|
|
@ -140,7 +107,9 @@ impl PutArgs {
|
|||
fn entity(self) -> Result<Entity> {
|
||||
let mut forms: Vec<LanguageForms> = Vec::new();
|
||||
for f in &self.forms {
|
||||
let (language, form) = f.split_once('=').context("--form is not language=form")?;
|
||||
let (language, form) = f
|
||||
.split_once('=')
|
||||
.with_context(|| format!("--form {f:?} is not language=form"))?;
|
||||
match forms.iter_mut().find(|lf| lf.language == language) {
|
||||
Some(lf) => lf.forms.push(form.nfc().collect()),
|
||||
None => forms.push(LanguageForms {
|
||||
|
|
@ -149,19 +118,16 @@ impl PutArgs {
|
|||
}),
|
||||
}
|
||||
}
|
||||
let origin = match (self.origin, self.persona, self.fixture, self.work) {
|
||||
(OriginKind::Cast, persona, fixture, None) => EntityOrigin::Cast {
|
||||
let origin = match (self.origin, self.persona, self.work) {
|
||||
(OriginKind::Cast, persona, None) => EntityOrigin::Cast {
|
||||
persona: need(persona, "persona")?,
|
||||
fixture: need(fixture, "fixture")?,
|
||||
},
|
||||
(OriginKind::PublicDomain, None, None, work) => EntityOrigin::PublicDomain {
|
||||
(OriginKind::PublicDomain, None, work) => EntityOrigin::PublicDomain {
|
||||
work: need(work, "work")?,
|
||||
},
|
||||
(OriginKind::Live, persona, None, None) => EntityOrigin::Live {
|
||||
persona: need(persona, "persona")?,
|
||||
},
|
||||
(OriginKind::Live, None, None) => EntityOrigin::Live,
|
||||
_ => bail!(
|
||||
"--fixture is only for --origin cast, --persona only for --origin cast or live and --work only for --origin public-domain"
|
||||
"--persona is only for --origin cast and --work only for --origin public-domain"
|
||||
),
|
||||
};
|
||||
Ok(Entity {
|
||||
|
|
@ -236,7 +202,6 @@ struct AddArgs {
|
|||
note: Option<String>,
|
||||
#[arg(long)]
|
||||
work: Option<String>,
|
||||
/// Who wrote the line: for `cast-authored` and `public-domain`.
|
||||
#[arg(long)]
|
||||
author: Option<String>,
|
||||
#[arg(long)]
|
||||
|
|
@ -261,37 +226,10 @@ fn need(value: Option<String>, flag: &str) -> Result<String> {
|
|||
impl AddArgs {
|
||||
fn source(&self) -> Result<Source> {
|
||||
let get = |v: &Option<String>, flag: &str| need(v.clone(), flag);
|
||||
let applies: &[&str] = match self.source {
|
||||
SourceKind::CastAuthored => &["persona", "fixture", "author"],
|
||||
SourceKind::OperatorWritten => &[],
|
||||
SourceKind::OperatorApproved => &["note"],
|
||||
SourceKind::PublicDomain => &["work", "author", "edition-url"],
|
||||
SourceKind::ModelWritten => &["model-family", "generator", "prompt-digest"],
|
||||
};
|
||||
let given = [
|
||||
("persona", self.persona.is_some()),
|
||||
("fixture", self.fixture.is_some()),
|
||||
("author", self.author.is_some()),
|
||||
("note", self.note.is_some()),
|
||||
("work", self.work.is_some()),
|
||||
("edition-url", self.edition_url.is_some()),
|
||||
("model-family", self.family.is_some()),
|
||||
("generator", self.generator.is_some()),
|
||||
("prompt-digest", self.prompt_digest.is_some()),
|
||||
];
|
||||
let stray: Vec<String> = given
|
||||
.into_iter()
|
||||
.filter(|(flag, given)| *given && !applies.contains(flag))
|
||||
.map(|(flag, _)| format!("--{flag}"))
|
||||
.collect();
|
||||
if !stray.is_empty() {
|
||||
bail!("{} does not apply to this --source", stray.join(", "));
|
||||
}
|
||||
Ok(match self.source {
|
||||
SourceKind::CastAuthored => Source::CastAuthored {
|
||||
persona: get(&self.persona, "persona")?,
|
||||
fixture: get(&self.fixture, "fixture")?,
|
||||
author: get(&self.author, "author")?,
|
||||
},
|
||||
SourceKind::OperatorWritten => Source::OperatorWritten,
|
||||
SourceKind::OperatorApproved => Source::OperatorApproved {
|
||||
|
|
@ -320,11 +258,9 @@ impl AddArgs {
|
|||
.map(|t| {
|
||||
let (speaker, text) = t
|
||||
.split_once('=')
|
||||
.context("--prior-turn is not speaker=text")?;
|
||||
.with_context(|| format!("--prior-turn {t:?} is not speaker=text"))?;
|
||||
Ok(Turn {
|
||||
speaker: speaker.parse().map_err(|_| {
|
||||
anyhow::anyhow!("--prior-turn speaker is not entity:<slug>")
|
||||
})?,
|
||||
speaker: speaker.parse()?,
|
||||
text: text.nfc().collect(),
|
||||
})
|
||||
})
|
||||
|
|
@ -333,13 +269,13 @@ impl AddArgs {
|
|||
.names
|
||||
.iter()
|
||||
.map(|n| {
|
||||
let bad = || anyhow::anyhow!("--name is not start..end=entity:<slug>");
|
||||
let (range, entity) = n.split_once('=').ok_or_else(bad)?;
|
||||
let (start, end) = range.split_once("..").ok_or_else(bad)?;
|
||||
let bad = || format!("--name {n:?} is not start..end=entity:<slug>");
|
||||
let (range, entity) = n.split_once('=').with_context(bad)?;
|
||||
let (start, end) = range.split_once("..").with_context(bad)?;
|
||||
Ok(NameSpan {
|
||||
entity: entity.parse().map_err(|_| bad())?,
|
||||
start: start.parse().map_err(|_| bad())?,
|
||||
end: end.parse().map_err(|_| bad())?,
|
||||
entity: entity.parse()?,
|
||||
start: start.parse().with_context(bad)?,
|
||||
end: end.parse().with_context(bad)?,
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
|
|
@ -402,7 +338,7 @@ fn main() -> Result<()> {
|
|||
}
|
||||
Command::Corpus(CorpusCommand::Add(args)) => {
|
||||
let mut store = CorpusStore::open(&cli.corpus)?;
|
||||
let key = store.next_key(&args.slug).map_err(refused)?;
|
||||
let key = store.next_key(&args.slug)?;
|
||||
let entities = EntityStore::open(&cli.entities)?;
|
||||
let outcome = store
|
||||
.add((*args).item(key.clone())?, &entities)
|
||||
|
|
@ -464,77 +400,6 @@ fn main() -> Result<()> {
|
|||
.map_err(refused)?;
|
||||
println!("seeded rev:1 with {count} entries");
|
||||
}
|
||||
Command::Inventory(InventoryCommand::Proof) => {
|
||||
let (rev, entries) = proof_revision();
|
||||
let count = entries.len();
|
||||
InventoryStore::open(&cli.inventory)?
|
||||
.admit_revision(rev, entries)
|
||||
.map_err(refused)?;
|
||||
println!("admitted rev:2 with {count} entries");
|
||||
}
|
||||
Command::Packet(PacketCommand::Proof { store }) => {
|
||||
let packet = proof_packet();
|
||||
let key = packet.id.to_string();
|
||||
PacketStore::open(&store)?
|
||||
.admit(
|
||||
packet,
|
||||
&InventoryStore::open(&cli.inventory)?,
|
||||
&CorpusStore::open(&cli.corpus)?,
|
||||
&EntityStore::open(&cli.entities)?,
|
||||
)
|
||||
.map_err(refused)?;
|
||||
println!("admitted {key}");
|
||||
}
|
||||
Command::Lower { store, key } => {
|
||||
let inventory_store = InventoryStore::open(&cli.inventory)?;
|
||||
let inventory = inventory_store.inventory()?;
|
||||
let mut packets = PacketStore::open(&store)?;
|
||||
let packet = packets
|
||||
.get(&key)?
|
||||
.with_context(|| format!("no packet {key}"))?;
|
||||
// Admission is the only source of a LoweringInput; re-admitting the
|
||||
// stored packet is a no-op write that re-runs every door check.
|
||||
let view = packets
|
||||
.admit(
|
||||
packet,
|
||||
&inventory_store,
|
||||
&CorpusStore::open(&cli.corpus)?,
|
||||
&EntityStore::open(&cli.entities)?,
|
||||
)
|
||||
.map_err(refused)?;
|
||||
println!("{}", lower(&view, &inventory)?);
|
||||
}
|
||||
Command::Score(args) => {
|
||||
let output = match &args.output {
|
||||
Some(path) => std::fs::read_to_string(path)?,
|
||||
None => std::io::read_to_string(std::io::stdin())?,
|
||||
};
|
||||
let corpus = CorpusStore::open(&cli.corpus)?;
|
||||
let references = args
|
||||
.items
|
||||
.iter()
|
||||
.map(|key| {
|
||||
Ok(corpus
|
||||
.get(key)?
|
||||
.with_context(|| format!("no item {key}"))?
|
||||
.text)
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
let inputs: Vec<ScoreInput> = references
|
||||
.iter()
|
||||
.map(|reference| ScoreInput {
|
||||
reference,
|
||||
output: output.trim(),
|
||||
})
|
||||
.collect();
|
||||
let scorer = Scorer::new(OllamaEmbedder::new(&args.ollama_url, &args.model));
|
||||
for (key, score) in args.items.iter().zip(scorer.score(&inputs)?) {
|
||||
match score.meaning {
|
||||
Some(m) => println!("{key} chrf={:.2} meaning={m:.4}", score.chrf),
|
||||
None => println!("{key} chrf={:.2} meaning=none", score.chrf),
|
||||
}
|
||||
}
|
||||
}
|
||||
Command::Split { item_key } => {
|
||||
let item = CorpusStore::open(&cli.corpus)?
|
||||
.get(&item_key)?
|
||||
|
|
|
|||
|
|
@ -347,86 +347,3 @@ pub struct Trace {
|
|||
pub notes: Vec<String>,
|
||||
pub rejected_readings: Vec<String>,
|
||||
}
|
||||
|
||||
/// The `nibu-proof-001` packet: the meaning of `examples/english-lowering/nibu-proof-001/interlingua.yaml`
|
||||
/// (warn the player off the damaged control; Nibu prevents it for them) as typed
|
||||
/// ids of inventory revision 2, for the corpus item `item:nibu:1`. No features:
|
||||
/// nothing here is style.
|
||||
pub fn proof_packet() -> Packet {
|
||||
fn id(s: &str) -> InvId {
|
||||
InvId::parse(s).expect("proof ids are well formed")
|
||||
}
|
||||
fn node(s: &str) -> NodeId {
|
||||
NodeId::parse(s).expect("proof nodes are well formed")
|
||||
}
|
||||
let referent = |n: &str, kind: &str, concept: &str, entity: Option<&str>| Referent {
|
||||
id: node(n),
|
||||
kind: id(kind),
|
||||
concept: id(concept),
|
||||
entity: entity.map(|e| EntityId::parse(e).expect("proof entity is well formed")),
|
||||
features: vec![],
|
||||
};
|
||||
let fill = |role: &str, filler: &str| RoleFill {
|
||||
role: id(role),
|
||||
filler: node(filler),
|
||||
};
|
||||
Packet {
|
||||
id: PacketId::parse("packet:6e6962752d303031").expect("proof id"),
|
||||
schema_version: SCHEMA_VERSION,
|
||||
inventory_rev: 2,
|
||||
kind: PacketKind::DialogueLine,
|
||||
provenance: Provenance {
|
||||
source: ProvenanceSource::Authored,
|
||||
corpus_item: CorpusItemId::parse("item:nibu:1").expect("proof item"),
|
||||
encoder: EncoderId::parse("encoder:6f70657261746f72").expect("proof encoder"),
|
||||
confidence: Grade::new(900).expect("proof grade"),
|
||||
},
|
||||
context: Context::default(),
|
||||
discourse: Discourse {
|
||||
speech_act: Some(id("speech_act:5")),
|
||||
addressee: Some(node("n1")),
|
||||
features: vec![],
|
||||
},
|
||||
referents: vec![
|
||||
referent("n1", "referent_kind:1", "concept:3", None),
|
||||
referent("n2", "referent_kind:5", "concept:4", None),
|
||||
referent("n3", "referent_kind:1", "concept:5", Some("entity:nibu")),
|
||||
],
|
||||
predications: vec![
|
||||
Predication {
|
||||
id: node("n4"),
|
||||
concept: id("concept:1"),
|
||||
frame: id("frame:1"),
|
||||
roles: vec![
|
||||
fill("role:1", "n1"),
|
||||
fill("role:4", "n2"),
|
||||
fill("role:2", "n1"),
|
||||
],
|
||||
features: vec![],
|
||||
},
|
||||
Predication {
|
||||
id: node("n5"),
|
||||
concept: id("concept:2"),
|
||||
frame: id("frame:2"),
|
||||
roles: vec![
|
||||
fill("role:1", "n3"),
|
||||
fill("role:2", "n4"),
|
||||
fill("role:8", "n1"),
|
||||
],
|
||||
features: vec![],
|
||||
},
|
||||
],
|
||||
relations: vec![],
|
||||
constraints: Constraints {
|
||||
preserve: vec![node("n1"), node("n2")],
|
||||
deterministic: true,
|
||||
max_latency_ms: None,
|
||||
},
|
||||
extensions: vec![],
|
||||
trace: Trace {
|
||||
notes: vec!["First proof packet for Nibu flavored-English lowering.".to_string()],
|
||||
rejected_readings: vec![],
|
||||
},
|
||||
stance: Stance::default(),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,14 +6,13 @@ use std::collections::BTreeSet;
|
|||
use std::fmt;
|
||||
use std::path::Path;
|
||||
|
||||
use caseless::default_case_fold_str;
|
||||
use cultcache_rs::{CultCache, SingleFileMessagePackBackingStore};
|
||||
use unicode_normalization::{UnicodeNormalization, char::is_combining_mark, is_nfc};
|
||||
use unicode_normalization::is_nfc;
|
||||
use unicode_segmentation::UnicodeSegmentation;
|
||||
|
||||
use crate::corpus::{CorpusItem, CorpusItemId, Source, item_key, key_slug};
|
||||
use crate::entity::{Entity, EntityOrigin};
|
||||
use crate::form::{is_attestation, is_joining_only, is_language_tag, is_refused_format};
|
||||
use crate::corpus::{CorpusItem, CorpusItemId, item_key, key_slug};
|
||||
use crate::entity::Entity;
|
||||
use crate::form::is_language_tag;
|
||||
use crate::inventory::{Change, InvId, Inventory, InventoryEntry, InventoryRevision, revision_key};
|
||||
use crate::packet::{EntityId, Packet};
|
||||
use crate::validate::{LoweringInput, Refusal, admissible};
|
||||
|
|
@ -92,12 +91,15 @@ pub enum EntityBound {
|
|||
Forms,
|
||||
/// A form of more than 64 bytes.
|
||||
FormBytes,
|
||||
/// A form of more than 4 words.
|
||||
FormWords,
|
||||
}
|
||||
|
||||
impl EntityBound {
|
||||
pub const MAX_LANGUAGES: usize = 8;
|
||||
pub const MAX_FORMS: usize = 8;
|
||||
pub const MAX_FORM_BYTES: usize = 64;
|
||||
pub const MAX_FORM_WORDS: usize = 4;
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
|
|
@ -186,7 +188,7 @@ impl CorpusStore {
|
|||
}
|
||||
|
||||
/// The next free key for `slug`; withdrawn items keep their numbers.
|
||||
pub fn next_key(&self, slug: &str) -> Result<String, StoreError> {
|
||||
pub fn next_key(&self, slug: &str) -> anyhow::Result<String> {
|
||||
let highest = self
|
||||
.all()?
|
||||
.iter()
|
||||
|
|
@ -194,10 +196,7 @@ impl CorpusStore {
|
|||
.filter_map(|i| i.id.rsplit_once(':')?.1.parse::<u32>().ok())
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
let next = highest
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| malformed(slug, "slug has no free key"))?;
|
||||
Ok(item_key(slug, next))
|
||||
Ok(item_key(slug, highest + 1))
|
||||
}
|
||||
|
||||
/// The item `id` names, or `UnknownCorpusItem` when it holds nothing.
|
||||
|
|
@ -240,14 +239,8 @@ impl CorpusStore {
|
|||
if !is_nfc(&item.text) || item.prior_turns.iter().any(|t| !is_nfc(&t.text)) {
|
||||
return Err(StoreRefusal::NotNfc { key: item.id }.into());
|
||||
}
|
||||
let text = normalized(&item.text);
|
||||
if !text.chars().any(char::is_alphanumeric) {
|
||||
return Err(malformed(&item.id, "text has no alphanumeric character"));
|
||||
}
|
||||
if let Some(why) = empty_attestation(&item.source) {
|
||||
return Err(malformed(&item.id, why));
|
||||
}
|
||||
check_entities(&item, entities)?;
|
||||
let text = normalized(&item.text);
|
||||
for held in self.all()? {
|
||||
if normalized(&held.text) == text && item.supersedes.as_deref() != Some(&held.id) {
|
||||
return Err(StoreRefusal::S6TextHeld {
|
||||
|
|
@ -360,58 +353,15 @@ fn check_entities(item: &CorpusItem, entities: &EntityStore) -> Result<(), Store
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// The S6 key of a text: Unicode compatibility caseless matching (D146: NFKC,
|
||||
/// default case fold, NFKC) keeping letters, digits and combining marks
|
||||
/// (General_Category L, N, M) so tone marks and viramas keep words apart. Composed and decomposed,
|
||||
/// fullwidth, ligature and case-fold variants (straße, STRASSE) and spacing and
|
||||
/// punctuation variants compare equal. Homoglyphs across scripts (Cyrillic a for
|
||||
/// Latin a) stay apart: no key here reads what a character looks like.
|
||||
/// Lowercased alphanumerics only, so case, spacing and punctuation variants of
|
||||
/// a text compare equal.
|
||||
fn normalized(text: &str) -> String {
|
||||
let folded = default_case_fold_str(&text.nfkc().collect::<String>());
|
||||
folded
|
||||
.nfkc()
|
||||
.filter(|c| c.is_alphanumeric() || is_combining_mark(*c))
|
||||
text.chars()
|
||||
.flat_map(char::to_lowercase)
|
||||
.filter(|c| c.is_alphanumeric())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The field that holds an unattested provenance string, if any: a string with
|
||||
/// no alphanumeric, a control or format character, or over 200 bytes attests
|
||||
/// nothing.
|
||||
fn empty_attestation(source: &Source) -> Option<&'static str> {
|
||||
let empty = |s: &String| !is_attestation(s);
|
||||
let empty_some = |s: &Option<String>| s.as_ref().is_some_and(empty);
|
||||
match source {
|
||||
Source::CastAuthored { persona, .. } if empty(persona) => {
|
||||
Some("a cast-authored source names its persona")
|
||||
}
|
||||
Source::CastAuthored { fixture, .. } if empty(fixture) => {
|
||||
Some("a cast-authored source names its fixture")
|
||||
}
|
||||
Source::CastAuthored { author, .. } if empty(author) => {
|
||||
Some("a cast-authored source names its author")
|
||||
}
|
||||
Source::OperatorApproved { note } if empty(note) => {
|
||||
Some("an operator-approved source carries its note")
|
||||
}
|
||||
Source::PublicDomain { work, .. } if empty(work) => {
|
||||
Some("a public-domain source names its work")
|
||||
}
|
||||
Source::PublicDomain { author, .. } if empty(author) => {
|
||||
Some("a public-domain source names its author")
|
||||
}
|
||||
Source::PublicDomain { edition_url, .. } if empty(edition_url) => {
|
||||
Some("a public-domain source names its edition url")
|
||||
}
|
||||
Source::ModelWritten { generator, .. } if empty_some(generator) => {
|
||||
Some("a model-written source names a non-empty generator")
|
||||
}
|
||||
Source::ModelWritten { prompt_digest, .. } if empty_some(prompt_digest) => {
|
||||
Some("a model-written source carries a non-empty prompt digest")
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub struct EntityStore {
|
||||
cache: CultCache,
|
||||
}
|
||||
|
|
@ -437,9 +387,8 @@ impl EntityStore {
|
|||
/// every stored language and form in order and origin as it was, and only
|
||||
/// appends: forms grow, never shrink, reorder or change. An entity has 1 to
|
||||
/// 8 languages, each a BCP 47-form tag named once, with 1 to 8 NFC forms of
|
||||
/// 1 to 64 bytes, no control or invisible format character (joiners and
|
||||
/// variation selectors stay), no edge whitespace, none repeated; the origin
|
||||
/// attests where it came from with strings that hold an alphanumeric, no control or format character and at most 200 bytes.
|
||||
/// 1 to 64 bytes and at most 4 words, no control character, no edge
|
||||
/// whitespace, none repeated.
|
||||
pub fn put(&mut self, entity: Entity) -> Result<AddOutcome, StoreError> {
|
||||
let key = entity.id.to_string();
|
||||
check_entity_shape(&entity)?;
|
||||
|
|
@ -472,20 +421,6 @@ fn check_entity_shape(entity: &Entity) -> Result<(), StoreError> {
|
|||
}
|
||||
.into()
|
||||
};
|
||||
let (persona, fixture, work) = match &entity.origin {
|
||||
EntityOrigin::Cast { persona, fixture } => (Some(persona), Some(fixture), None),
|
||||
EntityOrigin::Live { persona } => (Some(persona), None, None),
|
||||
EntityOrigin::PublicDomain { work } => (None, None, Some(work)),
|
||||
};
|
||||
if persona.is_some_and(|s| !is_attestation(s)) {
|
||||
return Err(bad("an origin names its persona"));
|
||||
}
|
||||
if fixture.is_some_and(|s| !is_attestation(s)) {
|
||||
return Err(bad("a cast origin names its fixture"));
|
||||
}
|
||||
if work.is_some_and(|s| !is_attestation(s)) {
|
||||
return Err(bad("a public-domain origin names its work"));
|
||||
}
|
||||
if entity.forms.is_empty() {
|
||||
return Err(bad("an entity has forms in at least one language"));
|
||||
}
|
||||
|
|
@ -511,6 +446,9 @@ fn check_entity_shape(entity: &Entity) -> Result<(), StoreError> {
|
|||
if f.len() > EntityBound::MAX_FORM_BYTES {
|
||||
return Err(bound(EntityBound::FormBytes));
|
||||
}
|
||||
if f.split_whitespace().count() > EntityBound::MAX_FORM_WORDS {
|
||||
return Err(bound(EntityBound::FormWords));
|
||||
}
|
||||
if !is_nfc(f) {
|
||||
return Err(StoreRefusal::NotNfc {
|
||||
key: entity.id.to_string(),
|
||||
|
|
@ -522,12 +460,6 @@ fn check_entity_shape(entity: &Entity) -> Result<(), StoreError> {
|
|||
"a form is not empty and has no control character or edge whitespace",
|
||||
));
|
||||
}
|
||||
if f.chars().any(is_refused_format) {
|
||||
return Err(bad("a form holds an invisible format character"));
|
||||
}
|
||||
if f.chars().all(is_joining_only) {
|
||||
return Err(bad("a form is more than joiners and selectors"));
|
||||
}
|
||||
if !seen.insert(f) {
|
||||
return Err(bad("a form is not repeated within its language"));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
mod common;
|
||||
use std::process::{Command, Output};
|
||||
use weksa::{CorpusStore, EntityStore, InventoryStore, PacketStore};
|
||||
use weksa::{InventoryStore, PacketStore};
|
||||
|
||||
fn weksa(dir: &std::path::Path, args: &[&str]) -> Output {
|
||||
Command::new(env!("CARGO_BIN_EXE_weksa"))
|
||||
|
|
@ -35,8 +35,6 @@ fn cast_entities(dir: &std::path::Path, slugs: &[&str]) {
|
|||
"cast",
|
||||
"--persona",
|
||||
slug,
|
||||
"--fixture",
|
||||
"f.yaml",
|
||||
],
|
||||
);
|
||||
assert!(put.status.success(), "{}", out(&put));
|
||||
|
|
@ -83,7 +81,7 @@ fn seed_add_dump_split_withdraw() {
|
|||
"{dump}"
|
||||
);
|
||||
|
||||
// A cast-authored item needs its persona, fixture and author.
|
||||
// A cast-authored item needs its persona and fixture.
|
||||
let missing = weksa(d.path(), &ADD);
|
||||
assert!(
|
||||
!missing.status.success() && out(&missing).contains("--persona"),
|
||||
|
|
@ -91,14 +89,7 @@ fn seed_add_dump_split_withdraw() {
|
|||
out(&missing)
|
||||
);
|
||||
let mut add = ADD.to_vec();
|
||||
add.extend([
|
||||
"--persona",
|
||||
"nibu",
|
||||
"--fixture",
|
||||
"f.yaml",
|
||||
"--author",
|
||||
"Ada Writer",
|
||||
]);
|
||||
add.extend(["--persona", "nibu", "--fixture", "f.yaml"]);
|
||||
let added = weksa(d.path(), &add);
|
||||
assert!(
|
||||
added.status.success() && out(&added).contains("Added item:nibu:1"),
|
||||
|
|
@ -303,15 +294,7 @@ fn schema_writes_the_published_files() {
|
|||
fn entity_put_and_name_spans() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
let put = |args: &[&str]| {
|
||||
let mut all = vec![
|
||||
"entity",
|
||||
"put",
|
||||
"entity:kay",
|
||||
"--origin",
|
||||
"cast",
|
||||
"--fixture",
|
||||
"f.yaml",
|
||||
];
|
||||
let mut all = vec!["entity", "put", "entity:kay", "--origin", "cast"];
|
||||
all.extend_from_slice(args);
|
||||
weksa(d.path(), &all)
|
||||
};
|
||||
|
|
@ -403,59 +386,24 @@ fn entity_origin_flags_must_match_origin() {
|
|||
weksa(d.path(), &all)
|
||||
};
|
||||
for mismatched in [
|
||||
&["--origin", "live"][..],
|
||||
&["--origin", "live", "--persona", "kay", "--work", "edda"],
|
||||
&[
|
||||
"--origin",
|
||||
"live",
|
||||
"--persona",
|
||||
"kay",
|
||||
"--fixture",
|
||||
"f.yaml",
|
||||
],
|
||||
&["--origin", "cast", "--persona", "kay"],
|
||||
&["--origin", "cast", "--fixture", "f.yaml"],
|
||||
&[
|
||||
"--origin",
|
||||
"cast",
|
||||
"--persona",
|
||||
"kay",
|
||||
"--fixture",
|
||||
"f.yaml",
|
||||
"--work",
|
||||
"edda",
|
||||
],
|
||||
&["--origin", "public-domain"],
|
||||
&["--origin", "live", "--persona", "kay"][..],
|
||||
&["--origin", "live", "--work", "edda"],
|
||||
&["--origin", "cast", "--persona", "kay", "--work", "edda"],
|
||||
&[
|
||||
"--origin",
|
||||
"public-domain",
|
||||
"--work",
|
||||
"edda",
|
||||
"--persona",
|
||||
"kay",
|
||||
],
|
||||
&[
|
||||
"--origin",
|
||||
"public-domain",
|
||||
"--work",
|
||||
"edda",
|
||||
"--fixture",
|
||||
"f.yaml",
|
||||
],
|
||||
&["--origin", "cast", "--persona", "", "--fixture", "f.yaml"],
|
||||
&["--origin", "cast", "--persona", "kay", "--fixture", ""],
|
||||
&["--origin", "live", "--persona", ""],
|
||||
&["--origin", "public-domain", "--work", ""],
|
||||
&["--origin", "public-domain", "--persona", "kay"],
|
||||
] {
|
||||
let o = put(mismatched);
|
||||
assert!(!o.status.success(), "{mismatched:?}: {}", out(&o));
|
||||
}
|
||||
assert!(!out(&weksa(d.path(), &["entity", "dump"])).contains("entity:kay"));
|
||||
assert!(
|
||||
put(&["--origin", "live", "--persona", "kay"])
|
||||
.status
|
||||
.success()
|
||||
);
|
||||
assert!(put(&["--origin", "live"]).status.success());
|
||||
let work = weksa(
|
||||
d.path(),
|
||||
&[
|
||||
|
|
@ -488,8 +436,6 @@ fn intake_normalizes_to_nfc() {
|
|||
"fr=Rene\u{301}",
|
||||
"--origin",
|
||||
"live",
|
||||
"--persona",
|
||||
"rene",
|
||||
],
|
||||
);
|
||||
assert!(put.status.success(), "{}", out(&put));
|
||||
|
|
@ -521,349 +467,3 @@ fn intake_normalizes_to_nfc() {
|
|||
let dump = out(&weksa(d.path(), &["corpus", "dump"]));
|
||||
assert!(dump.contains("Ren\u{e9} part."), "{dump}");
|
||||
}
|
||||
|
||||
/// `corpus add` of "Mind the hatch." by `entity:nibu` under `source`, with
|
||||
/// `extra` flags.
|
||||
fn add_with(dir: &std::path::Path, source: &str, extra: &[&str]) -> Output {
|
||||
let mut args = vec![
|
||||
"corpus",
|
||||
"add",
|
||||
"--slug",
|
||||
"nibu",
|
||||
"--text",
|
||||
"Mind the hatch.",
|
||||
"--speaker",
|
||||
"entity:nibu",
|
||||
"--group",
|
||||
"scene-1",
|
||||
"--admitted-on",
|
||||
"2026-10-29",
|
||||
"--source",
|
||||
source,
|
||||
];
|
||||
args.extend_from_slice(extra);
|
||||
weksa(dir, &args)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_flags_must_match_the_source() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
cast_entities(d.path(), &["nibu"]);
|
||||
// A provenance flag that does not apply to --source refuses, naming the flag.
|
||||
for (source, extra, flag) in [
|
||||
("operator-written", &["--persona", "p"][..], "--persona"),
|
||||
(
|
||||
"operator-approved",
|
||||
&["--note", "n", "--work", "w"],
|
||||
"--work",
|
||||
),
|
||||
(
|
||||
"public-domain",
|
||||
&[
|
||||
"--work",
|
||||
"w",
|
||||
"--author",
|
||||
"a",
|
||||
"--edition-url",
|
||||
"u",
|
||||
"--note",
|
||||
"n",
|
||||
],
|
||||
"--note",
|
||||
),
|
||||
(
|
||||
"cast-authored",
|
||||
&[
|
||||
"--persona",
|
||||
"p",
|
||||
"--fixture",
|
||||
"f",
|
||||
"--author",
|
||||
"a",
|
||||
"--generator",
|
||||
"g",
|
||||
],
|
||||
"--generator",
|
||||
),
|
||||
(
|
||||
"model-written",
|
||||
&["--model-family", "unrecorded", "--edition-url", "u"],
|
||||
"--edition-url",
|
||||
),
|
||||
(
|
||||
"operator-written",
|
||||
&["--model-family", "unrecorded"],
|
||||
"--model-family",
|
||||
),
|
||||
(
|
||||
"model-written",
|
||||
&[
|
||||
"--model-family",
|
||||
"unrecorded",
|
||||
"--prompt-digest",
|
||||
"d",
|
||||
"--fixture",
|
||||
"f",
|
||||
],
|
||||
"--fixture",
|
||||
),
|
||||
// A source missing a flag it needs.
|
||||
(
|
||||
"cast-authored",
|
||||
&["--persona", "p", "--fixture", "f"],
|
||||
"--author",
|
||||
),
|
||||
(
|
||||
"public-domain",
|
||||
&["--work", "w", "--edition-url", "u"],
|
||||
"--author",
|
||||
),
|
||||
// Empty provenance strings.
|
||||
(
|
||||
"model-written",
|
||||
&["--model-family", "unrecorded", "--generator", ""],
|
||||
"",
|
||||
),
|
||||
(
|
||||
"model-written",
|
||||
&["--model-family", "unrecorded", "--prompt-digest", ""],
|
||||
"",
|
||||
),
|
||||
(
|
||||
"cast-authored",
|
||||
&["--persona", "", "--fixture", "f", "--author", "a"],
|
||||
"",
|
||||
),
|
||||
("operator-approved", &["--note", ""], ""),
|
||||
] {
|
||||
let o = add_with(d.path(), source, extra);
|
||||
assert!(!o.status.success(), "{source} {extra:?}: {}", out(&o));
|
||||
assert!(out(&o).contains(flag), "{source} {extra:?}: {}", out(&o));
|
||||
}
|
||||
assert!(!out(&weksa(d.path(), &["corpus", "dump"])).contains("item:"));
|
||||
// A source whose flags all apply is admitted; --author reaches a
|
||||
// cast-authored item.
|
||||
let cast = add_with(
|
||||
d.path(),
|
||||
"cast-authored",
|
||||
&["--persona", "p", "--fixture", "f", "--author", "Ada Writer"],
|
||||
);
|
||||
assert!(cast.status.success(), "{}", out(&cast));
|
||||
let dump = out(&weksa(d.path(), &["corpus", "dump"]));
|
||||
assert!(dump.contains("author: \"Ada Writer\""), "{dump}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_intake_never_echoes_its_values() {
|
||||
const CANARY: &str = "CANARY-7f3a";
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
cast_entities(d.path(), &["nibu"]);
|
||||
let shape = |value: &str| format!("{CANARY}{value}");
|
||||
let puts = [
|
||||
(vec![shape("")], "--form"),
|
||||
(vec![shape("_x=Kay")], ""),
|
||||
(vec![format!("en-US={CANARY}\u{7}")], ""),
|
||||
(vec![format!("{CANARY}_x=Kay")], ""),
|
||||
];
|
||||
for (forms, flag) in puts {
|
||||
let mut args = vec![
|
||||
"entity",
|
||||
"put",
|
||||
"entity:kay",
|
||||
"--origin",
|
||||
"live",
|
||||
"--persona",
|
||||
"kay",
|
||||
];
|
||||
for f in &forms {
|
||||
args.extend(["--form", f]);
|
||||
}
|
||||
let o = weksa(d.path(), &args);
|
||||
assert!(!o.status.success(), "{forms:?}: {}", out(&o));
|
||||
assert!(!out(&o).contains(CANARY), "{forms:?}: {}", out(&o));
|
||||
assert!(out(&o).contains(flag), "{forms:?}: {}", out(&o));
|
||||
}
|
||||
let bad_turn = [shape(""), format!("{CANARY}=hello")];
|
||||
let bad_name = [
|
||||
shape(""),
|
||||
format!("0..1={CANARY}"),
|
||||
format!("{CANARY}..1=entity:nibu"),
|
||||
format!("0..{CANARY}=entity:nibu"),
|
||||
];
|
||||
let cases = bad_turn
|
||||
.iter()
|
||||
.map(|t| ("--prior-turn", t))
|
||||
.chain(bad_name.iter().map(|n| ("--name", n)));
|
||||
for (flag, value) in cases {
|
||||
let o = add_with(d.path(), "operator-written", &[flag, value]);
|
||||
assert!(!o.status.success(), "{flag} {value}: {}", out(&o));
|
||||
assert!(!out(&o).contains(CANARY), "{flag} {value}: {}", out(&o));
|
||||
assert!(out(&o).contains(flag), "{flag} {value}: {}", out(&o));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_decomposed_form_and_turn_is_stored_composed() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
cast_entities(d.path(), &["nibu"]);
|
||||
let put = weksa(
|
||||
d.path(),
|
||||
&[
|
||||
"entity",
|
||||
"put",
|
||||
"entity:rene",
|
||||
"--form",
|
||||
"fr=Rene\u{301}",
|
||||
"--form",
|
||||
"fr=Zoe\u{308}",
|
||||
"--origin",
|
||||
"live",
|
||||
"--persona",
|
||||
"rene",
|
||||
],
|
||||
);
|
||||
assert!(put.status.success(), "{}", out(&put));
|
||||
let rene = EntityStore::open(&d.path().join("e.cc"))
|
||||
.unwrap()
|
||||
.get(&"entity:rene".parse().unwrap())
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(rene.forms[0].forms, ["Ren\u{e9}", "Zo\u{eb}"]);
|
||||
let add = add_with(
|
||||
d.path(),
|
||||
"operator-written",
|
||||
&["--prior-turn", "entity:rene=Cafe\u{301} now"],
|
||||
);
|
||||
assert!(add.status.success(), "{}", out(&add));
|
||||
let item = CorpusStore::open(&d.path().join("c.cc"))
|
||||
.unwrap()
|
||||
.get("item:nibu:1")
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(item.prior_turns[0].text, "Caf\u{e9} now");
|
||||
}
|
||||
|
||||
/// An Ollama stand-in: every text embeds to `[1, its length]`, so identical
|
||||
/// texts have cosine 1.
|
||||
fn fake_embedder() -> String {
|
||||
use std::io::{Read, Write};
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let url = format!("http://{}", listener.local_addr().unwrap());
|
||||
std::thread::spawn(move || {
|
||||
for mut stream in listener.incoming().flatten() {
|
||||
let mut buf = vec![0u8; 65536];
|
||||
let mut n = 0;
|
||||
let body = loop {
|
||||
n += stream.read(&mut buf[n..]).unwrap();
|
||||
let text = String::from_utf8_lossy(&buf[..n]).into_owned();
|
||||
if let Some((head, body)) = text.split_once("\r\n\r\n") {
|
||||
let want: usize = head
|
||||
.to_lowercase()
|
||||
.split("content-length: ")
|
||||
.nth(1)
|
||||
.unwrap()
|
||||
.split("\r\n")
|
||||
.next()
|
||||
.unwrap()
|
||||
.parse()
|
||||
.unwrap();
|
||||
if body.len() >= want {
|
||||
break body.to_string();
|
||||
}
|
||||
}
|
||||
};
|
||||
let body: serde_json::Value = serde_json::from_str(&body).unwrap();
|
||||
let vectors: Vec<Vec<f64>> = body["input"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|t| vec![1.0, t.as_str().unwrap().len() as f64])
|
||||
.collect();
|
||||
let reply = serde_json::json!({ "embeddings": vectors }).to_string();
|
||||
write!(
|
||||
stream,
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{reply}",
|
||||
reply.len()
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
});
|
||||
url
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn proof_packet_lowers_and_the_lowering_scores_against_items() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
cast_entities(d.path(), &["nibu"]);
|
||||
for args in [&["inventory", "seed"][..], &["inventory", "proof"]] {
|
||||
let run = weksa(d.path(), args);
|
||||
assert!(run.status.success(), "{}", out(&run));
|
||||
}
|
||||
let item = weksa(
|
||||
d.path(),
|
||||
&[
|
||||
"corpus",
|
||||
"add",
|
||||
"--slug",
|
||||
"nibu",
|
||||
"--text",
|
||||
"Warning: you may endanger yourself by touching the control. The ship mind will prevent that to protect you.",
|
||||
"--speaker",
|
||||
"entity:nibu",
|
||||
"--group",
|
||||
"scene-1",
|
||||
"--admitted-on",
|
||||
"2026-11-01",
|
||||
"--source",
|
||||
"operator-written",
|
||||
],
|
||||
);
|
||||
assert!(item.status.success(), "{}", out(&item));
|
||||
let admitted = weksa(d.path(), &["packet", "proof", "./p.cc"]);
|
||||
assert!(
|
||||
admitted.status.success() && out(&admitted).contains("admitted packet:6e6962752d303031"),
|
||||
"{}",
|
||||
out(&admitted)
|
||||
);
|
||||
let lowered = weksa(d.path(), &["lower", "./p.cc", "packet:6e6962752d303031"]);
|
||||
assert!(lowered.status.success(), "{}", out(&lowered));
|
||||
let text = String::from_utf8(lowered.stdout).unwrap();
|
||||
assert!(
|
||||
text.starts_with("Warning: you may endanger yourself"),
|
||||
"{text}"
|
||||
);
|
||||
|
||||
let missing = weksa(d.path(), &["lower", "./p.cc", "packet:0000000000000000"]);
|
||||
assert!(!missing.status.success() && out(&missing).contains("no packet"));
|
||||
|
||||
std::fs::write(d.path().join("out.txt"), &text).unwrap();
|
||||
let url = fake_embedder();
|
||||
let scored = weksa(
|
||||
d.path(),
|
||||
&[
|
||||
"score",
|
||||
"item:nibu:1",
|
||||
"--output",
|
||||
"out.txt",
|
||||
"--ollama-url",
|
||||
&url,
|
||||
],
|
||||
);
|
||||
assert!(scored.status.success(), "{}", out(&scored));
|
||||
assert_eq!(
|
||||
out(&scored).trim(),
|
||||
"item:nibu:1 chrf=100.00 meaning=1.0000"
|
||||
);
|
||||
let unknown = weksa(
|
||||
d.path(),
|
||||
&[
|
||||
"score",
|
||||
"item:nibu:9",
|
||||
"--output",
|
||||
"out.txt",
|
||||
"--ollama-url",
|
||||
&url,
|
||||
],
|
||||
);
|
||||
assert!(!unknown.status.success() && out(&unknown).contains("no item item:nibu:9"));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -67,7 +67,6 @@ pub fn cast() -> Source {
|
|||
Source::CastAuthored {
|
||||
persona: "nibu".into(),
|
||||
fixture: "f.yaml".into(),
|
||||
author: "Ada Writer".into(),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -96,7 +95,6 @@ pub fn ent(slug: &str, language: &str, forms: &[&str]) -> Entity {
|
|||
}],
|
||||
origin: EntityOrigin::Cast {
|
||||
persona: slug.into(),
|
||||
fixture: "f.yaml".into(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -105,11 +105,7 @@ fn entity_forms_only_grow() {
|
|||
},
|
||||
|e| e.forms.reverse(),
|
||||
|e| e.forms[1].language = "fi".into(),
|
||||
|e| {
|
||||
e.origin = EntityOrigin::Live {
|
||||
persona: "kay".into(),
|
||||
}
|
||||
},
|
||||
|e| e.origin = EntityOrigin::Live,
|
||||
];
|
||||
assert_eq!(
|
||||
refusal(store.put(ent("kay", "en-US", &["Kay"]))),
|
||||
|
|
|
|||
|
|
@ -163,33 +163,28 @@ fn entity_capacity_is_bounded() {
|
|||
let make = |forms: Vec<LanguageForms>| Entity {
|
||||
id: eid("big"),
|
||||
forms,
|
||||
origin: EntityOrigin::Live {
|
||||
persona: "p".into(),
|
||||
},
|
||||
origin: EntityOrigin::Live,
|
||||
};
|
||||
|
||||
// A form is bounded by bytes, not words: a spaced name of any word count and
|
||||
// names in scripts that write no spaces are admitted (the Thai name counts
|
||||
// five UAX #29 words, a Han name one per character).
|
||||
for (i, name) in [
|
||||
"I will never leave you, not ever.",
|
||||
"Mary of the Isles too",
|
||||
"de la Cruz y Garcia",
|
||||
"\u{e2a}\u{e21}\u{e0a}\u{e32}\u{e22} \u{e43}\u{e08}\u{e14}\u{e35}",
|
||||
"\u{5c71}\u{7530}\u{592a}\u{90ce}",
|
||||
]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
w.entities
|
||||
.put(Entity {
|
||||
id: eid(&format!("name-{i}")),
|
||||
..make(vec![lf("en-US", one(name))])
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
// 64 bytes is the edge; 65 is over, counted in bytes (21 Han characters are
|
||||
// 63 bytes, 22 are 66).
|
||||
// A whole line is not a name: 7 words.
|
||||
let line = "I will never leave you, not ever.";
|
||||
let e = make(vec![lf("en-US", one(line))]);
|
||||
assert_eq!(
|
||||
refusal(w.entities.put(e.clone())),
|
||||
bounds(&e, EntityBound::FormWords)
|
||||
);
|
||||
// Four words is the edge; five is over. 64 bytes is the edge; 65 is over.
|
||||
w.entities
|
||||
.put(Entity {
|
||||
id: eid("four"),
|
||||
..make(vec![lf("en-US", one("Mary of the Isles"))])
|
||||
})
|
||||
.unwrap();
|
||||
let five = make(vec![lf("en-US", one("Mary of the Isles too"))]);
|
||||
assert_eq!(
|
||||
refusal(w.entities.put(five.clone())),
|
||||
bounds(&five, EntityBound::FormWords)
|
||||
);
|
||||
let wide = make(vec![lf("en-US", one(&"a".repeat(65)))]);
|
||||
assert_eq!(
|
||||
refusal(w.entities.put(wide.clone())),
|
||||
|
|
@ -201,17 +196,6 @@ fn entity_capacity_is_bounded() {
|
|||
..make(vec![lf("en-US", one(&"a".repeat(64)))])
|
||||
})
|
||||
.unwrap();
|
||||
w.entities
|
||||
.put(Entity {
|
||||
id: eid("han-edge"),
|
||||
..make(vec![lf("ja", one(&"\u{5c71}".repeat(21)))])
|
||||
})
|
||||
.unwrap();
|
||||
let han = make(vec![lf("ja", one(&"\u{5c71}".repeat(22)))]);
|
||||
assert_eq!(
|
||||
refusal(w.entities.put(han.clone())),
|
||||
bounds(&han, EntityBound::FormBytes)
|
||||
);
|
||||
|
||||
// 8 forms per language is the edge.
|
||||
let forms = |n: usize| (0..n).map(|i| format!("form{i}")).collect::<Vec<_>>();
|
||||
|
|
@ -337,7 +321,6 @@ fn entities_are_append_only_and_replay_needs_them() {
|
|||
let mut persona = base.clone();
|
||||
persona.origin = EntityOrigin::Cast {
|
||||
persona: "someone-else".into(),
|
||||
fixture: "f.yaml".into(),
|
||||
};
|
||||
assert_eq!(refusal(w.entities.put(persona)), not_append);
|
||||
let mut duplicate = base.clone();
|
||||
|
|
@ -363,276 +346,3 @@ fn entities_are_append_only_and_replay_needs_them() {
|
|||
}
|
||||
);
|
||||
}
|
||||
|
||||
/// One entity holding `form` as its only en-US form.
|
||||
fn with_form(slug: &str, form: &str) -> Entity {
|
||||
Entity {
|
||||
id: eid(slug),
|
||||
forms: vec![LanguageForms {
|
||||
language: "en-US".into(),
|
||||
forms: vec![form.into()],
|
||||
}],
|
||||
origin: EntityOrigin::Live {
|
||||
persona: "p".into(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invisible_format_characters_are_refused_in_forms() {
|
||||
let mut w = world();
|
||||
let hidden = |slug: &str| StoreRefusal::Malformed {
|
||||
key: format!("entity:{slug}"),
|
||||
why: "a form holds an invisible format character",
|
||||
};
|
||||
let at = |code: u32| char::from_u32(code).unwrap();
|
||||
// Every Cf code point except the joiners, plus U+2028 and U+2029, listed
|
||||
// from Python's unicodedata. The code-point sweep in form.rs pins the edges
|
||||
// of the rule; this checks the door applies it, in each position.
|
||||
let listed: Vec<u32> = include_str!("format_refused.txt")
|
||||
.lines()
|
||||
.filter(|l| !l.starts_with('#'))
|
||||
.map(|l| u32::from_str_radix(l, 16).unwrap())
|
||||
.collect();
|
||||
assert_eq!(listed.len(), 170);
|
||||
for (n, code) in listed.iter().enumerate() {
|
||||
let c = at(*code);
|
||||
let mut forms = vec![format!("Ka{c}y")];
|
||||
if !c.is_whitespace() {
|
||||
forms.extend([format!("{c}Kay"), format!("Kay{c}")]);
|
||||
}
|
||||
for form in forms {
|
||||
assert_eq!(
|
||||
refusal(w.entities.put(with_form("fmt", &form))),
|
||||
hidden("fmt"),
|
||||
"U+{code:X} (#{n}) in {form:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
// Default_Ignorable fillers that are not Cf: Hangul and Khmer.
|
||||
for code in [0x115f, 0x1160, 0x17b4, 0x17b5, 0x3164, 0xffa0] {
|
||||
assert_eq!(
|
||||
refusal(
|
||||
w.entities
|
||||
.put(with_form("fmt", &format!("Ka{}y", at(code))))
|
||||
),
|
||||
hidden("fmt"),
|
||||
"U+{code:X}"
|
||||
);
|
||||
}
|
||||
assert!(w.entities.get(&eid("fmt")).unwrap().is_none());
|
||||
|
||||
// The neighbours of the refused ranges, ZWNJ, ZWJ, CGJ, variation
|
||||
// selectors and the Mongolian selectors (all Mn) are admitted in a form.
|
||||
let admitted: [u32; 33] = [
|
||||
0xac,
|
||||
0xae,
|
||||
0x34f,
|
||||
0x5ff,
|
||||
0x606,
|
||||
0x61b,
|
||||
0x61d,
|
||||
0x6dc,
|
||||
0x6de,
|
||||
0x70e,
|
||||
0x710,
|
||||
0x88f,
|
||||
0x892,
|
||||
0x8e1,
|
||||
0x8e3,
|
||||
0x180b,
|
||||
0x180c,
|
||||
0x180d,
|
||||
0x180f,
|
||||
0x200a,
|
||||
0x200c,
|
||||
0x200d,
|
||||
0x2010,
|
||||
0x202f,
|
||||
0x2065,
|
||||
0x2070,
|
||||
0xfe00,
|
||||
0xfe0f,
|
||||
0xfeff - 1,
|
||||
0xfffc,
|
||||
0x110be,
|
||||
0x1d172,
|
||||
0xe0100,
|
||||
];
|
||||
for (n, code) in admitted.into_iter().enumerate() {
|
||||
let form = format!("Ka{}y", at(code));
|
||||
w.entities
|
||||
.put(with_form(&format!("ok-{n}"), &form))
|
||||
.unwrap_or_else(|e| panic!("U+{code:X}: {e}"));
|
||||
}
|
||||
// ZWNJ in a Persian form, ZWJ in an emoji sequence, VS16 and a Mongolian
|
||||
// free variation selector beside a letter.
|
||||
for (n, form) in [
|
||||
"\u{645}\u{6cc}\u{200c}\u{62e}\u{648}\u{627}\u{647}\u{645}",
|
||||
"\u{1f469}\u{200d}\u{1f4bb}",
|
||||
"\u{2764}\u{fe0f}",
|
||||
"\u{1820}\u{180b}",
|
||||
"\u{1f600}\u{e0100}",
|
||||
]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
w.entities
|
||||
.put(with_form(&format!("script-{n}"), form))
|
||||
.unwrap();
|
||||
}
|
||||
// A form of only joiners, variation selectors, CGJ or Mongolian selectors
|
||||
// shows nothing and is refused.
|
||||
let empty_looking = |slug: &str| StoreRefusal::Malformed {
|
||||
key: format!("entity:{slug}"),
|
||||
why: "a form is more than joiners and selectors",
|
||||
};
|
||||
for form in [
|
||||
"\u{200d}",
|
||||
"\u{200c}",
|
||||
"\u{200c}\u{200d}",
|
||||
"\u{fe0f}",
|
||||
"\u{fe00}\u{fe0f}",
|
||||
"\u{34f}",
|
||||
"\u{180b}\u{180c}\u{180d}\u{180f}",
|
||||
"\u{e0100}",
|
||||
"\u{e01ef}\u{34f}\u{200d}",
|
||||
] {
|
||||
assert_eq!(
|
||||
refusal(w.entities.put(with_form("blank", form))),
|
||||
empty_looking("blank"),
|
||||
"{form:?}"
|
||||
);
|
||||
}
|
||||
assert!(w.entities.get(&eid("blank")).unwrap().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_origin_attests_where_the_entity_came_from() {
|
||||
let mut w = world();
|
||||
let bad = |slug: &str, why: &'static str| StoreRefusal::Malformed {
|
||||
key: format!("entity:{slug}"),
|
||||
why,
|
||||
};
|
||||
let with = |slug: &str, origin: EntityOrigin| Entity {
|
||||
origin,
|
||||
..with_form(slug, "Kay")
|
||||
};
|
||||
let cast = |persona: &str, fixture: &str| EntityOrigin::Cast {
|
||||
persona: persona.into(),
|
||||
fixture: fixture.into(),
|
||||
};
|
||||
assert_eq!(
|
||||
refusal(w.entities.put(with("a", cast("", "f.yaml")))),
|
||||
bad("a", "an origin names its persona")
|
||||
);
|
||||
assert_eq!(
|
||||
refusal(w.entities.put(with("b", cast("kay", "")))),
|
||||
bad("b", "a cast origin names its fixture")
|
||||
);
|
||||
assert_eq!(
|
||||
refusal(w.entities.put(with(
|
||||
"c",
|
||||
EntityOrigin::Live {
|
||||
persona: String::new()
|
||||
}
|
||||
))),
|
||||
bad("c", "an origin names its persona")
|
||||
);
|
||||
assert_eq!(
|
||||
refusal(w.entities.put(with(
|
||||
"d",
|
||||
EntityOrigin::PublicDomain {
|
||||
work: String::new()
|
||||
}
|
||||
))),
|
||||
bad("d", "a public-domain origin names its work")
|
||||
);
|
||||
for slug in ["a", "b", "c", "d"] {
|
||||
assert!(w.entities.get(&eid(slug)).unwrap().is_none());
|
||||
}
|
||||
w.entities.put(with("e", cast("kay", "f.yaml"))).unwrap();
|
||||
w.entities
|
||||
.put(with(
|
||||
"f",
|
||||
EntityOrigin::PublicDomain {
|
||||
work: "edda".into(),
|
||||
},
|
||||
))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_origin_attestation_holds_something_readable() {
|
||||
let mut w = world();
|
||||
let with = |slug: &str, origin: EntityOrigin| Entity {
|
||||
origin,
|
||||
..with_form(slug, "Kay")
|
||||
};
|
||||
type Build = fn(String) -> EntityOrigin;
|
||||
let origins: [(Build, &str); 4] = [
|
||||
(
|
||||
|v| EntityOrigin::Cast {
|
||||
persona: v,
|
||||
fixture: "f.yaml".into(),
|
||||
},
|
||||
"an origin names its persona",
|
||||
),
|
||||
(
|
||||
|v| EntityOrigin::Cast {
|
||||
persona: "kay".into(),
|
||||
fixture: v,
|
||||
},
|
||||
"a cast origin names its fixture",
|
||||
),
|
||||
(
|
||||
|v| EntityOrigin::Live { persona: v },
|
||||
"an origin names its persona",
|
||||
),
|
||||
(
|
||||
|v| EntityOrigin::PublicDomain { work: v },
|
||||
"a public-domain origin names its work",
|
||||
),
|
||||
];
|
||||
let unreadable = [
|
||||
" ".to_string(),
|
||||
"\t".into(),
|
||||
"\n".into(),
|
||||
"\u{200b}".into(),
|
||||
"\u{200d}".into(),
|
||||
"\0".into(),
|
||||
"--".into(),
|
||||
"ok\u{200b}ok".into(),
|
||||
"ok\u{2028}ok".into(),
|
||||
"ok\0ok".into(),
|
||||
"x".repeat(201),
|
||||
"x".repeat(2 * 1024 * 1024),
|
||||
];
|
||||
let mut n = 0;
|
||||
for (build, why) in &origins {
|
||||
for bad in &unreadable {
|
||||
n += 1;
|
||||
let slug = format!("o-{n}");
|
||||
assert_eq!(
|
||||
refusal(w.entities.put(with(&slug, build(bad.clone())))),
|
||||
StoreRefusal::Malformed {
|
||||
key: format!("entity:{slug}"),
|
||||
why
|
||||
},
|
||||
"{why}: {:?}",
|
||||
bad.chars().take(12).collect::<String>()
|
||||
);
|
||||
assert!(w.entities.get(&eid(&slug)).unwrap().is_none());
|
||||
}
|
||||
}
|
||||
for (m, (build, _)) in origins.iter().enumerate() {
|
||||
for (k, good) in ["x".repeat(200), "two words, 2 more".to_string()]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
w.entities
|
||||
.put(with(&format!("g-{}", 2 * m + k), build(good)))
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,171 +0,0 @@
|
|||
# Cf minus ZWNJ/ZWJ, plus U+2028 and U+2029; Python 3.13 unicodedata 15.1.0 over 0..10FFFF
|
||||
AD
|
||||
600
|
||||
601
|
||||
602
|
||||
603
|
||||
604
|
||||
605
|
||||
61C
|
||||
6DD
|
||||
70F
|
||||
890
|
||||
891
|
||||
8E2
|
||||
180E
|
||||
200B
|
||||
200E
|
||||
200F
|
||||
2028
|
||||
2029
|
||||
202A
|
||||
202B
|
||||
202C
|
||||
202D
|
||||
202E
|
||||
2060
|
||||
2061
|
||||
2062
|
||||
2063
|
||||
2064
|
||||
2066
|
||||
2067
|
||||
2068
|
||||
2069
|
||||
206A
|
||||
206B
|
||||
206C
|
||||
206D
|
||||
206E
|
||||
206F
|
||||
FEFF
|
||||
FFF9
|
||||
FFFA
|
||||
FFFB
|
||||
110BD
|
||||
110CD
|
||||
13430
|
||||
13431
|
||||
13432
|
||||
13433
|
||||
13434
|
||||
13435
|
||||
13436
|
||||
13437
|
||||
13438
|
||||
13439
|
||||
1343A
|
||||
1343B
|
||||
1343C
|
||||
1343D
|
||||
1343E
|
||||
1343F
|
||||
1BCA0
|
||||
1BCA1
|
||||
1BCA2
|
||||
1BCA3
|
||||
1D173
|
||||
1D174
|
||||
1D175
|
||||
1D176
|
||||
1D177
|
||||
1D178
|
||||
1D179
|
||||
1D17A
|
||||
E0001
|
||||
E0020
|
||||
E0021
|
||||
E0022
|
||||
E0023
|
||||
E0024
|
||||
E0025
|
||||
E0026
|
||||
E0027
|
||||
E0028
|
||||
E0029
|
||||
E002A
|
||||
E002B
|
||||
E002C
|
||||
E002D
|
||||
E002E
|
||||
E002F
|
||||
E0030
|
||||
E0031
|
||||
E0032
|
||||
E0033
|
||||
E0034
|
||||
E0035
|
||||
E0036
|
||||
E0037
|
||||
E0038
|
||||
E0039
|
||||
E003A
|
||||
E003B
|
||||
E003C
|
||||
E003D
|
||||
E003E
|
||||
E003F
|
||||
E0040
|
||||
E0041
|
||||
E0042
|
||||
E0043
|
||||
E0044
|
||||
E0045
|
||||
E0046
|
||||
E0047
|
||||
E0048
|
||||
E0049
|
||||
E004A
|
||||
E004B
|
||||
E004C
|
||||
E004D
|
||||
E004E
|
||||
E004F
|
||||
E0050
|
||||
E0051
|
||||
E0052
|
||||
E0053
|
||||
E0054
|
||||
E0055
|
||||
E0056
|
||||
E0057
|
||||
E0058
|
||||
E0059
|
||||
E005A
|
||||
E005B
|
||||
E005C
|
||||
E005D
|
||||
E005E
|
||||
E005F
|
||||
E0060
|
||||
E0061
|
||||
E0062
|
||||
E0063
|
||||
E0064
|
||||
E0065
|
||||
E0066
|
||||
E0067
|
||||
E0068
|
||||
E0069
|
||||
E006A
|
||||
E006B
|
||||
E006C
|
||||
E006D
|
||||
E006E
|
||||
E006F
|
||||
E0070
|
||||
E0071
|
||||
E0072
|
||||
E0073
|
||||
E0074
|
||||
E0075
|
||||
E0076
|
||||
E0077
|
||||
E0078
|
||||
E0079
|
||||
E007A
|
||||
E007B
|
||||
E007C
|
||||
E007D
|
||||
E007E
|
||||
E007F
|
||||
|
|
@ -1,212 +0,0 @@
|
|||
//! The English lowerer against the committed inventory (revision 2) and the
|
||||
//! `nibu-proof-001` packet, and the rules that keep a table row from speaking
|
||||
//! for a meaning it does not carry.
|
||||
|
||||
mod common;
|
||||
use std::path::Path;
|
||||
|
||||
use common::{iv, nd};
|
||||
use weksa::inventory::proof_revision;
|
||||
use weksa::packet::{FeatValue, FeatureValue, Packet, proof_packet};
|
||||
use weksa::{
|
||||
CorpusStore, EntityStore, Inventory, InventoryStore, LowerError, LoweringInput, PacketStore,
|
||||
lower, validate,
|
||||
};
|
||||
|
||||
const PROOF: &str = "Warning: you may endanger yourself by touching the control. \
|
||||
The ship mind will prevent that to protect you.";
|
||||
|
||||
fn committed() -> (Inventory, tempfile::TempDir) {
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
|
||||
let inv = InventoryStore::open(&root.join("data/interlingua/inventory.cc")).unwrap();
|
||||
(inv.inventory().unwrap(), tempfile::tempdir().unwrap())
|
||||
}
|
||||
|
||||
/// A `LoweringInput` has one source, `PacketStore::admit`, so every packet
|
||||
/// under test goes through a throwaway store backed by the committed stores.
|
||||
fn view(packet: &Packet, _inv: &Inventory) -> LoweringInput {
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
|
||||
let inventory = InventoryStore::open(&root.join("data/interlingua/inventory.cc")).unwrap();
|
||||
admitted(packet, &inventory)
|
||||
}
|
||||
|
||||
/// The same door against an inventory store the test built.
|
||||
fn admitted(packet: &Packet, inventory: &InventoryStore) -> LoweringInput {
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
|
||||
let corpus = CorpusStore::open(&root.join("data/corpus/corpus.cc")).unwrap();
|
||||
let entities = EntityStore::open(&root.join("data/entity/entity.cc")).unwrap();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
PacketStore::open(&dir.path().join("p.cc"))
|
||||
.unwrap()
|
||||
.admit(packet.clone(), inventory, &corpus, &entities)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn lowered(packet: &Packet, inv: &Inventory) -> Result<String, LowerError> {
|
||||
lower(&view(packet, inv), inv)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_proof_packet_is_admitted_by_the_store_and_lowers_to_its_sentences() {
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
|
||||
let (inv, dir) = committed();
|
||||
let inventory = InventoryStore::open(&root.join("data/interlingua/inventory.cc")).unwrap();
|
||||
let corpus = CorpusStore::open(&root.join("data/corpus/corpus.cc")).unwrap();
|
||||
let entities = EntityStore::open(&root.join("data/entity/entity.cc")).unwrap();
|
||||
let mut packets = PacketStore::open(&dir.path().join("p.cc")).unwrap();
|
||||
packets
|
||||
.admit(proof_packet(), &inventory, &corpus, &entities)
|
||||
.unwrap();
|
||||
let stored = packets.get("packet:6e6962752d303031").unwrap().unwrap();
|
||||
assert_eq!(lowered(&stored, &inv).unwrap(), PROOF);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_the_addressee_is_you_and_is_reflexive() {
|
||||
let (inv, _dir) = committed();
|
||||
let mut p = proof_packet();
|
||||
p.discourse.addressee = None;
|
||||
assert_eq!(
|
||||
lowered(&p, &inv).unwrap(),
|
||||
"Warning: the player may endanger the player by touching the control. \
|
||||
The ship mind will prevent that to protect the player."
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_speech_act_means_no_opener_and_a_capital() {
|
||||
let (inv, _dir) = committed();
|
||||
let mut p = proof_packet();
|
||||
p.discourse.speech_act = None;
|
||||
assert!(lowered(&p, &inv).unwrap().starts_with("You may endanger"));
|
||||
p.discourse.speech_act = Some(iv("speech_act:1"));
|
||||
assert!(lowered(&p, &inv).unwrap().starts_with("You may endanger"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_packet_carrying_what_the_lowerer_does_not_render_is_refused() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut store = InventoryStore::open(&dir.path().join("i.cc")).unwrap();
|
||||
let (rev1, entries) = weksa::inventory::seed();
|
||||
store.admit_revision(rev1, entries).unwrap();
|
||||
let (rev2, entries) = proof_revision();
|
||||
store.admit_revision(rev2, entries).unwrap();
|
||||
let mut ext = common::entry("extension:1", 3);
|
||||
ext.value_kind = Some(weksa::ValueKind::Grade);
|
||||
common::admit(&mut store, 3, vec![ext]);
|
||||
let inv = store.inventory().unwrap();
|
||||
let discourse = |p: &mut Packet| {
|
||||
p.discourse.features.push(FeatureValue {
|
||||
feature: iv("feature:6"),
|
||||
value: FeatValue::Grade(common::grade(300)),
|
||||
})
|
||||
};
|
||||
let predication = |p: &mut Packet| {
|
||||
p.predications[0].features.push(FeatureValue {
|
||||
feature: iv("feature:1"),
|
||||
value: FeatValue::Option(common::lab("negative")),
|
||||
})
|
||||
};
|
||||
let relation = |p: &mut Packet| {
|
||||
p.relations.push(weksa::packet::Relation {
|
||||
relation: iv("relation:8"),
|
||||
from: nd("n4"),
|
||||
to: nd("n5"),
|
||||
features: vec![],
|
||||
})
|
||||
};
|
||||
let extension = |p: &mut Packet| {
|
||||
p.inventory_rev = 3;
|
||||
p.extensions.push(weksa::packet::Extension {
|
||||
key: iv("extension:1"),
|
||||
value: FeatValue::Grade(common::grade(3)),
|
||||
})
|
||||
};
|
||||
let referent_feature = |p: &mut Packet| {
|
||||
p.referents[0].features.push(FeatureValue {
|
||||
feature: iv("feature:6"),
|
||||
value: FeatValue::Grade(common::grade(1)),
|
||||
})
|
||||
};
|
||||
let no_predication = |p: &mut Packet| p.predications.clear();
|
||||
let extra_role = |p: &mut Packet| {
|
||||
let filler = p.predications[0].roles[0].filler.clone();
|
||||
p.predications[0].roles.push(weksa::packet::RoleFill {
|
||||
role: iv("role:3"),
|
||||
filler,
|
||||
})
|
||||
};
|
||||
let lone_referent = |p: &mut Packet| {
|
||||
let mut r = p.referents[0].clone();
|
||||
r.id = nd("n99");
|
||||
p.referents.push(r)
|
||||
};
|
||||
for (what, edit) in [
|
||||
("features", &discourse as &dyn Fn(&mut Packet)),
|
||||
("features", &predication),
|
||||
("relations", &relation),
|
||||
("extensions", &extension),
|
||||
("no predication", &no_predication),
|
||||
("a role the frame does not name", &extra_role),
|
||||
("a referent no role mentions", &lone_referent),
|
||||
] {
|
||||
let mut p = proof_packet();
|
||||
edit(&mut p);
|
||||
assert_eq!(
|
||||
lower(&admitted(&p, &store), &inv),
|
||||
Err(LowerError::Unrendered { what }),
|
||||
"{what}"
|
||||
);
|
||||
}
|
||||
// The seed lets no feature sit on a referent, so that arm is refused by validate.
|
||||
let mut p = proof_packet();
|
||||
referent_feature(&mut p);
|
||||
assert!(validate(&p, &inv).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unknown_row_and_an_unfilled_role_are_refused_by_name() {
|
||||
let (inv, _dir) = committed();
|
||||
let mut p = proof_packet();
|
||||
p.discourse.speech_act = Some(iv("speech_act:3"));
|
||||
assert_eq!(
|
||||
lowered(&p, &inv),
|
||||
Err(LowerError::NoRow {
|
||||
id: iv("speech_act:3")
|
||||
})
|
||||
);
|
||||
let mut p = proof_packet();
|
||||
p.predications[0].roles.pop();
|
||||
assert_eq!(
|
||||
lowered(&p, &inv),
|
||||
Err(LowerError::MissingRole {
|
||||
predication: nd("n4"),
|
||||
role: "patient".into()
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_row_cannot_speak_for_an_id_whose_gloss_differs() {
|
||||
// Revision 2 with concept:4 glossed "hatch" in place of "control": the row
|
||||
// for concept:4 says "control" and must not render it.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut store = InventoryStore::open(&dir.path().join("i.cc")).unwrap();
|
||||
let (rev1, entries) = weksa::inventory::seed();
|
||||
store.admit_revision(rev1, entries).unwrap();
|
||||
let (rev2, mut entries) = proof_revision();
|
||||
for e in &mut entries {
|
||||
if e.id == iv("concept:4") {
|
||||
e.gloss = "hatch".into();
|
||||
}
|
||||
}
|
||||
store.admit_revision(rev2, entries).unwrap();
|
||||
let inv = store.inventory().unwrap();
|
||||
assert_eq!(
|
||||
lower(&admitted(&proof_packet(), &store), &inv),
|
||||
Err(LowerError::GlossMismatch {
|
||||
id: iv("concept:4"),
|
||||
row: "control"
|
||||
})
|
||||
);
|
||||
}
|
||||
|
|
@ -22,7 +22,6 @@ fn cast() -> Source {
|
|||
Source::CastAuthored {
|
||||
persona: "p".into(),
|
||||
fixture: "f".into(),
|
||||
author: "a".into(),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -144,17 +144,6 @@ fn corpus_keys_are_canonical() {
|
|||
}
|
||||
// An alias of a stored key is not a corpus item id at all.
|
||||
assert!(weksa::CorpusItemId::parse("item:nibu:01").is_err());
|
||||
// The last free number is handed out; past u32::MAX the slug has no free
|
||||
// key, and asking neither wraps nor panics.
|
||||
store
|
||||
.add(item("item:b:4294967294", "one before the last"), &ents(&d))
|
||||
.unwrap();
|
||||
assert_eq!(store.next_key("b").unwrap(), "item:b:4294967295");
|
||||
let full = "a".repeat(48);
|
||||
assert_eq!(
|
||||
refusal(store.next_key(&full)),
|
||||
malformed(&full, "slug has no free key")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -275,322 +264,6 @@ fn corpus_admission_rules() {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn s6_reads_text_under_compatibility_case_folding() {
|
||||
let d = dir();
|
||||
let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap();
|
||||
// Each pair is one text to the key: fullwidth, ligature, superscript and
|
||||
// circled-digit compatibility forms and full case folds (straße, STRASSE).
|
||||
let pairs = [
|
||||
("Brokkr", "\u{ff22}\u{ff52}\u{ff4f}\u{ff4b}\u{ff4b}\u{ff52}"),
|
||||
("fine", "\u{fb01}ne"),
|
||||
("room 2", "room \u{b2}"),
|
||||
("gate 1", "gate \u{2460}"),
|
||||
("stra\u{df}e", "STRASSE"),
|
||||
(
|
||||
"\u{3a3}\u{3af}\u{3c3}\u{3c5}\u{3c6}\u{3bf}\u{3c2}",
|
||||
"\u{3c3}\u{3af}\u{3c3}\u{3c5}\u{3c6}\u{3bf}\u{3c3}",
|
||||
),
|
||||
// The first NFKC step: a compatibility letter folds only after it is
|
||||
// mapped (math-bold S has no case fold of its own; black-letter H is
|
||||
// not a case variant of h until NFKC reads it as H).
|
||||
(
|
||||
"sigrun",
|
||||
"\u{1d412}\u{1d422}\u{1d420}\u{1d42b}\u{1d42e}\u{1d427}",
|
||||
),
|
||||
("h", "\u{210c}"),
|
||||
// The second NFKC step: a full case fold can leave a decomposed or
|
||||
// reordered sequence that only a second NFKC composes to the same key
|
||||
// (dialytika tonos, caron under line, macron below).
|
||||
("\u{390}", "\u{3aa}\u{301}"),
|
||||
("\u{3b0}", "\u{3ab}\u{301}"),
|
||||
("\u{1e96}\u{30c}", "\u{21e}\u{331}"),
|
||||
];
|
||||
for (n, (a, b)) in pairs.into_iter().enumerate() {
|
||||
let (first, second) = (
|
||||
format!("item:s6:{}", 2 * n + 1),
|
||||
format!("item:s6:{}", 2 * n + 2),
|
||||
);
|
||||
store.add(item(&first, a), &ents(&d)).unwrap();
|
||||
assert_eq!(
|
||||
refusal(store.add(item(&second, b), &ents(&d))),
|
||||
StoreRefusal::S6TextHeld {
|
||||
key: second.clone(),
|
||||
held_by: first.clone()
|
||||
},
|
||||
"{a:?} vs {b:?}"
|
||||
);
|
||||
assert!(store.get(&second).unwrap().is_none());
|
||||
}
|
||||
// A decomposed text never reaches the key: the door refuses it as not NFC.
|
||||
assert_eq!(
|
||||
refusal(store.add(item("item:s6:90", "Bro\u{308}kkr"), &ents(&d))),
|
||||
StoreRefusal::NotNfc {
|
||||
key: "item:s6:90".into()
|
||||
}
|
||||
);
|
||||
// Cyrillic a is not Latin a: homoglyphs across scripts stay apart.
|
||||
store.add(item("item:s6:91", "cat"), &ents(&d)).unwrap();
|
||||
store
|
||||
.add(item("item:s6:92", "c\u{430}t"), &ents(&d))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn s6_keeps_tone_marks_and_viramas_apart() {
|
||||
let d = dir();
|
||||
let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap();
|
||||
// Minimal pairs that differ only by a combining mark: Thai mai tho and mai
|
||||
// ek on one word, Thai tone on a short vowel, a Devanagari virama.
|
||||
let pairs = [
|
||||
("\u{e19}\u{e49}\u{e33}", "\u{e19}\u{e33}"),
|
||||
("\u{e44}\u{e21}\u{e49}", "\u{e44}\u{e21}\u{e48}"),
|
||||
("\u{915}\u{94d}\u{92f}", "\u{915}\u{92f}"),
|
||||
];
|
||||
for (n, (a, b)) in pairs.into_iter().enumerate() {
|
||||
store
|
||||
.add(item(&format!("item:mk:{}", 2 * n + 1), a), &ents(&d))
|
||||
.unwrap_or_else(|e| panic!("{a:?}: {e}"));
|
||||
store
|
||||
.add(item(&format!("item:mk:{}", 2 * n + 2), b), &ents(&d))
|
||||
.unwrap_or_else(|e| panic!("{b:?} held by {a:?}: {e}"));
|
||||
}
|
||||
// A mark alone is still no text.
|
||||
for (n, text) in ["\u{e49}", "\u{94d}"].into_iter().enumerate() {
|
||||
let key = format!("item:mk:{}", 90 + n);
|
||||
assert_eq!(
|
||||
refusal(store.add(item(&key, text), &ents(&d))),
|
||||
malformed(&key, "text has no alphanumeric character")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_attestation_holds_something_readable() {
|
||||
let d = dir();
|
||||
let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap();
|
||||
let s = |x: &str| x.to_string();
|
||||
type Build = fn(String) -> Source;
|
||||
let builders: [(Build, &'static str); 9] = [
|
||||
(
|
||||
|v| Source::CastAuthored {
|
||||
persona: v,
|
||||
fixture: "f".into(),
|
||||
author: "a".into(),
|
||||
},
|
||||
"a cast-authored source names its persona",
|
||||
),
|
||||
(
|
||||
|v| Source::CastAuthored {
|
||||
persona: "p".into(),
|
||||
fixture: v,
|
||||
author: "a".into(),
|
||||
},
|
||||
"a cast-authored source names its fixture",
|
||||
),
|
||||
(
|
||||
|v| Source::CastAuthored {
|
||||
persona: "p".into(),
|
||||
fixture: "f".into(),
|
||||
author: v,
|
||||
},
|
||||
"a cast-authored source names its author",
|
||||
),
|
||||
(
|
||||
|v| Source::OperatorApproved { note: v },
|
||||
"an operator-approved source carries its note",
|
||||
),
|
||||
(
|
||||
|v| Source::PublicDomain {
|
||||
work: v,
|
||||
author: "a".into(),
|
||||
edition_url: "u".into(),
|
||||
},
|
||||
"a public-domain source names its work",
|
||||
),
|
||||
(
|
||||
|v| Source::PublicDomain {
|
||||
work: "w".into(),
|
||||
author: v,
|
||||
edition_url: "u".into(),
|
||||
},
|
||||
"a public-domain source names its author",
|
||||
),
|
||||
(
|
||||
|v| Source::PublicDomain {
|
||||
work: "w".into(),
|
||||
author: "a".into(),
|
||||
edition_url: v,
|
||||
},
|
||||
"a public-domain source names its edition url",
|
||||
),
|
||||
(
|
||||
|v| Source::ModelWritten {
|
||||
family: ModelFamily::Unrecorded,
|
||||
generator: Some(v),
|
||||
prompt_digest: None,
|
||||
},
|
||||
"a model-written source names a non-empty generator",
|
||||
),
|
||||
(
|
||||
|v| Source::ModelWritten {
|
||||
family: ModelFamily::Unrecorded,
|
||||
generator: None,
|
||||
prompt_digest: Some(v),
|
||||
},
|
||||
"a model-written source carries a non-empty prompt digest",
|
||||
),
|
||||
];
|
||||
let unreadable = [
|
||||
s(" "),
|
||||
s("\t"),
|
||||
s("\n"),
|
||||
s("\u{200b}"),
|
||||
s("\u{200d}"),
|
||||
s("\0"),
|
||||
s("--"),
|
||||
s("\u{e49}"),
|
||||
s("ok\u{200b}ok"),
|
||||
s("ok\u{200d}ok"),
|
||||
s("ok\u{2028}ok"),
|
||||
s("ok\nok"),
|
||||
s("ok\0ok"),
|
||||
"x".repeat(201),
|
||||
"x".repeat(2 * 1024 * 1024),
|
||||
];
|
||||
let mut n = 0;
|
||||
for (build, why) in &builders {
|
||||
for bad in &unreadable {
|
||||
n += 1;
|
||||
let key = format!("item:att:{n}");
|
||||
let refused = item_in(&key, &format!("line {n}"), "scene-1", build(bad.clone()));
|
||||
assert_eq!(
|
||||
refusal(store.add(refused, &ents(&d))),
|
||||
malformed(&key, why),
|
||||
"{why}: {:?}",
|
||||
bad.chars().take(12).collect::<String>()
|
||||
);
|
||||
}
|
||||
}
|
||||
assert!(store.all().unwrap().is_empty());
|
||||
// 200 bytes and plain words with spaces attest; 201 does not.
|
||||
for (m, (build, _)) in builders.iter().enumerate() {
|
||||
for (k, good) in ["x".repeat(200), s("two words, 2 more")]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
let key = format!("item:att-ok:{}", 2 * m + k + 1);
|
||||
store
|
||||
.add(
|
||||
item_in(&key, &format!("fine {m} {k}"), "scene-1", build(good)),
|
||||
&ents(&d),
|
||||
)
|
||||
.unwrap_or_else(|e| panic!("{key}: {e}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_text_with_no_alphanumeric_character_is_refused() {
|
||||
let d = dir();
|
||||
let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap();
|
||||
for (n, text) in ["...", "?!", " ", "\u{2014}\u{2014}", "\u{ff01}"]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
let key = format!("item:p:{}", n + 1);
|
||||
assert_eq!(
|
||||
refusal(store.add(item(&key, text), &ents(&d))),
|
||||
malformed(&key, "text has no alphanumeric character"),
|
||||
"{text:?}"
|
||||
);
|
||||
}
|
||||
assert!(store.all().unwrap().is_empty());
|
||||
store.add(item("item:p:1", "a..."), &ents(&d)).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_item_source_attests_with_non_empty_strings() {
|
||||
let d = dir();
|
||||
let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap();
|
||||
let s = |x: &str| x.to_string();
|
||||
let cast = |persona: &str, fixture: &str, author: &str| Source::CastAuthored {
|
||||
persona: s(persona),
|
||||
fixture: s(fixture),
|
||||
author: s(author),
|
||||
};
|
||||
let model = |generator: Option<&str>, digest: Option<&str>| Source::ModelWritten {
|
||||
family: ModelFamily::Unrecorded,
|
||||
generator: generator.map(s),
|
||||
prompt_digest: digest.map(s),
|
||||
};
|
||||
let public = |work: &str, author: &str, url: &str| Source::PublicDomain {
|
||||
work: s(work),
|
||||
author: s(author),
|
||||
edition_url: s(url),
|
||||
};
|
||||
let cases: [(Source, &'static str); 9] = [
|
||||
(
|
||||
cast("", "f", "a"),
|
||||
"a cast-authored source names its persona",
|
||||
),
|
||||
(
|
||||
cast("p", "", "a"),
|
||||
"a cast-authored source names its fixture",
|
||||
),
|
||||
(
|
||||
cast("p", "f", ""),
|
||||
"a cast-authored source names its author",
|
||||
),
|
||||
(
|
||||
Source::OperatorApproved { note: s("") },
|
||||
"an operator-approved source carries its note",
|
||||
),
|
||||
(
|
||||
public("", "a", "u"),
|
||||
"a public-domain source names its work",
|
||||
),
|
||||
(
|
||||
public("w", "", "u"),
|
||||
"a public-domain source names its author",
|
||||
),
|
||||
(
|
||||
public("w", "a", ""),
|
||||
"a public-domain source names its edition url",
|
||||
),
|
||||
(
|
||||
model(Some(""), None),
|
||||
"a model-written source names a non-empty generator",
|
||||
),
|
||||
(
|
||||
model(None, Some("")),
|
||||
"a model-written source carries a non-empty prompt digest",
|
||||
),
|
||||
];
|
||||
for (n, (source, why)) in cases.into_iter().enumerate() {
|
||||
let key = format!("item:prov:{}", n + 1);
|
||||
let refused = item_in(&key, &format!("line {n}"), "scene-1", source);
|
||||
assert_eq!(refusal(store.add(refused, &ents(&d))), malformed(&key, why));
|
||||
}
|
||||
assert!(store.all().unwrap().is_empty());
|
||||
// The same sources with every string present are admitted.
|
||||
for (n, source) in [
|
||||
cast("p", "f", "a"),
|
||||
Source::OperatorApproved { note: s("n") },
|
||||
Source::OperatorWritten,
|
||||
public("w", "a", "u"),
|
||||
model(Some("g"), Some("d")),
|
||||
model(None, None),
|
||||
]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
let key = format!("item:prov:{}", n + 1);
|
||||
let item = item_in(&key, &format!("line {n}"), "scene-1", source);
|
||||
assert_eq!(store.add(item, &ents(&d)).unwrap(), AddOutcome::Added);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn s5_supersede_keeps_the_group() {
|
||||
let d = dir();
|
||||
|
|
@ -1233,23 +906,6 @@ fn the_committed_stores_load_and_hold_clean_data() {
|
|||
"no item claims a recorded human author"
|
||||
);
|
||||
assert_eq!(items.len(), 3);
|
||||
// Every committed entity attests, with non-empty strings, where it came
|
||||
// from, and a cast entity's fixture is a file of this repository.
|
||||
let entities = EntityStore::open(&root.join("data/entity/entity.cc"))
|
||||
.unwrap()
|
||||
.all()
|
||||
.unwrap();
|
||||
assert!(!entities.is_empty());
|
||||
for e in &entities {
|
||||
match &e.origin {
|
||||
weksa::entity::EntityOrigin::Cast { persona, fixture } => {
|
||||
assert!(!persona.is_empty(), "{}", e.id);
|
||||
assert!(root.join(fixture).is_file(), "{}: {fixture}", e.id);
|
||||
}
|
||||
weksa::entity::EntityOrigin::Live { persona } => assert!(!persona.is_empty()),
|
||||
weksa::entity::EntityOrigin::PublicDomain { work } => assert!(!work.is_empty()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -1270,16 +926,14 @@ fn seed_reseeded() {
|
|||
.all()
|
||||
.unwrap();
|
||||
assert_eq!(items.len(), 3);
|
||||
// The committed inventory is the seed, reseeded (stance features in, the
|
||||
// displayed-affect features out), then the nibu-proof revision.
|
||||
// The committed inventory is the seed, reseeded: stance features in, the
|
||||
// displayed-affect features out.
|
||||
let committed = InventoryStore::open(&root.join("data/interlingua/inventory.cc"))
|
||||
.unwrap()
|
||||
.inventory()
|
||||
.unwrap();
|
||||
let (rev, mut entries) = seed();
|
||||
let (proof, proof_entries) = weksa::inventory::proof_revision();
|
||||
assert!(committed.has_revision(rev.n) && committed.has_revision(proof.n));
|
||||
entries.extend(proof_entries);
|
||||
let (rev, entries) = seed();
|
||||
assert!(committed.has_revision(rev.n));
|
||||
let mut have: Vec<_> = committed.entries().cloned().collect();
|
||||
let mut want = entries;
|
||||
have.sort_by_key(|e| e.id);
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
“²cultcache.store.v1‘—¬weksa.entity¬weksa.entity¯weksa.entity.v1¬weksa.entityÙL{"schemaName":"weksa.entity","schemaVersion":"weksa.entity.v1","members":[]}‘¬weksa.entity<EFBFBD>’”entity:brokkr¬weksa.entity´2026-10-29T16:43:28ZÄn“entity:brokkr‘’¥en-US‘¦Brokkr<EFBFBD>¤Cast’¦brokkrÙ?examples/persona-lowering/brokkr-edda-voice/lowered-output.yaml”«entity:nibu¬weksa.entity´2026-10-29T16:43:28ZÄe“«entity:nibu‘’¥en-US‘¤Nibu<EFBFBD>¤Cast’¤nibuÙ<examples/english-lowering/nibu-proof-001/lowered-output.yaml
|
||||
“²cultcache.store.v1‘—¬weksa.entity¬weksa.entity¯weksa.entity.v1¬weksa.entityÙL{"schemaName":"weksa.entity","schemaVersion":"weksa.entity.v1","members":[]}‘¬weksa.entity<EFBFBD>’”entity:brokkr¬weksa.entity´2026-10-29T14:59:18ZÄ-“entity:brokkr‘’¥en-US‘¦Brokkr<EFBFBD>¤Cast‘¦brokkr”«entity:nibu¬weksa.entity´2026-10-29T14:59:18ZÄ'“«entity:nibu‘’¥en-US‘¤Nibu<EFBFBD>¤Cast‘¤nibu
|
||||
Binary file not shown.
|
|
@ -29,15 +29,15 @@ means no store holds it and nothing writes it.
|
|||
|
||||
| Kind | What names it | Lifecycle | Who decides |
|
||||
|---|---|---|---|
|
||||
| Packet | Key `packet:<16 hex>`, an opaque digest of item, inventory revision and encoder (the item key lives in `provenance.corpus_item`, which the lowerer never sees); CultCache type `weksa.interlingua_packet`, carrying `schema_version` and `inventory_rev`. Every id inside is an opaque form type (`n<k>` nodes, `encoder:<16 hex>`, canonical `<family>:<n>`). From cut `speaker-state` (schema v4) it also carries the utterance **stance**: the speaker's felt affect and restraint, as inventory features of slot `Stance` (see Speaker state below) | Minted by an encoder for one corpus item under one spec version; immutable; a new spec version means a new packet, never an edit. Admitted only through `PacketStore::admit`, which validates against the inventory store at the packet's revision | The encoder writes; the form types refuse malformed ids at construction and load, the validator refuses broken references and over-capacity packets: a feature at most once per slot, at most 64 nodes (`typed-packet`); nothing else writes packets |
|
||||
| Packet | Key `packet:<16 hex>`, an opaque digest of item, inventory revision and encoder (the item key lives in `provenance.corpus_item`, which the lowerer never sees); CultCache type `weksa.interlingua_packet`, carrying `schema_version` and `inventory_rev`. Every id inside is an opaque form type (`n<k>` nodes, `encoder:<16 hex>`, canonical `<family>:<n>`). From cut `speaker-state` (schema v2) it also carries the utterance **stance**: the speaker's felt affect and restraint, as inventory features of slot `Stance` (see Speaker state below) | Minted by an encoder for one corpus item under one spec version; immutable; a new spec version means a new packet, never an edit. Admitted only through `PacketStore::admit`, which validates against the inventory store at the packet's revision | The encoder writes; the form types refuse malformed ids at construction and load, the validator refuses broken references and over-capacity packets: a feature at most once per slot, at most 64 nodes (`typed-packet`); nothing else writes packets |
|
||||
| Schema version | `weksa.interlingua_packet.v<N>` constant in `crates/weksa`; published JSON Schema at `schemas/cultnet/` | Bumped by a commit that changes the Rust types; the published schema is regenerated in the same commit and a test fails on drift | A cut (Hands, under a spec); never the loop |
|
||||
| Inventory entry | Key `<family>:<n>`, `n` canonical decimal >= 1 (families: concept, frame, role, relation, speech_act, feature, referent_kind, extension); type `weksa.inventory_entry`, with a bounded gloss, value kind (option labels bounded), the slots a feature may sit in (`Discourse`, `Referent`, `Predication`, `Relation`; `Stance` from cut `speaker-state`, `Speaker` and `Scene` from cut `speaker-scene`), `introduced_rev` and optional `retired_rev`. Speaker axes, scene relations and affects are features of those slots, so the loop grows them like any closed-class family and convergence covers them | Added or retired by an inventory revision; never edited in place (a changed meaning is a retire plus an add); a retired id still resolves for packets of earlier revisions | The loop's accepted rounds, whose minting call the context builder assembles from Train items only; Self on merge to main. Admission checks form, not text |
|
||||
| Inventory entry | Key `<family>:<n>`, `n` canonical decimal >= 1 (families: concept, frame, role, relation, speech_act, feature, referent_kind, extension); type `weksa.inventory_entry`, with a bounded gloss, value kind (option labels bounded), the slots a feature may sit in (`Discourse`, `Referent`, `Predication`, `Relation`; from cut `speaker-state` also `Stance`, `Speaker`, `Scene`), `introduced_rev` and optional `retired_rev`. Speaker axes, scene relations and affects are features of those slots, so the loop grows them like any closed-class family and convergence covers them | Added or retired by an inventory revision; never edited in place (a changed meaning is a retire plus an add); a retired id still resolves for packets of earlier revisions | The loop's accepted rounds, whose minting call the context builder assembles from Train items only; Self on merge to main. Admission checks form, not text |
|
||||
| Inventory revision | Key `rev:<n>`, type `weksa.inventory_revision`, with parent, change list and `frozen` | Appended per accepted round; exactly one revision is ever marked frozen, by the convergence test | The loop's accept rule appends; the convergence test freezes; Self records the freeze as a ruling |
|
||||
| Target profile | Locale or language-project id (`en-US`, `ja-JP`, a conlang's slug) plus a revision `profile:<language>:<n>` | Authored, revised by commit or by an accepted round that changed the lowering; each lowering names the revision it used | Profile authors and accepted rounds; never the lowerer's defaults (`persona-drives-register`) |
|
||||
| Language ontology entry (cut `language-ontology`) | Key `onto:<language>:<n>`; type `weksa.ontology_entry`: conditions over feature values (stance, speaker, scene, packet features) and community ids, and the surface choice they select (layer: lexicon, grammar, discourse, phonology, orthography; a bounded descriptor and at most three bounded examples); introduced and retired by a profile revision | Added or retired by a profile revision, never edited; converted from `data/target-language-ontology/*.yaml`, which is deleted in the same cut so the YAML is not a second owner | The language's owner (the operator for her settings' languages) and accepted rounds; an ontology-minting call is assembled from Train items only, like an inventory-minting call, and its text is in the scorer's leak diagnostic. Selection is deterministic code (`select(profile, stance, speaker, scene)`); the lowerer realizes the selected choices and the trace lists their ids (`lowering-traced`) |
|
||||
| Community (cut `language-ontology`) | Key `community:<slug>`; type `weksa.community`, converted from `data/cultural-ontology/cultures/` and `referents/` (YAML deleted in the same cut) | Revised by commit; a speaker names the communities it belongs to by id | The cultural ontology's author (owner-attested evidence only; Discord-only content stays deleted, `discord-only-content-deleted`) |
|
||||
| Speaker (cut `speaker-scene`) | Key `speaker:<slug>@<n>`; type `weksa.speaker`: stable axes as feature values of slot `Speaker` (origin, socioeconomic standing, age cohort, education and whatever the loop adds), at an inventory revision, and an `Annotator` | A new revision appends, never edits; a corpus item and every prior turn name a speaker slug, and a round records the speaker revisions it read | The annotator it names: `PersonaSnapshot` (a projection of a `gamecult.persona_state.v0` document, naming its source document and `updatedAt`), `Operator`, `WorkMetadata` (a public-domain work's own cast list and narration), or `Model { family, generator }` (the 2b `ModelFamily`) |
|
||||
| Scene (cut `speaker-scene`) | Key `scene:<group>@<n>`, the item's split group; type `weksa.scene`: participants (speaker slugs) and, per ordered pair, relation features of slot `Scene` (relative status, intimacy, in-group, audience and whatever the loop adds), at an inventory revision, and an `Annotator` | As Speaker. Keyed by the group, so a scene never straddles the split | As Speaker |
|
||||
| Speaker (cut `speaker-state`) | Key `speaker:<slug>@<n>`; type `weksa.speaker`: stable axes as feature values of slot `Speaker` (origin, socioeconomic standing, age cohort, education and whatever the loop adds), at an inventory revision, and an `Annotator` | A new revision appends, never edits; a corpus item and every prior turn name a speaker slug, and a round records the speaker revisions it read | The annotator it names: `PersonaSnapshot` (a projection of a `gamecult.persona_state.v0` document, naming its source document and `updatedAt`), `Operator`, `WorkMetadata` (a public-domain work's own cast list and narration), or `Model { family, generator }` (the 2b `ModelFamily`) |
|
||||
| Scene (cut `speaker-state`) | Key `scene:<group>@<n>`, the item's split group; type `weksa.scene`: participants (speaker slugs) and, per ordered pair, relation features of slot `Scene` (relative status, intimacy, in-group, audience and whatever the loop adds), at an inventory revision, and an `Annotator` | As Speaker. Keyed by the group, so a scene never straddles the split | As Speaker |
|
||||
| Corpus item | Key `item:<source-slug>:<n>`, `n` canonical decimal >= 1 (no sign, no leading zero, from cut 2b); type `weksa.corpus_item` with text, language, speaker, split group, prior turns, source. Source `ModelWritten` carries a `ModelFamily` enum, the generator string as the fixture wrote it, and a prompt digest (cut 2b). The item carries no speaker state of its own: its speaker slug and group name the Speaker and Scene documents | Admitted once; the whole item is immutable (re-adding an identical item is a no-op, any difference is refused; a correction is a new item that `supersedes` the old); text already held under another key is refused unless the new item supersedes that key (cut 2b); withdrawn, never deleted, so old scores stay explainable | Importers and `weksa corpus add`, under `corpus-cast-plus-public-domain`; the source enum has no variant for Discord or any training-forbidden source (`corpus-clean`) |
|
||||
| Split assignment | Derived: `split(scheme, item.group)` | Fixed for a scheme; a new scheme is a new constant, never a reassignment. A model-written item whose group derives test is in no split; a superseding item keeps its predecessor's group | The hash function. Nobody assigns by hand, so there is no writer to forbid |
|
||||
| Loop round | Key `run:<label>:round:<n>`; type `weksa.loop_round` with spec version, profile revisions, model ids and prompt digests, sampled train items, proposed change, validation scores before and after, outcome | Opened, scored, closed accepted or rejected; immutable once closed. Lives in the run store on Yggdrasil | The loop driver's accept rule (validation composite improves past the measured noise floor and no metric regresses past tolerance) |
|
||||
|
|
@ -180,7 +180,7 @@ operator-attested annotations alone. Who annotates is question
|
|||
carries its name spans (byte ranges of its text, each naming an entity and
|
||||
on extended grapheme cluster boundaries of its NFC text and equal to one of that
|
||||
entity's NFC forms in the item's language; an entity holds at most 8 languages
|
||||
of BCP 47-form tags, 8 forms each, a form at most 64 bytes with no invisible format character), and its speaker
|
||||
of BCP 47-form tags, 8 forms each, a form at most 64 bytes and 4 words), and its speaker
|
||||
and prior-turn speakers are entity ids. The packet store refuses a packet
|
||||
whose referent cites an entity the store lacks, and a packet with a
|
||||
`corpus_item` (required, typed `CorpusItemId`) that cites an entity neither annotated on that item nor one of
|
||||
|
|
@ -227,21 +227,6 @@ operator-attested annotations alone. Who annotates is question
|
|||
|
||||
## Body facts (probes of 2026-10-29, map pass for speaker state, 2b and scorer)
|
||||
|
||||
- Weksa main `869c82b` (packet-store, 2b corpus labels, 2c entity store, scorer
|
||||
landed; packet schema v3). Annotators land with their consumers: `Operator`
|
||||
and `Model { family, generator }` with cut `speaker-scene` (the cast seeds are
|
||||
Model-annotated and an operator attests by appending an `Operator` revision),
|
||||
`WorkMetadata` with `pd-importer`, `PersonaSnapshot` with `live-lowering`.
|
||||
- UAX #29 word probe (unicode-segmentation 1, scratch crate, 2026-10-29):
|
||||
`unicode_words()` counts (whitespace count in brackets) "Brokkr" 1 [1],
|
||||
"Mary Ann Evans" 3 [3], "de la Cruz y Garcia" 5 [5], Thai "สมชาย" 5 [1],
|
||||
"สมชาย ใจดี" 8 [2], "山田太郎" 4 [1], "李小龍" 3 [1], "ブロッカー" 1 [1],
|
||||
"김민준" 1 [1], "Jean-Luc" 2 [1], "O'Brien" 1, five letters glued by ZWSP 5 [1],
|
||||
glued by WJ U+2060 1 [1]. Without a dictionary UAX #29 splits Thai and Han
|
||||
per character, so a 4-word UAX #29 bound refuses ordinary Thai names (question
|
||||
`form-word-bound`, recommended bytes-only). NFKC, default case fold, NFKC,
|
||||
alphanumerics only: "fine" -> "fine", "Ｂｒｏｋｋｒ" -> "brokkr", ZWSP and WJ drop out.
|
||||
|
||||
- Weksa main `28284cc`: `cargo test -p weksa` on Yggdrasil (ygg-verify, rust
|
||||
image) 73 passed. `weksa corpus dump` holds three items: `item:brokkr:1`
|
||||
(group `brokkr-edda-voice`, Validation, `ModelWritten { model_family:
|
||||
|
|
@ -328,73 +313,16 @@ checkpoint the test-split composite is within the noise floor of validation.
|
|||
The test split is read only at checkpoints. Self then records the freeze, and
|
||||
the training set is the whole corpus re-encoded under the frozen revision.
|
||||
|
||||
## Lowerer first cut (`golf-loop-lowerer-cut`)
|
||||
|
||||
Authority map for the smallest lowerer, written before its code (Imagination
|
||||
pass by the Weksa Self run, 2026-11-01). The cut exists to give the golf loop a
|
||||
measurable baseline from a packet: `lower` gives text, `score` gives numbers.
|
||||
|
||||
| Decision | Owner | Inputs | Outputs | Not the owner |
|
||||
|---|---|---|---|---|
|
||||
| Whether a packet exists | `validate` / `PacketStore::admit` (unchanged) | packet, inventory, corpus, entities | admitted packet or typed refusal | the lowerer |
|
||||
| Wording of a packet in English | `crates/weksa/src/lower.rs`: `lower(&LoweringInput, &Inventory) -> Result<String, LowerError>` | the validated lowering view and the inventory at its revision | text, or a typed refusal naming the missing template | the corpus item, the scorer, any model |
|
||||
| English template table (concept words, frame templates, speech-act openers) | the same module, as rows `(inventory id, gloss it speaks for, surface)` | none; it is data in code | rows the lowerer looks up | the inventory (it owns the ids and glosses, not the English) |
|
||||
| The inventory's concepts and frames | an inventory revision; here `inventory::proof_revision()` (rev 2), hand-seeded as `seed()` is | none | the 5 concepts and 2 frames `nibu-proof-001` names | the lowerer |
|
||||
| What a score counts | `Scorer::score` (unchanged) | output text, reference text | `ItemScore` | `weksa score`, which only reads text and calls it |
|
||||
| The proof packet | `packet::proof_packet()`, typed ids of rev 2, admitted by `weksa packet proof <store>` through `PacketStore::admit`. Not JSON: `Packet`'s serde is CultCache's positional tuple, and JSON in is the xenos boundary the cut 7 encoder owns | none | admitted packet | the CLI decides nothing about validity |
|
||||
|
||||
Decisions this map makes, and why:
|
||||
|
||||
- **`lower` takes `&LoweringInput`, not `&Packet`.** A `Packet` carries `trace`
|
||||
and `provenance.corpus_item`; the model page says the lowerer sees the packet
|
||||
without its trace and never the corpus item (`lowering-traced`,
|
||||
`leak-owned-by-flow-and-scorer-3`). `validate` already yields the view, so the
|
||||
CLI runs `validate` and hands it over. Cut 4 narrows the door to
|
||||
`PacketStore::admit`; this cut does not wait on it.
|
||||
- **The frame owns the verb phrase; a predication's concept must resolve but is
|
||||
not rendered.** Referent concepts supply nouns. A frame template names its
|
||||
roles by inventory gloss (`{agent}`), resolved against the inventory at the
|
||||
packet's revision, so a retired or re-glossed role cannot be silently
|
||||
mis-keyed. A row whose gloss differs from the inventory's entry for its id is
|
||||
a `LowerError`, so a table row cannot speak for a different meaning than its
|
||||
id now carries.
|
||||
- **Rendering rules, all deterministic and all from packet fields:** the
|
||||
discourse addressee renders "you" ("yourself" when it already filled a role
|
||||
in the same predication), other referents render "the <concept word>", a
|
||||
predication filling a role renders "that", the speech-act row supplies an
|
||||
optional opener, sentences are capitalised and end in a full stop.
|
||||
Entity-bearing referents render by concept word: the lowerer does not read
|
||||
the entity store in this cut (follow-up: entity forms in the lowerer).
|
||||
- **No style, no register, no features.** Hedge, certainty, modality and
|
||||
polarity are not rendered, so the baseline scores what meaning alone buys.
|
||||
A lowerer that ignores `polarity: negative` would assert the opposite, so
|
||||
`lower` refuses any packet that carries a feature, relation or extension it
|
||||
does not render rather than render it wrong (`packet-owns-meaning`); features arrive as templates
|
||||
learn them.
|
||||
- **Deleted by later cuts, not this one:** the table in `lower.rs` and
|
||||
`proof_revision()` are temporary owners. Cut 5 (`language-ontology`) moves
|
||||
the table to `weksa.ontology_entry` documents selected by the target
|
||||
profile; cut 7 replaces `proof_revision()` with the loop's minting round.
|
||||
Nothing is deleted today because no earlier lowerer exists; the YAML
|
||||
`examples/english-lowering/` hand-lowerings are fixtures, not owners, and
|
||||
stay.
|
||||
- **`weksa score` is a thin reader of `Scorer::score`**: `weksa score <item>...
|
||||
[--output FILE]` embeds the output text once against every named item. It
|
||||
stores nothing (Score documents arrive with cut 7) and gates nothing.
|
||||
|
||||
## Cut sequence
|
||||
|
||||
One line per cut; specs exist for the cuts about to run (2d and 4, siblings on `869c82b`; 2d merges first and 4 rebases onto it).
|
||||
One line per cut; specs exist for the cuts about to run (2b and 3, siblings).
|
||||
|
||||
1. `park-node-body`: park the Node daemon, MiMo command, RUDP shims, launchers and the Discord-sourced Metame harness at tag `parked/weksa-node-body`; README front door tells the truth. Landed.
|
||||
2. `packet-store`: Rust crate with packet, inventory and corpus types, the validator, the derived split, the stores and the `weksa` CLI. Landed at `28284cc`.
|
||||
2b. Landed. `packet-store-corpus`: corpus labels tell the truth (`corpus-labels-tell-truth`): `ModelFamily` and the generator string on `ModelWritten`; `item:nibu:1` ModelWritten (Unrecorded); `item:brokkr:1` and its fixture carry `codex-local-source-aware-draft`, family OpenAi; intake refuses aliased keys, a whole item changed under its key, and a held item's text under a new key; corpus.cc reseeded. Speaker state is not in 2b: it lives in Speaker and Scene documents and the packet's stance, not on the item. Waits on 2. (spec)
|
||||
2c. Landed. `entity-store`: `ProperName` and `Referent.name` deleted; `EntityId`, `Referent.entity`, the `weksa.entity` document and its store; corpus name spans and entity-id speakers; the packet store's entity checks; seed entities and spans for the cast items; schema v2. Waits on 2b. (spec r1)
|
||||
3. Landed at `869c82b`. `scorer`: chrF, meaning cosine behind an `Embedder` port (Ollama adapter with `num_thread`), leak diagnostics, aggregation by derived split; library only, `weksa score` comes with the loop. h2 landed the r2 fix pass on its branch; r3 deletes the name mask (the scorer reads output and reference only) and pools novel coverage per order over word keys that split joined labels. Waits on 2c for merge order. (spec r3)
|
||||
2d. `entity-door-lows`: invisible format characters refused in entity forms (ZWJ, ZWNJ, CGJ and variation selectors kept); the form length bound per question `form-word-bound` (recommended: bytes only); attested origins (`Cast { persona, fixture }`, `Live { persona }`) and `CastAuthored.author`; empty provenance strings refused at the store; S6 under NFKC plus case fold and no-alphanumeric texts refused; `next_key` refuses at `u32::MAX`; CLI flags that do not apply refuse and no error echoes a value. Owns follow-ups `entity-door-lows` and `s6-unicode-normal-form`. (spec r1)
|
||||
3b. `lowerer-first-cut` (follow-up `golf-loop-lowerer-cut`): the section above. `lower`, `weksa lower`, `weksa score`, `weksa packet proof`, inventory rev 2 for `nibu-proof-001`.
|
||||
4. `speaker-state`: packet `stance` (felt affect and restraint, slot `Stance`, packet schema v4); formality, warmth and intensity out of the seed and the standard; only `PacketStore::admit` yields a `LoweringInput`, and it refuses packets for withdrawn items (follow-up `lowering-input-from-packet-store`). (spec r1)
|
||||
4b. `speaker-scene`: slots `Speaker` and `Scene` with their seed features; `weksa.speaker` (`speaker:<entity-slug>@<n>`) and `weksa.scene` (`scene:<group>@<n>`, participants and ordered-pair relations) documents with `Annotator` `Operator` and `Model { family, generator }`, validated by the packet's shared feature check, append-only per key; their stores, `weksa speaker` and `weksa scene` commands, published schemas; Model-annotated cast speakers and scenes for the committed items. Waits on 2d and 4; `pd-speaker-annotation` blocks only the public-domain annotator.
|
||||
2b. `packet-store-corpus`: corpus labels tell the truth (`corpus-labels-tell-truth`): `ModelFamily` and the generator string on `ModelWritten`; `item:nibu:1` ModelWritten (Unrecorded); `item:brokkr:1` and its fixture carry `codex-local-source-aware-draft`, family OpenAi; intake refuses aliased keys, a whole item changed under its key, and a held item's text under a new key; corpus.cc reseeded. Speaker state is not in 2b: it lives in Speaker and Scene documents and the packet's stance, not on the item. Waits on 2. (spec)
|
||||
2c. `entity-store`: `ProperName` and `Referent.name` deleted; `EntityId`, `Referent.entity`, the `weksa.entity` document and its store; corpus name spans and entity-id speakers; the packet store's entity checks; seed entities and spans for the cast items; schema v2. Waits on 2b. (spec r1)
|
||||
3. `scorer`: chrF, meaning cosine behind an `Embedder` port (Ollama adapter with `num_thread`), leak diagnostics, aggregation by derived split; library only, `weksa score` comes with the loop. h2 landed the r2 fix pass on its branch; r3 deletes the name mask (the scorer reads output and reference only) and pools novel coverage per order over word keys that split joined labels. Waits on 2c for merge order. (spec r3)
|
||||
4. `speaker-state`: inventory slots `Stance`, `Speaker`, `Scene`; packet schema v3 with `stance` (felt affect, restraint); formality, warmth and intensity removed from the seed and the standard; `weksa.speaker` and `weksa.scene` documents with `Annotator` (reusing 2b's `ModelFamily`), validated by the packet's feature check; Speaker documents keyed by the entity ids of 2c; cast speakers and scenes for the three items. Waits on 2b and question `pd-speaker-annotation` only for the annotator variants it admits.
|
||||
5. `language-ontology`: `weksa.community` from `data/cultural-ontology/` (owns follow-up `edda-referent-yaml-parse`), `weksa.ontology_entry` for en-US and ja-JP from `data/target-language-ontology/` (YAML deleted), speaker communities, deterministic `select`, the lowering view (packet without trace, speaker, scene, selected entries). Waits on 4.
|
||||
6. `pd-importer`: public-domain play and fiction dialogue into the corpus with speaker, scene group, edition provenance, and Speaker and Scene annotations by the ruled annotator. Waits on 4, follow-up `gutenberg-terms` and question `pd-speaker-annotation`.
|
||||
7. `golf-loop-en`: the loop driver, the Train-only context builder for inventory- and ontology-minting calls (follow-up `loop-context-builder-train-only`), the packet store bound to one inventory store (`packet-store-inventory-binding`), `ModelPort` with the `claude -p` adapter, encoder (packet with stance) and lowerer prompts, round records, Score documents, the composite and accept rule, `weksa score`, the detached unit on Yggdrasil. Waits on 3, 5, 6 and follow-up `claude-on-yggdrasil`.
|
||||
|
|
|
|||
|
|
@ -121,15 +121,12 @@
|
|||
]
|
||||
},
|
||||
{
|
||||
"description": "A line from a cast member's hand-authored fixture, with the human who\nwrote it. A fixture that records no author is `ModelWritten` with family\n`Unrecorded`, never this (`corpus-labels-tell-truth`).",
|
||||
"description": "A line from a cast member's hand-authored fixture.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"CastAuthored": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"author": {
|
||||
"type": "string"
|
||||
},
|
||||
"fixture": {
|
||||
"type": "string"
|
||||
},
|
||||
|
|
@ -139,8 +136,7 @@
|
|||
},
|
||||
"required": [
|
||||
"persona",
|
||||
"fixture",
|
||||
"author"
|
||||
"fixture"
|
||||
]
|
||||
}
|
||||
},
|
||||
|
|
|
|||
|
|
@ -31,22 +31,18 @@
|
|||
"description": "Where an entity was minted.",
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "From a cast member's fixture: the persona and the fixture path the\nentity came from.",
|
||||
"description": "From a cast member's fixture.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"Cast": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"fixture": {
|
||||
"type": "string"
|
||||
},
|
||||
"persona": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"persona",
|
||||
"fixture"
|
||||
"persona"
|
||||
]
|
||||
}
|
||||
},
|
||||
|
|
@ -77,25 +73,9 @@
|
|||
]
|
||||
},
|
||||
{
|
||||
"description": "Minted live by the named persona before it names the entity.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"Live": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"persona": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"persona"
|
||||
]
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"Live"
|
||||
]
|
||||
"description": "Minted live by a persona before it names the entity.",
|
||||
"type": "string",
|
||||
"const": "Live"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
@ -104,7 +84,7 @@
|
|||
"type": "object",
|
||||
"properties": {
|
||||
"forms": {
|
||||
"description": "1 to 64 bytes each, no control or invisible format character, no edge\nwhitespace, none repeated.",
|
||||
"description": "1 to 64 bytes each, no control character, no edge whitespace, none repeated.",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
|
|
|
|||
Loading…
Reference in a new issue