GameCult/Weksa
Watch
9
0
Fork
You've already forked Weksa
 0
Code Issues Pull requests 2 Projects Releases Packages Wiki Activity Actions

language-ontology: weksa.ontology_entry, its store, select, wired into the lowering prompt #51

Merged
weksa merged 5 commits from weksa-lowering/language-ontology into main 2026-11-12 15:12:14 +00:00
Conversation 0 Commits 5 Files changed 18 +1644 -60
18 changed files with 1644 additions and 60 deletions
Download patch file Download diff file Expand all files Collapse all files

51
crates/weksa/src/context.rs
View file

@ -10,7 +10,9 @@
//! [`Inventory`] and an [`EditFeedback`] and nothing else, so no held-out text,
//! key or per-item score can reach it. A lowering prompt names the language it
//! is said in on a language line; every language is named by its tag, none by a
//! default.
//! default. A lowering prompt also prints the target language's ontology
//! entries that `ontology::select` chose for the view; the caller never names
//! entries.
//!
//! Item text may hold line breaks; the renderer indents every continuation line
//! (`indented`), so item text cannot forge a `[key]` line or a paragraph.
@ -28,6 +30,7 @@ use serde::Serialize;
use crate::corpus::{CorpusItem, CorpusItemId};
use crate::inventory::{Inventory, InventoryEntry, OptionLabel, ValueKind};
use crate::lower::{EntityForms, LowerError, entity};
use crate::ontology::{Ontology, select};
use crate::packet::{
Context, Discourse, EntityId, Extension, Predication, Referent, Relation, Stance,
};
@ -68,7 +71,7 @@ A feature is {feature, value}; value is {\"Grade\": 0 to 1000}, {\"Option\": lab
/// Fixed per run by `PROMPT_SOURCE_DIGEST`.
pub const LOWER_INSTRUCTIONS: &str = "\
Say the packet below in the language on the language line, as one utterance of its speaker: all of its meaning and nothing it does not carry.
Every id is the inventory's listed below and every feature value is as listed; a grade is permille, 0 to 1000. The stance is the speaker's felt affect and restraint: it shapes how the utterance is said, not what it says.
Every id is the inventory's listed below and every feature value is as listed; a grade is permille, 0 to 1000. The stance is the speaker's felt affect and restraint: it shapes how the utterance is said, not what it says. The ontology lines are choices this language makes for this speaker and stance: realize each where the packet allows; they add no meaning.
A referent with an entity is named by one of that entity's forms on the entities line.
Answer with one JSON object and nothing else: {\"text\": \"...\"}. No code fence, no commentary.";
@ -86,7 +89,7 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat
/// ```no_run
/// # let dir = tempfile::tempdir().unwrap();
/// # let p = |n: &str| dir.path().join(n);
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") };
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc"), ontology: &p("o.cc") };
/// # let panel = ["en-US".to_string()];
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &panel).unwrap();
/// let t = run.train().unwrap();
@ -100,8 +103,8 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat
/// ```compile_fail
/// # let dir = tempfile::tempdir().unwrap();
/// # let p = |n: &str| dir.path().join(n);
/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); }
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") };
/// # for n in ["c.cc", "i.cc", "e.cc", "o.cc"] { std::fs::write(p(n), [0x90]).unwrap(); }
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc"), ontology: &p("o.cc") };
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &["en-US".to_string()]).unwrap();
/// let t = run.train().unwrap();
/// let _u = t.clone();
@ -111,8 +114,8 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat
/// fn s<T: serde::Serialize>(_: &T) {}
/// # let dir = tempfile::tempdir().unwrap();
/// # let p = |n: &str| dir.path().join(n);
/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); }
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") };
/// # for n in ["c.cc", "i.cc", "e.cc", "o.cc"] { std::fs::write(p(n), [0x90]).unwrap(); }
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc"), ontology: &p("o.cc") };
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &["en-US".to_string()]).unwrap();
/// let t = run.train().unwrap();
/// s(&t);
@ -334,7 +337,7 @@ impl EditFeedback {
/// ```no_run
/// # let dir = tempfile::tempdir().unwrap();
/// # let p = |n: &str| dir.path().join(n);
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") };
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc"), ontology: &p("o.cc") };
/// # let panel = ["en-US".to_string()];
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &panel).unwrap();
/// let t = run.train().unwrap();
@ -352,8 +355,8 @@ impl EditFeedback {
/// ```compile_fail,E0061
/// # let dir = tempfile::tempdir().unwrap();
/// # let p = |n: &str| dir.path().join(n);
/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); }
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") };
/// # for n in ["c.cc", "i.cc", "e.cc", "o.cc"] { std::fs::write(p(n), [0x90]).unwrap(); }
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc"), ontology: &p("o.cc") };
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &["en-US".to_string()]).unwrap();
/// let t = run.train().unwrap();
/// let inv = run.inventory().inventory().unwrap();
@ -365,8 +368,8 @@ impl EditFeedback {
/// fn s<T: serde::Serialize>(_: &T) {}
/// # let dir = tempfile::tempdir().unwrap();
/// # let p = |n: &str| dir.path().join(n);
/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); }
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") };
/// # for n in ["c.cc", "i.cc", "e.cc", "o.cc"] { std::fs::write(p(n), [0x90]).unwrap(); }
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc"), ontology: &p("o.cc") };
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &["en-US".to_string()]).unwrap();
/// let t = run.train().unwrap();
/// let inv = run.inventory().inventory().unwrap();
@ -523,9 +526,13 @@ impl Prompt {
/// [`LOWER_INSTRUCTIONS`], the `language: <tag>` line, the meaning of
/// `input` (no kind, constraints, scene, speaker or group), the forms in
/// `language` of every entity its referents cite, and the entries of `inventory` live at the view's revision. `item`
/// goes only to [`Prompt::item_keys`], never into the text. The view's
/// speaker and group fields are never read; a speaker's entity id appears
/// `language` of every entity its referents cite, the `ontology:` section
/// and the entries of `inventory` live at the view's revision. The section
/// holds the entries `ontology::select` chooses from the view's stance and
/// discourse features and `language`, never the caller; the view's speaker
/// and group are still never read. `item`
/// goes only to [`Prompt::item_keys`], never into the text. A speaker's
/// entity id appears
/// only when the encoder cites that entity on a referent, which is naming a
/// cited entity like any other. A cited entity the port lacks, or one with
/// no form in `language`, is an error and no prompt. The caller chooses the
@ -536,6 +543,7 @@ impl Prompt {
language: &str,
inventory: &Inventory,
entities: &dyn EntityForms,
ontology: &Ontology,
) -> Result<Prompt, LowerError> {
let mut forms: BTreeMap<&str, Vec<String>> = BTreeMap::new();
for id in input.referents().iter().filter_map(|r| r.entity.as_ref()) {
@ -556,8 +564,19 @@ impl Prompt {
stance: input.stance(),
};
let rev = input.inventory_rev();
let chosen = select(ontology, language, input);
let mut lines = String::new();
for e in &chosen {
lines.push_str(&format!("\n{} [{}] {}", e.key(), e.layer.name(), e.choice));
for x in &e.examples {
lines.push_str(&format!(" | e.g. {x}"));
}
}
if chosen.is_empty() {
lines.push_str(" none");
}
let text = format!(
"{LOWER_INSTRUCTIONS}\n\nlanguage: {language}\n\npacket: {}\n\nentities: {}\n\ninventory rev:{rev}{}",
"{LOWER_INSTRUCTIONS}\n\nlanguage: {language}\n\npacket: {}\n\nentities: {}\n\nontology:{lines}\n\ninventory rev:{rev}{}",
serde_json::to_string(&meaning).expect("a meaning is plain data"),
serde_json::to_string(&forms).expect("forms are plain data"),
inventory_lines(inventory, rev)

79
crates/weksa/src/encode.rs
View file

@ -19,10 +19,12 @@ use crate::corpus::CorpusItemId;
use crate::inventory::{
Change, Family, InvId, Inventory, InventoryEntry, InventoryRevision, Slot, ValueKind,
};
use crate::ontology::{Condition, Layer, OntologyEntry};
use crate::packet::{
Constraints, Context, Discourse, EncoderId, Grade, Packet, PacketId, PacketKind, Predication,
Provenance, ProvenanceSource, Referent, Relation, SCHEMA_VERSION, Stance, Trace,
};
use crate::speaker::Annotator;
use crate::split::{fnv1a64, hex16};
/// The meaning fields of a packet, as a model writes them.
@ -271,6 +273,59 @@ impl EditDraft {
}
}
/// One ontology entry as a model writes it: no language, number, inventory
/// revision or annotator.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct EntryOntologyDraft {
pub layer: Layer,
#[serde(default)]
pub when: Vec<Condition>,
pub choice: String,
#[serde(default)]
pub examples: Vec<String>,
}
/// A batch of ontology entries as a model drafts them: the xenos door of a
/// seed answer.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct OntologyDraft {
pub entries: Vec<EntryOntologyDraft>,
}
impl OntologyDraft {
pub fn parse(answer: &str) -> Result<Self, DraftError> {
from_json(answer)
}
/// The entries this draft becomes: numbered from `first_n` in draft order,
/// all of `language` read at `inventory_rev`, annotated `annotator`.
/// Whether they are admissible is `OntologyStore::admit`'s to refuse.
pub fn into_entries(
self,
language: &str,
inventory_rev: u32,
first_n: u32,
annotator: &Annotator,
) -> Vec<OntologyEntry> {
self.entries
.into_iter()
.zip(first_n..)
.map(|(d, n)| OntologyEntry {
language: language.to_string(),
n,
inventory_rev,
layer: d.layer,
when: d.when,
choice: d.choice,
examples: d.examples,
annotator: annotator.clone(),
})
.collect()
}
}
/// A lowering as the lowerer writes it: the English text, nothing else.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
@ -626,4 +681,28 @@ mod tests {
}
assert!(EditDraft::parse("3").is_err() && EditDraft::parse("\"x\"").is_err());
}
#[test]
fn ontology_draft_refuses_unknown_fields_and_numbers_from_first_n() {
for bad in [
r#"{"entries": [], "extra": 1}"#,
r#"{"entries": [{"layer": "Lexicon", "choice": "c", "language": "ja-JP"}]}"#,
"```json\n{\"entries\": []}\n```",
"[]",
] {
assert!(OntologyDraft::parse(bad).is_err(), "{bad}");
}
let draft = OntologyDraft::parse(
r#"{"entries": [{"layer": "Grammar", "choice": "a"}, {"layer": "Lexicon", "choice": "b", "examples": ["x"]}]}"#,
)
.unwrap();
let by = Annotator::Operator;
let entries = draft.into_entries("ja-JP", 3, 5, &by);
assert_eq!(
entries.iter().map(|e| (e.language.as_str(), e.n, e.inventory_rev)).collect::<Vec<_>>(),
[("ja-JP", 5, 3), ("ja-JP", 6, 3)]
);
assert!(entries.iter().all(|e| e.annotator == by));
assert_eq!(entries[1].examples, ["x"]);
}
}

57
crates/weksa/src/form.rs
View file

@ -91,20 +91,45 @@ pub fn is_calendar_date(s: &str) -> bool {
(1..=days).contains(&day)
}
/// A BCP 47-form language tag: 1 to 35 bytes of ASCII alphanumerics in
/// hyphen-separated subtags of 1 to 8 bytes; the first is 2 to 8 letters, or the
/// single letter `x` of a private-use tag. Form only: no registry is consulted.
/// A BCP 47 language tag in its canonical case (RFC 5646 section 2.1.1): 1 to 35
/// bytes of ASCII alphanumerics in hyphen-separated subtags of 1 to 8 bytes; the
/// first is 2 to 8 letters, or the single letter `x` of a private-use tag. The
/// language subtag is lowercase, a four-letter script subtag is Titlecase, a
/// two-letter region subtag is UPPERCASE, and every other subtag, and every
/// subtag after a singleton such as `x`, is lowercase. Form only: no registry is
/// consulted. Selection and comparison are by exact tag, so a case variant would
/// be a different, dead language; this is the one tag check of corpus items,
/// entities, ontology entries and the run panel.
pub(crate) fn is_language_tag(s: &str) -> bool {
if s.len() > 35 {
return false;
}
let mut subtags = s.split('-');
let first = subtags.next().unwrap_or("");
let first_ok = first.bytes().all(|b| b.is_ascii_alphabetic())
let first_ok = first.bytes().all(|b| b.is_ascii_lowercase())
&& ((2..=8).contains(&first.len()) || first == "x");
first_ok
&& subtags
.all(|t| (1..=8).contains(&t.len()) && t.bytes().all(|b| b.is_ascii_alphanumeric()))
if !first_ok {
return false;
}
let mut singleton = first == "x";
subtags.all(|t| {
if !(1..=8).contains(&t.len()) || !t.bytes().all(|b| b.is_ascii_alphanumeric()) {
return false;
}
let alpha = t.bytes().all(|b| b.is_ascii_alphabetic());
let canonical = if singleton {
t.bytes().all(|b| !b.is_ascii_uppercase())
} else if alpha && t.len() == 2 {
t.bytes().all(|b| b.is_ascii_uppercase())
} else if alpha && t.len() == 4 {
t.as_bytes()[0].is_ascii_uppercase()
&& t.bytes().skip(1).all(|b| b.is_ascii_lowercase())
} else {
t.bytes().all(|b| !b.is_ascii_uppercase())
};
singleton |= t.len() == 1;
canonical
})
}
/// A slug: `[a-z0-9]` and inner `-`, 1 to 48 bytes, no edge `-`.
@ -214,6 +239,24 @@ mod tests {
assert!(!is_format('\u{200c}') && !is_format('\u{200d}'));
}
#[test]
fn a_language_tag_is_in_canonical_case() {
for good in [
"en", "en-US", "ja-JP", "zh-Hant-TW", "sr-Latn", "es-419", "de-1996", "en-CA-x-ca",
"x-private", "en-u-nu-latn", "ja-JP-x-a1",
] {
assert!(is_language_tag(good), "{good}");
}
for bad in [
"", "e", "EN", "En", "en-us", "en-Us", "en-uS", "ja-jp", "JA-JP", "zh-hant-TW",
"zh-HANT-TW", "zh-hAnt-TW", "zh-Hant-tw", "en-CA-x-CA", "en-CA-x-Ca", "de-1996-X",
"es-419-LATN", "en--US", "en-", "-en", "en_US", "abcdefghi", "en-abcdefghi",
"xx-\u{e9}",
] {
assert!(!is_language_tag(bad), "{bad}");
}
}
#[test]
fn an_attestation_is_bounded_readable_text() {
assert!(is_attestation("a"));

4
crates/weksa/src/lib.rs
View file

@ -12,6 +12,7 @@ pub mod form;
pub mod inventory;
pub mod lower;
pub mod model;
pub mod ontology;
pub mod packet;
pub mod phones;
pub mod run;
@ -32,6 +33,7 @@ pub use inventory::{
};
pub use lower::{EntityForms, LowerError, PhoneLine, TARGET_LANGUAGE, lower, phones, pronounce};
pub use model::{AnswerFiles, ModelPort};
pub use ontology::{Condition, Layer, Ontology, OntologyEntry, Test, select};
pub use packet::{
EncoderId, EntityId, Grade, NodeId, Packet, PacketId, PacketKind, SCHEMA_VERSION, Stance,
};
@ -41,7 +43,7 @@ pub use score::{Embedder, ItemScore, OllamaEmbedder, Scorer, chrf};
pub use speaker::{Annotator, PairRelation, Party, Scene, Speaker};
pub use split::{Split, SplitScheme};
pub use store::{
AddOutcome, CorpusStore, EntityBound, EntityStore, InventoryStore, PacketStore,
AddOutcome, CorpusStore, EntityBound, EntityStore, InventoryStore, OntologyStore, PacketStore,
SpeakerStateStore, StoreError, StoreRefusal,
};
pub use validate::{LoweringInput, Refusal, SceneRead, validate};

79
crates/weksa/src/main.rs
View file

@ -8,6 +8,7 @@ use clap::error::{ContextKind, ContextValue, ErrorKind};
use clap::{Args, Parser, Subcommand, ValueEnum};
use unicode_normalization::UnicodeNormalization;
use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn};
use weksa::encode::OntologyDraft;
use weksa::entity::{Entity, EntityOrigin, LanguageForms, Pronunciation};
use weksa::inventory::{proof_revision, seed};
use weksa::packet::proof_packet;
@ -17,7 +18,7 @@ use weksa::run::{
};
use weksa::score::ScoreInput;
use weksa::{
AnswerFiles, CorpusStore, EntityId, EntityStore, InventoryStore, OllamaEmbedder, PacketStore,
Annotator, AnswerFiles, CorpusStore, EntityId, EntityStore, InventoryStore, OllamaEmbedder, OntologyStore, PacketStore,
Scorer, Split, SplitScheme, StoreError, lower, pronounce, validate,
};
@ -36,6 +37,9 @@ struct Cli {
/// The entity store.
#[arg(long, global = true, default_value = "data/entity/entity.cc")]
entities: PathBuf,
/// The language ontology store.
#[arg(long, global = true, default_value = "data/ontology/ontology.cc")]
ontology: PathBuf,
#[command(subcommand)]
command: Command,
}
@ -51,6 +55,8 @@ enum Command {
#[command(subcommand)]
Entity(EntityCommand),
#[command(subcommand)]
Ontology(OntologyCommand),
#[command(subcommand)]
Packet(PacketCommand),
/// Lower a stored packet to English text.
Lower {
@ -81,7 +87,7 @@ enum Command {
#[derive(Subcommand)]
enum LoopCommand {
/// Pin the global --corpus, --inventory and --entities as a run at ROOT.
/// Pin the global --corpus, --inventory, --entities and --ontology as a run at ROOT.
Open {
root: PathBuf,
#[arg(long)]
@ -105,8 +111,8 @@ enum LoopCommand {
panel: Vec<String>,
},
/// Run one round of the run at ROOT. The stores are read only from the
/// run's record: the global --corpus, --inventory and --entities are not
/// read. Encode prompts are written first; once they are answered a rerun
/// run's record: the global --corpus, --inventory, --entities and --ontology
/// are not read. Encode prompts are written first; once they are answered a rerun
/// writes the lowering prompts. The command exits 3 each time until every
/// prompt has an answer.
Round {
@ -195,6 +201,28 @@ enum EntityCommand {
Dump,
}
#[derive(Subcommand)]
enum OntologyCommand {
/// Admit a model's ontology draft (a JSON file) for one language as a
/// batch, numbered from the language's latest plus one and annotated as
/// the model that drafted it.
Admit {
answer: PathBuf,
#[arg(long)]
language: String,
/// The inventory revision the draft's conditions are read at.
#[arg(long)]
inventory_rev: u32,
#[arg(long)]
model_family: ModelFamilyArg,
/// The attestation string naming the drafting model.
#[arg(long)]
generator: String,
},
/// Text projection of the ontology for review; not a store.
Dump,
}
#[derive(Args)]
struct PutArgs {
/// `entity:<slug>`.
@ -648,6 +676,48 @@ fn main() -> Result<()> {
println!("{} {:?}\n forms: {:?}", e.id, e.origin, e.forms);
}
}
Command::Ontology(OntologyCommand::Admit {
answer,
language,
inventory_rev,
model_family,
generator,
}) => {
let text = std::fs::read_to_string(&answer).context("the draft would not read")?;
let draft = OntologyDraft::parse(&text)?;
let mut store = OntologyStore::open(&cli.ontology)?;
let first = store
.all()?
.iter()
.filter(|e| e.language == language)
.map(|e| e.n)
.max()
.unwrap_or(0)
+ 1;
let annotator = Annotator::Model {
family: model_family.into(),
generator,
};
let entries = draft.into_entries(&language, inventory_rev, first, &annotator);
let count = entries.len();
let inventory = InventoryStore::open(&cli.inventory)?;
let outcome = store.admit(entries, &inventory).map_err(refused)?;
println!("{outcome:?} {count} entries of {language}");
}
Command::Ontology(OntologyCommand::Dump) => {
for e in OntologyStore::open(&cli.ontology)?.all()? {
println!(
"{} {} rev:{} {:?}\n when: {:?}\n {}\n e.g. {:?}",
e.key(),
e.layer.name(),
e.inventory_rev,
e.annotator,
e.when,
e.choice,
e.examples
);
}
}
Command::Inventory(InventoryCommand::Seed) => {
let (rev, entries) = seed();
let count = entries.len();
@ -779,6 +849,7 @@ fn main() -> Result<()> {
corpus: &cli.corpus,
inventory: &cli.inventory,
entities: &cli.entities,
ontology: &cli.ontology,
},
&answerer,
&embed_model,

170
crates/weksa/src/ontology.rs Normal file
View file

@ -0,0 +1,170 @@
//! The language ontology: per target language, the choices its culture makes
//! about word, grammar, discourse, sound and script for a speaker in a stance
//! (ruling `persona-drives-ontology`). An entry says when it applies (feature
//! conditions over the lowering view's stance and discourse) and what it
//! chooses. `select` is the only reader that decides which entries a lowering
//! prompt carries, and it runs only inside `Prompt::lower`;
//! `OntologyStore::admit` is the only writer. Entries are immutable.
use cultcache_rs::DatabaseEntry;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use crate::inventory::{InvId, Slot};
use crate::packet::{FeatValue, FeatureValue, Grade};
use crate::speaker::Annotator;
use crate::validate::LoweringInput;
pub const MAX_CONDITIONS: usize = 4;
pub const MAX_EXAMPLES: usize = 3;
pub const MAX_CHOICE_BYTES: usize = 200;
pub const MAX_EXAMPLE_BYTES: usize = 120;
pub const MAX_ENTRIES_PER_LANGUAGE: usize = 64;
/// Which part of the language an entry shapes.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub enum Layer {
Lexicon,
Grammar,
Discourse,
Phonology,
Orthography,
}
impl Layer {
pub fn name(self) -> &'static str {
match self {
Layer::Lexicon => "lexicon",
Layer::Grammar => "grammar",
Layer::Discourse => "discourse",
Layer::Phonology => "phonology",
Layer::Orthography => "orthography",
}
}
}
/// What a feature's value must be for a condition to hold.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub enum Test {
AtLeast(Grade),
AtMost(Grade),
Is(FeatValue),
}
/// One feature of one slot of the view, tested. Only `Stance` and `Discourse`
/// reach a lowering view today.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct Condition {
pub slot: Slot,
pub feature: InvId,
pub test: Test,
}
impl Condition {
fn holds(&self, features: &[FeatureValue]) -> bool {
let Some(found) = features.iter().find(|f| f.feature == self.feature) else {
return false;
};
match (&self.test, &found.value) {
(Test::AtLeast(g), FeatValue::Grade(v)) => v >= g,
(Test::AtMost(g), FeatValue::Grade(v)) => v <= g,
(Test::Is(want), v) => want == v,
_ => false,
}
}
/// The feature value this condition names, as the inventory check reads it.
pub(crate) fn as_value(&self) -> FeatureValue {
FeatureValue {
feature: self.feature,
value: match &self.test {
Test::AtLeast(g) | Test::AtMost(g) => FeatValue::Grade(*g),
Test::Is(v) => v.clone(),
},
}
}
}
/// One choice of one language. Key `onto:<language>:<n>`.
///
/// The store judges the form of an entry's text (bounds, normalization, no
/// control or format character) and never its meaning: no door judges text
/// (ruling `leak-owned-by-flow-and-scorer-3`). No Train-only ontology mint
/// exists yet, so a seed or a drafted entry is checked against held-out text by
/// nothing today; follow-up `leak-diagnostic-in-round` owns that diagnostic.
#[derive(Clone, Debug, PartialEq, Eq, JsonSchema, DatabaseEntry)]
#[cultcache(type = "weksa.ontology_entry")]
pub struct OntologyEntry {
#[cultcache(key = 0)]
pub language: String,
/// The entry's number within its language, from 1.
#[cultcache(key = 1)]
pub n: u32,
/// The inventory revision the conditions read against.
#[cultcache(key = 2)]
pub inventory_rev: u32,
#[cultcache(key = 3)]
pub layer: Layer,
/// Every condition must hold; none means the entry always applies.
#[cultcache(key = 4)]
pub when: Vec<Condition>,
#[cultcache(key = 5)]
pub choice: String,
#[cultcache(key = 6)]
pub examples: Vec<String>,
#[cultcache(key = 7)]
pub annotator: Annotator,
}
/// `onto:<language>:<n>`.
pub fn ontology_key(language: &str, n: u32) -> String {
format!("onto:{language}:{n}")
}
impl OntologyEntry {
pub fn key(&self) -> String {
ontology_key(&self.language, self.n)
}
}
/// Every entry of the run's ontology store, by language then number.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct Ontology {
entries: Vec<OntologyEntry>,
}
impl Ontology {
pub fn empty() -> Self {
Self::default()
}
pub(crate) fn of(mut entries: Vec<OntologyEntry>) -> Self {
entries.sort_by(|a, b| (&a.language, a.n).cmp(&(&b.language, b.n)));
Self { entries }
}
pub fn entries(&self) -> &[OntologyEntry] {
&self.entries
}
}
/// The entries of exactly `language` whose every condition holds against the
/// view's stance and discourse features, in `n` order. An absent feature meets
/// no condition. Nothing else of the view is read.
pub fn select<'a>(
o: &'a Ontology,
language: &str,
input: &LoweringInput,
) -> Vec<&'a OntologyEntry> {
o.entries
.iter()
.filter(|e| e.language == language)
.filter(|e| {
e.when.iter().all(|c| match c.slot {
Slot::Stance => c.holds(&input.stance().features),
Slot::Discourse => c.holds(&input.discourse().features),
_ => false,
})
})
.collect()
}

89
crates/weksa/src/run.rs
View file

@ -41,13 +41,15 @@ use unicode_segmentation::UnicodeSegmentation;
use crate::context::{EditFeedback, EncoderOffer, Prompt, TrainItems};
use crate::corpus::{CorpusItem, CorpusItemId, ModelFamily, Source};
use crate::encode::{EditDraft, LoweringDraft, PacketDraft, encoder_id};
use crate::form::{is_attestation, is_slug};
use crate::form::{is_attestation, is_language_tag, is_slug};
use crate::model::{ModelPort, PortError};
use crate::ontology::OntologyEntry;
use crate::packet::{Packet, Stance};
use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate};
use crate::split::{Split, SplitScheme, fnv1a64, fnv1a64_of, hex16};
use crate::store::{
CorpusStore, EntityStore, InventoryStore, PacketStore, StoreError, open_cache_over,
CorpusStore, EntityStore, InventoryStore, OntologyStore, PacketStore, StoreError,
check_ontology_entry, open_cache_over,
};
use crate::validate::LoweringInput;
@ -55,7 +57,9 @@ use crate::validate::LoweringInput;
/// whose scheme and digest function decide which lines the prompt holds as
/// Train and name every prompt, then `lower.rs`, whose `EntityForms` port and
/// `entity()` the lowering prompt reads, then `packet.rs`, whose serde
/// shapes the lowering prompt prints as the packet. Each file enters the digest
/// shapes the lowering prompt prints as the packet, then `ontology.rs`, whose
/// entry shape and `select` decide which ontology lines the lowering prompt
/// prints. Each file enters the digest
/// behind its length, so the parts cannot be re-cut into other parts with the
/// same digest. Any edit to any of them, comments included, ends a run; code
/// outside those files is not covered.
@ -64,6 +68,7 @@ pub const PROMPT_SOURCE_DIGEST: u64 = fnv1a64_of(&[
include_bytes!("split.rs"),
include_bytes!("lower.rs"),
include_bytes!("packet.rs"),
include_bytes!("ontology.rs"),
]);
/// The least composite gain an edit must show over item noise to be kept, and
@ -125,6 +130,13 @@ pub struct RunRecord {
/// language is never offered, minted from, probed or scored.
#[cultcache(key = 14)]
pub panel: Vec<String>,
/// Canonical absolute path of the ontology store: read-only for the run,
/// pinned by `ontology_digest`.
#[cultcache(key = 15)]
pub ontology: String,
/// 16 hex of the ontology file's bytes.
#[cultcache(key = 16)]
pub ontology_digest: String,
}
/// How a run's lowering prompts reach the model that answers them (rulings
@ -429,6 +441,7 @@ impl RoundRecord {
pub enum DriftOf {
Corpus,
Entities,
Ontology,
Prompts,
Model,
Embedder,
@ -464,6 +477,9 @@ pub enum RunError {
/// The panel is empty, repeats or misspells a language, or names one with no
/// live Train or Validation item in the pinned corpus.
Panel,
/// An ontology entry of a panel language does not read against the run's
/// head inventory at its revision.
Ontology,
}
impl fmt::Display for RunError {
@ -473,7 +489,7 @@ impl fmt::Display for RunError {
RunError::NoRun => f.write_str("the root holds no run"),
RunError::Malformed(what) => write!(f, "malformed {what}"),
RunError::Drift(DriftOf::Prompts) => f.write_str(
"drift in Prompts (the prompt source digest covers context.rs, split.rs, lower.rs and packet.rs)",
"drift in Prompts (the prompt source digest covers context.rs, split.rs, lower.rs, packet.rs and ontology.rs)",
),
RunError::Drift(of) => write!(f, "drift in {of:?}"),
RunError::NoInventory => f.write_str("the inventory holds no revision"),
@ -485,6 +501,9 @@ impl fmt::Display for RunError {
RunError::Embed => f.write_str("the embedder failed"),
RunError::NoFloor => f.write_str("the run has no noise floor: record two rounds first"),
RunError::NoValidation => f.write_str("a reading has no validation item"),
RunError::Ontology => f.write_str(
"an ontology entry of a panel language does not read against the run's inventory",
),
RunError::Panel => f.write_str(
"the panel is empty, repeats or misspells a language, or names one with no live Train or Validation item",
),
@ -500,12 +519,13 @@ impl fmt::Debug for RunError {
impl std::error::Error for RunError {}
/// The three stores a run pins.
/// The four stores a run pins.
#[derive(Clone, Copy)]
pub struct RunPaths<'a> {
pub corpus: &'a Path,
pub inventory: &'a Path,
pub entities: &'a Path,
pub ontology: &'a Path,
}
/// A run's open stores, reopened only from its record. The one public producer
@ -517,6 +537,7 @@ pub struct RunStores {
corpus: CorpusStore,
inventory: InventoryStore,
entities: EntityStore,
ontology: OntologyStore,
}
/// The run's own copy of the inventory, made at `create`.
@ -530,6 +551,9 @@ fn check_pins(cache: &CultCache, root: &Path, record: &RunRecord) -> Result<(),
if file_digest(&record.entities).as_ref() != Some(&record.entity_digest) {
return Err(RunError::Drift(DriftOf::Entities));
}
if file_digest(&record.ontology).as_ref() != Some(&record.ontology_digest) {
return Err(RunError::Drift(DriftOf::Ontology));
}
let (path, digest, _) = head_of(cache, root, record)?;
if file_digest_at(&path).as_ref() != Some(&digest) {
return Err(RunError::Drift(DriftOf::Inventory));
@ -651,7 +675,10 @@ impl RunStores {
panel.sort();
// A tag no item carries (misspelt, wrong case, not a tag) is refused by
// `check_panel` once the corpus is open.
if panel.is_empty() || panel.windows(2).any(|w| w[0] == w[1]) {
if panel.is_empty()
|| panel.windows(2).any(|w| w[0] == w[1])
|| !panel.iter().all(|t| is_language_tag(t))
{
return Err(RunError::Panel);
}
let cache = run_cache(root)?;
@ -663,6 +690,8 @@ impl RunStores {
corpus: canonical(paths.corpus)?,
inventory: canonical(paths.inventory)?,
entities: canonical(paths.entities)?,
ontology: canonical(paths.ontology)?,
ontology_digest: String::new(),
corpus_digest: String::new(),
entity_digest: String::new(),
prompt_digest: hex16(PROMPT_SOURCE_DIGEST),
@ -684,11 +713,13 @@ impl RunStores {
corpus_digest: file_digest(&record.corpus).ok_or(RunError::Store)?,
entity_digest: file_digest(&record.entities).ok_or(RunError::Store)?,
inventory_digest: file_digest_at(&head).ok_or(RunError::Store)?,
ontology_digest: file_digest(&record.ontology).ok_or(RunError::Store)?,
..record
};
between();
let mut run = Self::with(root, cache, record)?;
run.check_panel()?;
run.check_ontology()?;
// The stores were read between the digests: unless they still match
// the pins, what was opened is not what was pinned. Nothing is
// recorded for a run that did not pin.
@ -739,6 +770,7 @@ impl RunStores {
corpus: CorpusStore::open(Path::new(&record.corpus)).map_err(store)?,
inventory: InventoryStore::open(&head).map_err(store)?,
entities: EntityStore::open(Path::new(&record.entities)).map_err(store)?,
ontology: OntologyStore::open(Path::new(&record.ontology)).map_err(store)?,
cache,
record,
})
@ -748,6 +780,29 @@ impl RunStores {
&self.record
}
/// Every ontology entry of a panel language reads against the run's head
/// inventory at its own revision: the check the store applied at admission,
/// applied again to this inventory.
fn check_ontology(&self) -> Result<(), RunError> {
let inventory = self.inventory.inventory().map_err(|_| RunError::Store)?;
let entries = self.ontology.all().map_err(|_| RunError::Store)?;
let head = inventory.latest().map(|r| r.n);
entries
.iter()
.filter(|e| self.record.panel.contains(&e.language))
.try_for_each(|e| {
// At its own revision, and again at the head: an entry on a
// feature the head retired could never select.
let at_head = OntologyEntry {
inventory_rev: head.ok_or(RunError::Ontology)?,
..e.clone()
};
check_ontology_entry(e, &inventory)
.and_then(|()| check_ontology_entry(&at_head, &inventory))
.map_err(|_| RunError::Ontology)
})
}
/// Every panel language holds a live Train and a live Validation item.
fn check_panel(&self) -> Result<(), RunError> {
let items = self.corpus.all().map_err(|_| RunError::Store)?;
@ -1171,7 +1226,9 @@ impl RunStores {
/// Train and Validation packet of plain round `n`, per panel language, asks the lowerer again
/// (`model`, the replay), with the packet's stance emptied (drop) and with
/// the next item's stance in its split (permute), and reads how far the
/// lowering moves against the round's own answer (`recorded`). Test and
/// lowering moves against the round's own answer (`recorded`). The variant
/// stance also changes which ontology entries are selected, so the figure is
/// the stance's whole effect on the lowering, packet and ontology together. Test and
/// withdrawn items are never asked. Records nothing and writes nothing in
/// `round-<n>`; the variant stores are rebuilt each call. An answer that
/// is not a usable lowering drops its pair and is counted in `unusable`.
@ -1190,6 +1247,7 @@ impl RunStores {
.find(|r| r.n() == n && r.cycle().is_none())
.ok_or(RunError::Malformed("round"))?;
let inventory = self.inventory.inventory().map_err(store)?;
let ontology = self.ontology.ontology().map_err(store)?;
let dir = self.ablation_dir(n);
std::fs::create_dir_all(&dir).map_err(|e| RunError::Io(e.kind()))?;
@ -1242,7 +1300,14 @@ impl RunStores {
let view = stores[slot]
.admit(packet, &self.inventory, &self.corpus, &self.entities)
.map_err(|_| RunError::Store)?;
Prompt::lower(&view, &packet_item, &item.language, &inventory, &self.entities)
Prompt::lower(
&view,
&packet_item,
&item.language,
&inventory,
&self.entities,
&ontology,
)
.map_err(|_| RunError::Store)
};
let base = prompt_of(0, packet.clone())?;
@ -1444,6 +1509,7 @@ fn score_round<M: ModelPort, E: Embedder>(
) -> Result<RoundOutcome, RunError> {
let store = |_| RunError::Store;
let inventory = inventory_store.inventory().map_err(store)?;
let ontology = run.ontology.ontology().map_err(store)?;
let rev = inventory.latest().ok_or(RunError::NoInventory)?.n;
let train = run.train().map_err(store)?;
@ -1524,7 +1590,7 @@ fn score_round<M: ModelPort, E: Embedder>(
// Each item is lowered in its own language: the only reference the
// scorer has is the item's own text.
let language = &items[*at].language;
let Ok(prompt) = Prompt::lower(view, id, language, &inventory, &run.entities) else {
let Ok(prompt) = Prompt::lower(view, id, language, &inventory, &run.entities, &ontology) else {
fates[*at] = Fate::Refused(Stage::Lower, Some(packet_key.clone()));
continue;
};
@ -2725,6 +2791,8 @@ mod tests {
lowerer_reach: LowererReach::ToolLessRelay,
answerer_family: ModelFamily::Unrecorded,
panel: vec!["en-US".into()],
ontology: String::new(),
ontology_digest: String::new(),
}
}
@ -2915,6 +2983,8 @@ mod tests {
copy("inventory.cc", "i.cc"),
copy("entity.cc", "e.cc"),
);
let ontology = dir.path().join("o.cc");
fs::write(&ontology, [0x90]).unwrap();
let root = dir.path().join("run");
let mut run = RunStores::create(
&root,
@ -2923,6 +2993,7 @@ mod tests {
corpus: &corpus,
inventory: &inventory,
entities: &entities,
ontology: &ontology,
},
"fake-model",
"bag",

2
crates/weksa/src/schema.rs
View file

@ -4,6 +4,7 @@
use crate::corpus::CorpusItem;
use crate::entity::Entity;
use crate::inventory::{InventoryEntry, InventoryRevision};
use crate::ontology::OntologyEntry;
use crate::packet::Packet;
use crate::run::{Decision, RoundRecord, RunRecord};
use crate::speaker::{Scene, Speaker};
@ -25,6 +26,7 @@ pub fn published() -> Vec<(String, String)> {
published_for::<Entity>(),
published_for::<Speaker>(),
published_for::<Scene>(),
published_for::<OntologyEntry>(),
published_for::<RunRecord>(),
published_for::<RoundRecord>(),
published_for::<Decision>(),

136
crates/weksa/src/store.rs
View file

@ -19,7 +19,11 @@ use crate::form::{
use crate::inventory::{
Change, InvId, Inventory, InventoryEntry, InventoryRevision, Slot, revision_key,
};
use crate::packet::{EntityId, Packet};
use crate::ontology::{
MAX_CONDITIONS, MAX_CHOICE_BYTES, MAX_ENTRIES_PER_LANGUAGE, MAX_EXAMPLE_BYTES, MAX_EXAMPLES,
Ontology, OntologyEntry, Test,
};
use crate::packet::{EntityId, FeatValue, Packet};
use crate::speaker::{Annotator, MAX_PARTICIPANTS, Party, Scene, Speaker, scene_key, speaker_key};
use crate::validate::{LoweringInput, Refusal, feature_refusals, validate};
@ -1207,3 +1211,133 @@ fn check_next(key: &str, latest: u32, got: u32) -> Result<(), StoreError> {
}
Ok(())
}
/// The language ontology: immutable entries admitted in all-or-nothing
/// batches. The only writer of `weksa.ontology_entry`.
pub struct OntologyStore {
cache: CultCache,
}
impl OntologyStore {
pub fn open(path: &Path) -> anyhow::Result<Self> {
Ok(Self {
cache: open_cache(path, |c| c.register_entry_type::<OntologyEntry>())?,
})
}
/// Every entry, by language then number.
pub fn all(&self) -> anyhow::Result<Vec<OntologyEntry>> {
Ok(self.ontology()?.entries().to_vec())
}
pub fn ontology(&self) -> anyhow::Result<Ontology> {
Ok(Ontology::of(self.cache.get_all::<OntologyEntry>()?))
}
/// Admits a batch whole or not at all; an identical replay of stored
/// entries is `Unchanged`. Per language the batch's numbers run the
/// language's latest plus one, consecutively, to at most
/// `MAX_ENTRIES_PER_LANGUAGE` entries in all. Every entry passes
/// [`check_ontology_entry`] against the inventory before any is put. No
/// message echoes an entry's text.
pub fn admit(
&mut self,
batch: Vec<OntologyEntry>,
inventory: &InventoryStore,
) -> Result<AddOutcome, StoreError> {
let all = self.cache.get_all::<OntologyEntry>()?;
if batch
.iter()
.all(|e| all.iter().any(|old| old == e))
{
return Ok(AddOutcome::Unchanged);
}
let inv = inventory.inventory()?;
let mut counted: std::collections::BTreeMap<&str, u32> = std::collections::BTreeMap::new();
for e in &batch {
let key = e.key();
if !is_language_tag(&e.language) {
return Err(malformed(&key, "an ontology language is a language tag"));
}
let latest = match counted.get(e.language.as_str()) {
Some(n) => *n,
None => all
.iter()
.filter(|o| o.language == e.language)
.map(|o| o.n)
.max()
.unwrap_or(0),
};
check_next(&key, latest, e.n)?;
if e.n as usize > MAX_ENTRIES_PER_LANGUAGE {
return Err(malformed(&key, "a language holds at most 64 entries"));
}
counted.insert(e.language.as_str(), e.n);
check_ontology_entry(e, &inv)?;
}
let mut staged = Vec::new();
for e in &batch {
staged.push(self.cache.prepare_entry_named(e.key(), e)?.0);
}
self.cache.put_prepared_batch(staged)?;
Ok(AddOutcome::Added)
}
}
/// The rules of one ontology entry against an inventory: the annotator, the
/// bounded and normalized text of the choice and examples, the condition count
/// and each condition's feature (slot `Stance` or `Discourse`, a grade feature
/// for `AtLeast` and `AtMost`, an option or flag value for `Is`) read at the
/// entry's inventory revision. The store applies it on admission and a run
/// applies it again against its own head inventory.
pub(crate) fn check_ontology_entry(
e: &OntologyEntry,
inventory: &Inventory,
) -> Result<(), StoreError> {
let key = e.key();
check_annotator(&key, &e.annotator)?;
let text_ok = |t: &str, max: usize| {
(1..=max).contains(&t.len())
&& !t.chars().any(|c| c.is_control() || is_refused_format(c))
&& is_nfc(t)
};
if !text_ok(&e.choice, MAX_CHOICE_BYTES) {
return Err(malformed(
&key,
"a choice is 1 to 200 normalized bytes with no control or format character",
));
}
if e.examples.len() > MAX_EXAMPLES
|| !e.examples.iter().all(|x| text_ok(x, MAX_EXAMPLE_BYTES))
{
return Err(malformed(
&key,
"an entry has at most 3 examples of 1 to 120 normalized bytes with no control or format character",
));
}
if e.when.len() > MAX_CONDITIONS {
return Err(malformed(&key, "an entry has at most 4 conditions"));
}
let mut out = Vec::new();
for c in &e.when {
if !matches!(c.slot, Slot::Stance | Slot::Discourse) {
return Err(malformed(&key, "a condition reads slot Stance or Discourse"));
}
if matches!(c.test, Test::Is(FeatValue::Grade(_))) {
return Err(malformed(&key, "Is names an option or a flag"));
}
out.extend(feature_refusals(
inventory,
e.inventory_rev,
&[c.as_value()],
c.slot,
));
}
if e.when.is_empty() && !inventory.has_revision(e.inventory_rev) {
out.extend(feature_refusals(inventory, e.inventory_rev, &[], Slot::Stance));
}
if !out.is_empty() {
return Err(StoreError::Invalid(out));
}
Ok(())
}

48
crates/weksa/tests/cli.rs
View file

@ -12,6 +12,8 @@ fn weksa(dir: &std::path::Path, args: &[&str]) -> Output {
"./i.cc",
"--entities",
"./e.cc",
"--ontology",
"./o.cc",
])
.args(args)
.output()
@ -316,6 +318,7 @@ fn schema_writes_the_published_files() {
"weksa.loop_decision.schema.json",
"weksa.loop_round.schema.json",
"weksa.loop_run.schema.json",
"weksa.ontology_entry.schema.json",
"weksa.scene.schema.json",
"weksa.speaker.schema.json"
]
@ -1337,6 +1340,7 @@ fn dao_stores(dir: &std::path::Path) {
] {
std::fs::copy(src.join(from), dir.join(to)).unwrap();
}
std::fs::write(dir.join("o.cc"), [0x90]).unwrap();
}
#[test]
@ -1763,3 +1767,47 @@ fn panel_and_language_are_required() {
assert!(!o.status.success());
assert!(out(&o).contains("--language"), "{}", out(&o));
}
#[test]
fn ontology_admit_and_dump() {
let d = tempfile::tempdir().unwrap();
dao_stores(d.path());
let draft = |name: &str, body: &str| {
std::fs::write(d.path().join(name), body).unwrap();
};
draft(
"good.json",
r#"{"entries": [{"layer": "Discourse", "when": [{"slot": "Stance", "feature": "feature:18", "test": {"AtLeast": 600}}], "choice": "SOFTCHOICE", "examples": ["EXAMPLETEXT"]}, {"layer": "Lexicon", "choice": "plain"}]}"#,
);
let admit = |file: &str| {
weksa(
d.path(),
&[
"ontology", "admit", file, "--language", "ja-JP", "--inventory-rev", "1",
"--model-family", "anthropic", "--generator", "a drafting model",
],
)
};
let o = admit("good.json");
assert!(o.status.success(), "{}", out(&o));
let o = weksa(d.path(), &["ontology", "dump"]);
let text = out(&o);
assert_eq!(text.matches("onto:ja-JP:").count(), 2, "{text}");
for want in ["onto:ja-JP:1", "onto:ja-JP:2", "Anthropic", "a drafting model", "SOFTCHOICE", "EXAMPLETEXT", "Stance"] {
assert!(text.contains(want), "{want}: {text}");
}
// A second draft numbers on from the language's latest.
let o = admit("good.json");
assert!(o.status.success(), "{}", out(&o));
assert!(out(&weksa(d.path(), &["ontology", "dump"])).contains("onto:ja-JP:4"));
// A refused draft writes nothing and echoes no choice text.
draft(
"bad.json",
r#"{"entries": [{"layer": "Discourse", "when": [{"slot": "Stance", "feature": "feature:9999", "test": {"AtLeast": 600}}], "choice": "LEAKYCHOICE"}]}"#,
);
let before = out(&weksa(d.path(), &["ontology", "dump"]));
let o = admit("bad.json");
assert!(!o.status.success());
assert!(!out(&o).contains("LEAKYCHOICE"), "{}", out(&o));
assert_eq!(out(&weksa(d.path(), &["ontology", "dump"])), before);
}

12
crates/weksa/tests/common/mod.rs
View file

@ -114,6 +114,15 @@ pub fn entities_in(dir: &std::path::Path) -> EntityStore {
store
}
/// An empty ontology store at `<dir>/ontology.cc`, made if absent.
pub fn ontology_in(dir: &std::path::Path) -> std::path::PathBuf {
let path = dir.join("ontology.cc");
if !path.exists() {
std::fs::write(&path, [0x90]).unwrap();
}
path
}
/// A run over the stores `world()` and `corpus()` keep in `dir`: `c.cc`,
/// `i.cc` and `entities.cc`. Read at the call, so create it after the items.
/// The run's en-US panel needs a live Train and a live Validation item: a
@ -125,6 +134,7 @@ pub fn run_over(dir: &std::path::Path) -> weksa::run::RunStores {
}
}
let entities = entities_in(dir);
ontology_in(dir);
let mut corpus = CorpusStore::open(&dir.join("c.cc")).unwrap();
for (n, split) in [(1, weksa::Split::Train), (2, weksa::Split::Validation)] {
let held = corpus
@ -147,6 +157,7 @@ pub fn run_over(dir: &std::path::Path) -> weksa::run::RunStores {
corpus: &dir.join("c.cc"),
inventory: &dir.join("i.cc"),
entities: &dir.join("entities.cc"),
ontology: &dir.join("ontology.cc"),
},
"fake-model",
"bag",
@ -230,6 +241,7 @@ pub fn world() -> World {
let dir = tempfile::tempdir().unwrap();
let mut corpus = CorpusStore::open(&dir.path().join("c.cc")).unwrap();
let entities = entities_in(dir.path());
ontology_in(dir.path());
let train = group_in_bucket(10);
let mut first = item_in("item:nibu:1", "Nibu says mind the hatch.", &train, cast());
first.names = vec![NameSpan {

165
crates/weksa/tests/context.rs
View file

@ -537,9 +537,10 @@ mod lowering {
use weksa::daodejing::packets;
use weksa::entity::{Entity, EntityOrigin, LanguageForms};
use weksa::packet::{EntityId, Packet, proof_packet};
use weksa::packet::Grade;
use weksa::{
CorpusItemId, CorpusStore, EntityForms, EntityStore, Inventory, InventoryStore,
LowerError, LoweringInput, PacketStore,
Annotator, Condition, CorpusItemId, CorpusStore, EntityForms, EntityStore, Inventory,
InventoryStore, Layer, LowerError, LoweringInput, OntologyEntry, OntologyStore, PacketStore, Test, select,
};
fn data(sub: &str) -> PathBuf {
@ -602,7 +603,7 @@ mod lowering {
fn lower_prompt_names_its_language() {
let (view, inv, entities, item) = dao(2);
for tag in ["en-US", "ja-JP", "pt-BR"] {
let prompt = Prompt::lower(&view, &item, tag, &inv, &entities).unwrap();
let prompt = Prompt::lower(&view, &item, tag, &inv, &entities, &weksa::Ontology::empty()).unwrap();
let want = format!("{LOWER}\n\nlanguage: {tag}\n\npacket: ");
assert!(prompt.text().starts_with(&want), "{tag}");
}
@ -614,10 +615,10 @@ mod lowering {
#[test]
fn lower_prompt_text_is_pinned() {
let (view, inv, entities, item) = dao(2);
let prompt = Prompt::lower(&view, &item, "en-US", &inv, &entities).unwrap();
let prompt = Prompt::lower(&view, &item, "en-US", &inv, &entities, &weksa::Ontology::empty()).unwrap();
let (head, listing) = prompt.text().split_once("inventory rev:3\n").unwrap();
let expected = format!(
"{LOWER}\n\nlanguage: en-US\n\npacket: {}\n\nentities: {{}}\n\n",
"{LOWER}\n\nlanguage: en-US\n\npacket: {}\n\nentities: {{}}\n\nontology: none\n\n",
concat!(
r#"{"context":{"setting":null,"medium":null,"participants":[],"audience":[]},"#,
r#""discourse":{"speech_act":"speech_act:1","addressee":null,"addressee_consent":null,"features":[]},"#,
@ -676,7 +677,7 @@ mod lowering {
let mut packet = packets().remove(2);
packet.stance = stance.clone();
let view = admitted(packet, &store, "daodejing/corpus.cc", &entities);
let prompt = Prompt::lower(&view, &item, "en-US", &inv, &entities).unwrap();
let prompt = Prompt::lower(&view, &item, "en-US", &inv, &entities, &weksa::Ontology::empty()).unwrap();
(stance, prompt)
};
let three = [prompt_of(own), prompt_of(angry), prompt_of(Stance::default())];
@ -713,7 +714,7 @@ mod lowering {
("sa-Latn", &["Nibuwa"]),
("en-US", &["Nibu", "Nibu's"]),
])]);
let prompt = Prompt::lower(&view, &item, "en-US", &cast.inventory().unwrap(), &held).unwrap();
let prompt = Prompt::lower(&view, &item, "en-US", &cast.inventory().unwrap(), &held, &weksa::Ontology::empty()).unwrap();
assert!(
prompt
.text()
@ -743,9 +744,157 @@ mod lowering {
),
(Held(vec![]), LowerError::UnknownEntity { entity: id }),
] {
assert_eq!(Prompt::lower(&view, &item, "en-US", &inv, &held).err(), Some(want));
assert_eq!(Prompt::lower(&view, &item, "en-US", &inv, &held, &weksa::Ontology::empty()).err(), Some(want));
}
}
fn feature(inv: &Inventory, gloss: &str) -> weksa::InvId {
inv.entries().find(|e| e.gloss == gloss).unwrap().id
}
fn entry(language: &str, n: u32, when: Vec<Condition>, choice: &str) -> OntologyEntry {
OntologyEntry {
language: language.into(),
n,
inventory_rev: 1,
layer: Layer::Discourse,
when,
choice: choice.into(),
examples: vec![],
annotator: Annotator::Operator,
}
}
/// The ontology the store holds after admitting `entries` as one batch.
fn ontology_of(cast: &InventoryStore, entries: Vec<OntologyEntry>) -> weksa::Ontology {
let dir = tempfile::tempdir().unwrap();
let mut store = OntologyStore::open(&dir.path().join("o.cc")).unwrap();
store.admit(entries, cast).unwrap();
store.ontology().unwrap()
}
fn at_least(inv: &Inventory, gloss: &str, g: u16) -> Condition {
Condition {
slot: weksa::Slot::Stance,
feature: feature(inv, gloss),
test: Test::AtLeast(Grade::new(g).unwrap()),
}
}
/// The proof packet's view with the given stance restraint and discourse irony.
fn view_of(
restraint: Option<u16>,
irony: Option<bool>,
cast: &InventoryStore,
entities: &EntityStore,
) -> LoweringInput {
use weksa::packet::{FeatValue, FeatureValue};
let inv = cast.inventory().unwrap();
let mut packet = proof_packet();
packet.stance.features = restraint
.map(|v| FeatureValue {
feature: feature(&inv, "restraint"),
value: FeatValue::Grade(Grade::new(v).unwrap()),
})
.into_iter()
.collect();
packet.discourse.features = irony
.map(|v| FeatureValue {
feature: feature(&inv, "irony"),
value: FeatValue::Flag(v),
})
.into_iter()
.collect();
admitted(packet, cast, "corpus/corpus.cc", entities)
}
#[test]
fn select_reads_stance_and_discourse_of_its_language() {
use weksa::packet::FeatValue;
let cast = InventoryStore::open(&data("interlingua/inventory.cc")).unwrap();
let entities = EntityStore::open(&data("entity/entity.cc")).unwrap();
let inv = cast.inventory().unwrap();
let ontology = ontology_of(&cast, vec![
entry("ja-JP", 1, vec![at_least(&inv, "restraint", 600)], "high"),
entry(
"ja-JP",
2,
vec![Condition {
slot: weksa::Slot::Stance,
feature: feature(&inv, "restraint"),
test: Test::AtMost(Grade::new(300).unwrap()),
}],
"low",
),
entry(
"ja-JP",
3,
vec![Condition {
slot: weksa::Slot::Discourse,
feature: feature(&inv, "irony"),
test: Test::Is(FeatValue::Flag(true)),
}],
"ironic",
),
entry("ja-JP", 4, vec![], "always"),
entry("en-US", 1, vec![], "other language"),
]);
let keys = |view: &LoweringInput, language: &str| -> Vec<String> {
select(&ontology, language, view).iter().map(|e| e.key()).collect()
};
let both = view_of(Some(700), Some(true), &cast, &entities);
assert_eq!(keys(&both, "ja-JP"), ["onto:ja-JP:1", "onto:ja-JP:3", "onto:ja-JP:4"]);
assert_eq!(keys(&both, "en-US"), ["onto:en-US:1"]);
let low = view_of(Some(200), Some(false), &cast, &entities);
assert_eq!(keys(&low, "ja-JP"), ["onto:ja-JP:2", "onto:ja-JP:4"]);
// A grade equal to the bound meets AtLeast and AtMost alike.
let edge = view_of(Some(600), None, &cast, &entities);
assert_eq!(keys(&edge, "ja-JP"), ["onto:ja-JP:1", "onto:ja-JP:4"]);
let edge = view_of(Some(300), None, &cast, &entities);
assert_eq!(keys(&edge, "ja-JP"), ["onto:ja-JP:2", "onto:ja-JP:4"]);
// An absent feature meets no condition, however lax.
let bare = view_of(None, None, &cast, &entities);
assert_eq!(keys(&bare, "ja-JP"), ["onto:ja-JP:4"]);
}
#[test]
fn lowering_prompt_prints_only_selected_entries() {
let cast = InventoryStore::open(&data("interlingua/inventory.cc")).unwrap();
let cast_entities = EntityStore::open(&data("entity/entity.cc")).unwrap();
let inv = cast.inventory().unwrap();
let mut hit = entry("ja-JP", 1, vec![at_least(&inv, "restraint", 600)], "HITCHOICE");
hit.examples = vec!["HITEXAMPLE".into()];
let ontology = ontology_of(&cast, vec![
hit,
entry("ja-JP", 2, vec![at_least(&inv, "restraint", 900)], "MISSCHOICE"),
entry("en-US", 1, vec![], "ENGLISHCHOICE"),
]);
let held = Held(vec![nibu(&[("ja-JP", &["ニブ"]), ("en-US", &["Nibu"])])]);
let item = CorpusItemId::parse("item:nibu:1").unwrap();
let view = view_of(Some(700), None, &cast, &cast_entities);
let lowered = |view: &LoweringInput, o: &weksa::Ontology| {
Prompt::lower(view, &item, "ja-JP", &inv, &held, o).unwrap()
};
let prompt = lowered(&view, &ontology);
let text = prompt.text();
assert!(text.contains("\n\nontology:\nonto:ja-JP:1 [discourse] HITCHOICE | e.g. HITEXAMPLE\n\n"), "{text}");
for absent in ["MISSCHOICE", "ENGLISHCHOICE", "onto:ja-JP:2", "onto:en-US:1"] {
assert!(!text.contains(absent), "{absent}");
}
// The same view with no entries: the section reads none.
let none = lowered(&view, &weksa::Ontology::empty());
assert!(none.text().contains("\n\nontology: none\n\n"));
// Entries move the prompt and nothing else of the view does.
assert_ne!(prompt.digest(), none.digest());
// The group is still private; the speaker's entity appears only because
// the packet cites it on a referent.
let held_item = CorpusStore::open(&data("corpus/corpus.cc"))
.unwrap()
.get(item.as_str())
.unwrap()
.unwrap();
assert!(!text.contains(held_item.group.as_str()));
}
}
/// First real rounds: five of six encoders wrote bare integers (`"concept": 7`)

278
crates/weksa/tests/golf_loop.rs
View file

@ -24,8 +24,13 @@ fn rewrite_run(root: &Path, change: impl FnOnce(&mut RunRecord)) {
cache.put(format!("run:{}", record.label), &record).unwrap();
}
fn paths(dir: &Path) -> (PathBuf, PathBuf, PathBuf) {
(dir.join("c.cc"), dir.join("i.cc"), dir.join("entities.cc"))
fn paths(dir: &Path) -> (PathBuf, PathBuf, PathBuf, PathBuf) {
(
dir.join("c.cc"),
dir.join("i.cc"),
dir.join("entities.cc"),
common::ontology_in(dir),
)
}
fn hex(path: &Path) -> String {
@ -37,7 +42,7 @@ fn a_run_pins_its_stores_and_refuses_drift() {
let mut w = world();
let dir = w.dir.path().to_path_buf();
let root = dir.join("run");
let (c, i, e) = paths(&dir);
let (c, i, e, o) = paths(&dir);
let at = |label: &str, model: &str| {
RunStores::create(
&root,
@ -46,6 +51,7 @@ fn a_run_pins_its_stores_and_refuses_drift() {
corpus: &c,
inventory: &i,
entities: &e,
ontology: &o,
},
model,
"bag",
@ -141,7 +147,7 @@ fn a_run_works_on_its_own_copy_of_the_inventory() {
let w = world();
let dir = w.dir.path().to_path_buf();
let root = dir.join("run");
let (c, i, e) = paths(&dir);
let (c, i, e, o) = paths(&dir);
let source_bytes = std::fs::read(&i).unwrap();
let mut run = RunStores::create(
&root,
@ -150,6 +156,7 @@ fn a_run_works_on_its_own_copy_of_the_inventory() {
corpus: &c,
inventory: &i,
entities: &e,
ontology: &o,
},
"fake-model",
"bag",
@ -228,6 +235,7 @@ fn create_refuses_a_root_that_holds_an_inventory_and_never_truncates_it() {
corpus: &dao.corpus,
inventory,
entities: &dao.entities,
ontology: &dao.ontology,
},
"fake-model",
"bag",
@ -276,6 +284,7 @@ fn a_create_that_fails_leaves_no_inventory_and_the_retry_succeeds() {
corpus: &dao.corpus,
inventory,
entities,
ontology: &dao.ontology,
},
"fake-model",
"bag",
@ -388,10 +397,12 @@ fn a_store_changed_between_its_digest_and_its_open_is_drift() {
assert!(dir.join(name).exists(), "{name}");
}
let (c, i, e) = (dir.join("c.cc"), dir.join("i.cc"), dir.join("entities.cc"));
let o = dir.join("ontology.cc");
let paths = || weksa::run::RunPaths {
corpus: &c,
inventory: &i,
entities: &e,
ontology: &o,
};
assert_eq!(
RunStores::create_between(
@ -440,10 +451,12 @@ fn window(
let dir = w.dir.path().to_path_buf();
let root = dir.join("run");
let (c, i, e) = (dir.join("c.cc"), dir.join("i.cc"), dir.join("entities.cc"));
let o = dir.join("ontology.cc");
let paths = weksa::run::RunPaths {
corpus: &c,
inventory: &i,
entities: &e,
ontology: &o,
};
let created = RunStores::create_between(
&root,
@ -501,12 +514,13 @@ fn source_digests_cover_their_files() {
let digest = |name: &str| fnv1a64(&read(name));
// The prompt's partition and names come from split.rs, and the lowering
// prompt reads lower.rs's target language and entity port and prints
// packet.rs's serde shapes: all four are covered.
let (context, split, lower, packet) = (
// packet.rs's serde shapes and ontology.rs's selection: all five are covered.
let (context, split, lower, packet, ontology) = (
read("context.rs"),
read("split.rs"),
read("lower.rs"),
read("packet.rs"),
read("ontology.rs"),
);
// Each file enters behind its length, so the parts have one framing.
let framed = [
@ -518,12 +532,14 @@ fn source_digests_cover_their_files() {
&lower,
(packet.len() as u64).to_le_bytes().as_slice(),
&packet,
(ontology.len() as u64).to_le_bytes().as_slice(),
&ontology,
]
.concat();
assert_eq!(weksa::run::PROMPT_SOURCE_DIGEST, fnv1a64(&framed));
assert_ne!(
weksa::run::PROMPT_SOURCE_DIGEST,
fnv1a64(&[context.clone(), split.clone(), lower.clone(), packet.clone()].concat()),
fnv1a64(&[context.clone(), split.clone(), lower.clone(), packet.clone(), ontology.clone()].concat()),
"bare concatenation is not the digest"
);
// Moving the boundary between the parts moves the digest.
@ -532,6 +548,7 @@ fn source_digests_cover_their_files() {
weksa::split::fnv1a64_of(&[b"a", b"bc"])
);
assert_ne!(weksa::run::PROMPT_SOURCE_DIGEST, digest("context.rs"));
assert_ne!(weksa::run::PROMPT_SOURCE_DIGEST, digest("ontology.rs"));
}
/// The encode prompt of `item:nibu:1` over `world()` at revision 4.
@ -837,6 +854,7 @@ struct Dao {
corpus: PathBuf,
inventory: PathBuf,
entities: PathBuf,
ontology: PathBuf,
run: RunStores,
}
@ -853,6 +871,16 @@ fn dao_over(edit: impl FnOnce(&Path, &Path)) -> Dao {
/// `dao_over` with the run opened on `panel`.
fn dao_panel(edit: impl FnOnce(&Path, &Path), panel: &[&str]) -> Dao {
dao_build(edit, panel, |_, _| {})
}
/// `dao_panel` with `seed_ontology` (given the ontology and inventory paths)
/// run before the run pins the ontology.
fn dao_build(
edit: impl FnOnce(&Path, &Path),
panel: &[&str],
seed_ontology: impl FnOnce(&Path, &Path),
) -> Dao {
let panel: Vec<String> = panel.iter().map(|t| t.to_string()).collect();
let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/daodejing");
let dir = tempfile::tempdir().unwrap();
@ -866,6 +894,8 @@ fn dao_panel(edit: impl FnOnce(&Path, &Path), panel: &[&str]) -> Dao {
copy("entity.cc", "e.cc"),
);
edit(&corpus, &entities);
let ontology = common::ontology_in(dir.path());
seed_ontology(&ontology, &inventory);
let root = dir.path().join("run");
let run = RunStores::create(
&root,
@ -874,6 +904,7 @@ fn dao_panel(edit: impl FnOnce(&Path, &Path), panel: &[&str]) -> Dao {
corpus: &corpus,
inventory: &inventory,
entities: &entities,
ontology: &ontology,
},
"fake-model",
"bag",
@ -888,6 +919,7 @@ fn dao_panel(edit: impl FnOnce(&Path, &Path), panel: &[&str]) -> Dao {
corpus,
inventory,
entities,
ontology,
run,
}
}
@ -1055,7 +1087,7 @@ fn lowering_prompts(dao: &Dao, inventory: &InventoryStore) -> Vec<(String, Promp
let view = store
.admit(packet, inventory, &corpus, dao.run.entities())
.unwrap();
let prompt = Prompt::lower(&view, &key, "en-US", &inv, dao.run.entities()).unwrap();
let prompt = Prompt::lower(&view, &key, "en-US", &inv, dao.run.entities(), &weksa::Ontology::empty()).unwrap();
(key.to_string(), prompt)
})
.collect()
@ -1527,6 +1559,7 @@ fn errors_echo_no_input() {
corpus: &missing,
inventory: &missing,
entities: &missing,
ontology: &missing,
};
let e = RunStores::create(
&root,
@ -1595,6 +1628,7 @@ fn errors_echo_no_input() {
corpus: &dao.corpus,
inventory: &dao.inventory,
entities: &dao.entities,
ontology: &dao.ontology,
},
"fake-model",
"bag",
@ -2516,6 +2550,7 @@ fn a_changed_head_is_drift() {
corpus: &dao.corpus,
inventory: &dao.inventory,
entities: &dao.entities,
ontology: &dao.ontology,
},
"fake-model",
"bag",
@ -2626,6 +2661,7 @@ fn source_tripwires() {
assert!(!signature.contains(banned), "Prompt::lower takes {banned}");
}
assert!(signature.contains("&LoweringInput") && signature.contains("&CorpusItemId"));
assert!(signature.contains("&Ontology"), "the whole ontology goes in; select chooses");
assert_eq!(count("admit_revision("), 1, "one door call");
assert_eq!(
count("std::fs::copy"),
@ -2854,6 +2890,7 @@ fn a_summary_line_says_how_lowerings_were_answered() {
corpus: &dao.corpus,
inventory: &dao.inventory,
entities: &dao.entities,
ontology: &dao.ontology,
};
RunStores::create(
&root,
@ -3577,6 +3614,7 @@ fn mixed_family_round(family: ModelFamily) -> u32 {
corpus: &dir.join("c.cc"),
inventory: &dir.join("i.cc"),
entities: &dir.join("entities.cc"),
ontology: &dir.join("ontology.cc"),
},
"not-a-draft",
"bag",
@ -4355,6 +4393,7 @@ fn open_refuses_a_bad_panel() {
vec![],
vec![tag("en-US"), tag("en-US")],
vec![tag("en-US"), tag("not a tag!")],
vec![tag("en-us")],
// No live item at all, and a Train item without a Validation one.
vec![tag("en-US"), tag("de-DE")],
vec![tag("en-US"), tag("fr-FR")],
@ -4363,7 +4402,7 @@ fn open_refuses_a_bad_panel() {
let created = RunStores::create(
&root,
"p",
RunPaths { corpus, inventory, entities },
RunPaths { corpus, inventory, entities, ontology: &dao.ontology },
"fake-model",
"bag",
weksa::run::LowererReach::ToolLessRelay,
@ -4557,3 +4596,224 @@ fn ablation_reads_each_language_apart() {
[("en-US", Train, 1), ("en-US", Validation, 2), ("ja-JP", Train, 1), ("ja-JP", Validation, 2)]
);
}
/// A Dao ontology for en-US and ja-JP: entry 1 of each for a stance with
/// affection, and a ja-JP entry 2 for almost no restraint (which no Dao packet
/// has).
fn dao_ontology(path: &Path, inventory: &Path) {
use weksa::packet::Grade;
use weksa::{Condition, Layer, OntologyEntry, OntologyStore, Slot, Test};
let inv = InventoryStore::open(inventory).unwrap();
let feature = |gloss: &str| {
inv.inventory()
.unwrap()
.entries()
.find(|e| e.gloss == gloss)
.unwrap()
.id
};
let entry = |language: &str, n, gloss: &str, test, choice: &str| OntologyEntry {
language: language.into(),
n,
inventory_rev: 1,
layer: Layer::Discourse,
when: vec![Condition {
slot: Slot::Stance,
feature: feature(gloss),
test,
}],
choice: choice.into(),
examples: vec![],
annotator: weksa::Annotator::Operator,
};
let affection = || Test::AtLeast(Grade::new(200).unwrap());
OntologyStore::open(path)
.unwrap()
.admit(
vec![
entry("en-US", 1, "affection", affection(), "ENWARM"),
entry("ja-JP", 1, "affection", affection(), "JAWARM"),
entry(
"ja-JP",
2,
"restraint",
Test::AtMost(Grade::new(100).unwrap()),
"JABLUNT",
),
],
&inv,
)
.unwrap();
}
fn inv_store(path: &Path) -> InventoryStore {
InventoryStore::open(path).unwrap()
}
#[test]
fn a_run_pins_its_ontology_and_refuses_drift() {
let dao = dao_build(|_, _| {}, &["en-US"], dao_ontology);
let r = dao.run.record();
assert_eq!(r.ontology, std::fs::canonicalize(&dao.ontology).unwrap().to_str().unwrap());
assert_eq!(r.ontology_digest, hex(&dao.ontology));
let root = dao.root.clone();
assert_eq!(RunStores::open(&root).unwrap().record(), dao.run.record());
// Another entry after the pin is another ontology.
let entry = weksa::OntologyStore::open(&dao.ontology).unwrap().all().unwrap().remove(0);
let mut store = weksa::OntologyStore::open(&dao.ontology).unwrap();
store
.admit(
vec![weksa::OntologyEntry { n: 2, ..entry }],
&inv_store(&dao.inventory),
)
.unwrap();
assert_eq!(
RunStores::open(&root).err(),
Some(RunError::Drift(DriftOf::Ontology))
);
}
#[test]
fn create_refuses_an_entry_its_inventory_cannot_read() {
use weksa::packet::Grade;
use weksa::{Condition, Layer, OntologyEntry, OntologyStore, Slot, Test};
// The entry reads feature 18 at revision 1; the run's head inventory is
// another file where that feature is retired or absent, so the entry is
// admitted against a richer inventory and refused against the run's.
let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/daodejing");
let make = |language: &str, dir: &Path, retire: bool| {
for (from, to) in [("corpus.cc", "c.cc"), ("inventory.cc", "i.cc"), ("entity.cc", "e.cc")] {
std::fs::copy(src.join(from), dir.join(to)).unwrap();
}
let ontology = dir.join("o.cc");
// A richer inventory the entry is admitted against: the Dao one plus a feature.
let rich = dir.join("rich.cc");
std::fs::copy(src.join("inventory.cc"), &rich).unwrap();
let mut store = InventoryStore::open(&rich).unwrap();
let inv = store.inventory().unwrap();
let head = inv.latest().unwrap().n;
let n = head + 1;
let next = std::num::NonZeroU32::new(
inv.entries().filter(|e| e.id.family() == weksa::Family::Feature).map(|e| e.id.n().get()).max().unwrap() + 1,
)
.unwrap();
let id = weksa::Family::Feature.id(next);
store
.admit_revision(
weksa::InventoryRevision {
n,
parent: Some(head),
changes: vec![weksa::Change::Add(id)],
frozen: false,
},
vec![weksa::InventoryEntry {
id,
gloss: "pride".into(),
value_kind: Some(weksa::ValueKind::Grade),
applies_to: vec![Slot::Stance],
introduced_rev: n,
retired_rev: None,
}],
)
.unwrap();
OntologyStore::open(&ontology)
.unwrap()
.admit(
vec![OntologyEntry {
language: language.into(),
n: 1,
inventory_rev: n,
layer: Layer::Lexicon,
when: vec![Condition {
slot: Slot::Stance,
feature: id,
test: Test::AtLeast(Grade::new(1).unwrap()),
}],
choice: "boast".into(),
examples: vec![],
annotator: weksa::Annotator::Operator,
}],
&store,
)
.unwrap();
if retire {
// The run's head inventory retires the feature after the entry's revision.
store
.admit_revision(
weksa::InventoryRevision {
n: n + 1,
parent: Some(n),
changes: vec![weksa::Change::Retire(id)],
frozen: false,
},
vec![],
)
.unwrap();
std::fs::copy(&rich, dir.join("i.cc")).unwrap();
}
ontology
};
let create = |dir: &Path, ontology: &Path| {
RunStores::create(
&dir.join("run"),
"x",
RunPaths {
corpus: &dir.join("c.cc"),
inventory: &dir.join("i.cc"),
entities: &dir.join("e.cc"),
ontology,
},
"fake-model",
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&["en-US".to_string()],
)
};
let panel_dir = tempfile::tempdir().unwrap();
let o = make("en-US", panel_dir.path(), false);
assert_eq!(create(panel_dir.path(), &o).err(), Some(RunError::Ontology));
assert!(!panel_dir.path().join("run/run.cc").exists());
assert!(!panel_dir.path().join("run/inventory.cc").exists());
// The same entry under a language outside the panel does not refuse.
let other_dir = tempfile::tempdir().unwrap();
let o = make("ja-JP", other_dir.path(), false);
assert!(create(other_dir.path(), &o).is_ok());
// An entry on a feature the head inventory has retired can never select.
let retired_dir = tempfile::tempdir().unwrap();
let o = make("en-US", retired_dir.path(), true);
assert_eq!(create(retired_dir.path(), &o).err(), Some(RunError::Ontology));
assert!(!retired_dir.path().join("run/run.cc").exists());
}
#[test]
fn round_lowering_prompts_carry_selected_entries() {
let mut dao = dao_build(
|corpus, entities| add_languages(corpus, entities, LAOZI_JA),
&["ja-JP", "en-US"],
dao_ontology,
);
let mut model = bilingual_fake();
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
let mut seen = std::collections::BTreeSet::new();
for ((_, text), call) in model.seen.iter().zip(r.calls()) {
if !text.starts_with(LOWER_INSTRUCTIONS) {
continue;
}
let item = call.items[0].as_str();
let (language, warm, other) = if text.contains("\n\nlanguage: ja-JP\n\n") {
("ja-JP", "JAWARM", "ENWARM")
} else {
("en-US", "ENWARM", "JAWARM")
};
// Only the third packet of each language carries affection.
if item.ends_with(":3") || item.ends_with(":13") {
assert!(text.contains(&format!("\nonto:{language}:1 [discourse] {warm}\n")), "{text}");
} else {
assert!(text.contains("\n\nontology: none\n\n"), "{item}: {text}");
}
assert!(!text.contains(other) && !text.contains("JABLUNT"), "{item}: {text}");
seen.insert(item.to_string());
}
assert_eq!(seen.len(), 6, "three items in each language");
}

7
crates/weksa/tests/original_corpus.rs
View file

@ -274,6 +274,8 @@ fn the_recall_probe_covers_every_live_original_item_and_names_no_work() {
tmp.path().join(f)
};
let (c, i, e) = (path("corpus.cc"), path("inventory.cc"), path("entity.cc"));
let o = tmp.path().join("ontology.cc");
std::fs::write(&o, [0x90]).unwrap();
let run = RunStores::create(
&tmp.path().join("run"),
"orig",
@ -281,6 +283,7 @@ fn the_recall_probe_covers_every_live_original_item_and_names_no_work() {
corpus: &c,
inventory: &i,
entities: &e,
ontology: &o,
},
"m",
"bag",
@ -350,6 +353,7 @@ fn original_round(family: ModelFamily) -> (weksa::run::RunStores, weksa::run::Ro
for f in ["corpus.cc", "entity.cc", "inventory.cc"] {
std::fs::copy(src.join(f), tmp.join(f)).unwrap();
}
std::fs::write(tmp.join("ontology.cc"), [0x90]).unwrap();
let mut run = RunStores::create(
&tmp.join("run"),
"orig",
@ -357,6 +361,7 @@ fn original_round(family: ModelFamily) -> (weksa::run::RunStores, weksa::run::Ro
corpus: &tmp.join("corpus.cc"),
inventory: &tmp.join("inventory.cc"),
entities: &tmp.join("entity.cc"),
ontology: &tmp.join("ontology.cc"),
},
"nope",
"bag",
@ -506,6 +511,7 @@ fn no_test_item_is_offered_to_a_round_or_recall_or_recorded() {
for f in ["corpus.cc", "entity.cc", "inventory.cc"] {
std::fs::copy(src.join(f), tmp.path().join(f)).unwrap();
}
std::fs::write(tmp.path().join("ontology.cc"), [0x90]).unwrap();
let p = |f: &str| tmp.path().join(f);
let mut run = RunStores::create(
&p("run"),
@ -514,6 +520,7 @@ fn no_test_item_is_offered_to_a_round_or_recall_or_recorded() {
corpus: &p("corpus.cc"),
inventory: &p("inventory.cc"),
entities: &p("entity.cc"),
ontology: &p("ontology.cc"),
},
"nope",
"bag",

222
crates/weksa/tests/stores.rs
View file

@ -1628,3 +1628,225 @@ fn a_public_domain_source_without_an_edition_year_is_refused() {
.add(item_in(key, "line", "scene-1", source(1891)), &ents(&d))
.unwrap();
}
mod ontology {
use super::*;
use weksa::packet::{FeatValue, Grade};
use weksa::speaker::Annotator;
use weksa::{Condition, Layer, OntologyEntry, OntologyStore, Test};
fn feature(gloss: &str) -> weksa::InvId {
seed().1.into_iter().find(|e| e.gloss == gloss).unwrap().id
}
fn when(slot: Slot, gloss: &str, test: Test) -> Condition {
Condition {
slot,
feature: feature(gloss),
test,
}
}
fn entry(language: &str, n: u32) -> OntologyEntry {
OntologyEntry {
language: language.into(),
n,
inventory_rev: 1,
layer: Layer::Lexicon,
when: vec![when(
Slot::Stance,
"restraint",
Test::AtLeast(Grade::new(600).unwrap()),
)],
choice: "prefer the humble form".into(),
examples: vec!["example one".into()],
annotator: Annotator::Operator,
}
}
fn store(d: &TempDir) -> OntologyStore {
OntologyStore::open(&d.path().join("o.cc")).unwrap()
}
fn refused(r: Result<AddOutcome, StoreError>) -> StoreRefusal {
refusal(r)
}
#[test]
#[allow(clippy::vec_init_then_push)]
fn ontology_admit_refuses_each_rule() {
let d = dir();
let w = world();
let mut s = store(&d);
let long = |n| "x".repeat(n);
let mut cases: Vec<(&str, Vec<OntologyEntry>)> = Vec::new();
cases.push(("non-tag language", vec![entry("not a tag", 1)]));
cases.push(("n not latest+1", vec![entry("ja-JP", 2)]));
cases.push(("case-variant language", vec![entry("ja-jp", 1)]));
cases.push((
"decomposed choice",
vec![OntologyEntry { choice: "e\u{301}".into(), ..entry("ja-JP", 1) }],
));
cases.push((
"decomposed example",
vec![OntologyEntry { examples: vec!["e\u{301}".into()], ..entry("ja-JP", 1) }],
));
for sep in ["\u{2028}", "\u{2029}", "\u{202e}", "\u{200b}", "\u{3164}", "\u{115f}", "\u{ffa0}"] {
cases.push((
"line separator or format character in a choice",
vec![OntologyEntry { choice: format!("a{sep}b"), ..entry("ja-JP", 1) }],
));
cases.push((
"line separator or format character in an example",
vec![OntologyEntry { examples: vec![format!("a{sep}b")], ..entry("ja-JP", 1) }],
));
}
cases.push((
"unconditioned entry at an unknown revision",
vec![OntologyEntry { when: vec![], inventory_rev: 99, ..entry("ja-JP", 1) }],
));
cases.push((
"Predication slot on a feature that sits there",
vec![OntologyEntry {
when: vec![when(
Slot::Predication,
"modality",
Test::Is(FeatValue::Option(lab("actual"))),
)],
..entry("ja-JP", 1)
}],
));
cases.push((
"65th entry",
(1..=65).map(|n| entry("ja-JP", n)).collect(),
));
for (why, choice) in [("empty choice", String::new()), ("201-byte choice", long(201))] {
cases.push((why, vec![OntologyEntry { choice, ..entry("ja-JP", 1) }]));
}
cases.push((
"line break in an example",
vec![OntologyEntry { examples: vec!["a\nb".into()], ..entry("ja-JP", 1) }],
));
cases.push((
"121-byte example",
vec![OntologyEntry { examples: vec![long(121)], ..entry("ja-JP", 1) }],
));
cases.push((
"four examples",
vec![OntologyEntry { examples: vec!["a".into(); 4], ..entry("ja-JP", 1) }],
));
let restraint = || when(Slot::Stance, "restraint", Test::AtLeast(Grade::new(1).unwrap()));
cases.push((
"five conditions",
vec![OntologyEntry { when: (0..5).map(|_| restraint()).collect(), ..entry("ja-JP", 1) }],
));
for slot in [Slot::Speaker, Slot::Scene, Slot::Predication] {
cases.push((
"slot not Stance or Discourse",
vec![OntologyEntry {
when: vec![when(slot, "restraint", Test::AtLeast(Grade::new(1).unwrap()))],
..entry("ja-JP", 1)
}],
));
}
cases.push((
"AtLeast on an option feature",
vec![OntologyEntry {
when: vec![when(Slot::Discourse, "irony", Test::AtLeast(Grade::new(1).unwrap()))],
..entry("ja-JP", 1)
}],
));
cases.push((
"Is with a grade",
vec![OntologyEntry {
when: vec![when(Slot::Stance, "restraint", Test::Is(FeatValue::Grade(Grade::new(1).unwrap())))],
..entry("ja-JP", 1)
}],
));
cases.push((
"unknown feature",
vec![OntologyEntry {
when: vec![Condition {
slot: Slot::Stance,
feature: "feature:9999".parse().unwrap(),
test: Test::AtLeast(Grade::new(1).unwrap()),
}],
..entry("ja-JP", 1)
}],
));
cases.push((
"unknown inventory revision",
vec![OntologyEntry { inventory_rev: 99, ..entry("ja-JP", 1) }],
));
cases.push((
"model generator that is not an attestation",
vec![OntologyEntry {
annotator: Annotator::Model {
family: ModelFamily::Anthropic,
generator: "--".into(),
},
..entry("ja-JP", 1)
}],
));
for (why, batch) in cases {
assert!(
s.admit(batch, &w.inventory).is_err(),
"{why} must be refused"
);
assert!(s.all().unwrap().is_empty(), "{why} wrote something");
}
// The refusal of a bad choice names the key and no text.
let secret = OntologyEntry { choice: "SECRETCHOICE\n".into(), ..entry("ja-JP", 1) };
let shown = refused(s.admit(vec![secret], &w.inventory)).to_string();
assert!(!shown.contains("SECRETCHOICE"), "{shown}");
}
#[test]
fn ontology_admit_accepts_the_bounds_and_joiners() {
let d = dir();
let w = world();
let mut s = store(&d);
// Exactly 64 entries in one language; exactly the byte bounds; a ZWNJ.
let mut batch: Vec<OntologyEntry> = (1..=64).map(|n| entry("ja-JP", n)).collect();
batch[0].choice = "x".repeat(200);
batch[0].examples = vec!["y".repeat(120); 3];
batch[1].choice = "a\u{200c}b".into();
batch[2].when = (0..4)
.map(|_| when(Slot::Stance, "restraint", Test::AtMost(Grade::new(1000).unwrap())))
.collect();
assert_eq!(s.admit(batch, &w.inventory).unwrap(), AddOutcome::Added);
assert_eq!(s.all().unwrap().len(), 64);
}
#[test]
fn ontology_admit_is_all_or_nothing() {
let d = dir();
let w = world();
let mut s = store(&d);
let bad = OntologyEntry { choice: String::new(), ..entry("ja-JP", 3) };
assert!(s.admit(vec![entry("ja-JP", 1), entry("ja-JP", 2), bad], &w.inventory).is_err());
assert!(s.all().unwrap().is_empty());
let batch = vec![entry("ja-JP", 1), entry("en-US", 1), entry("ja-JP", 2), entry("en-US", 2)];
assert_eq!(s.admit(batch.clone(), &w.inventory).unwrap(), AddOutcome::Added);
assert_eq!(s.all().unwrap().len(), 4);
// An identical replay, from a reopened store, changes nothing.
let mut again = store(&d);
assert_eq!(again.admit(batch, &w.inventory).unwrap(), AddOutcome::Unchanged);
// A batch of one stored entry and one new one is not a replay.
let mixed = vec![entry("ja-JP", 1), entry("ja-JP", 5)];
assert!(again.admit(mixed, &w.inventory).is_err());
assert_eq!(again.all().unwrap().len(), 4);
// Each language numbers from its own latest.
assert_eq!(
again
.admit(vec![entry("ja-JP", 3), entry("en-US", 3)], &w.inventory)
.unwrap(),
AddOutcome::Added
);
// An entry never changes in place.
let changed = OntologyEntry { choice: "something else".into(), ..entry("ja-JP", 1) };
assert!(again.admit(vec![changed], &w.inventory).is_err());
assert_eq!(again.all().unwrap().len(), 6);
}
}

29
docs/targets/interlingua-lowering-map.md
View file

@ -33,9 +33,9 @@ means no store holds it and nothing writes it.
| Schema version | `weksa.interlingua_packet.v<N>` constant in `crates/weksa`; published JSON Schema at `schemas/cultnet/` | Bumped by a commit that changes the Rust types; the published schema is regenerated in the same commit and a test fails on drift | A cut (Hands, under a spec); never the loop |
| Inventory entry | Key `<family>:<n>`, `n` canonical decimal >= 1 (families: concept, frame, role, relation, speech_act, feature, referent_kind, extension); type `weksa.inventory_entry`, with a bounded gloss, value kind (option labels bounded), the slots a feature may sit in (`Discourse`, `Referent`, `Predication`, `Relation`; `Stance` from cut `speaker-state`, `Speaker` and `Scene` from cut `speaker-scene`), `introduced_rev` and optional `retired_rev`. Speaker axes, scene relations and affects are features of those slots, so the loop grows them like any closed-class family and convergence covers them | Added or retired by an inventory revision; never edited in place (a changed meaning is a retire plus an add); a retired id still resolves for packets of earlier revisions | The loop's accepted rounds, whose minting call the context builder assembles from Train items only; Self on merge to main. Admission checks form, not text |
| Inventory revision | Key `rev:<n>`, type `weksa.inventory_revision`, with parent, change list and `frozen` | Appended per accepted round; exactly one revision is ever marked frozen, by the convergence test | The loop's accept rule appends; the convergence test freezes; Self records the freeze as a ruling |
| Target profile | Locale or language-project id (`en-US`, `ja-JP`, a conlang's slug) plus a revision `profile:<language>:<n>` | Authored, revised by commit or by an accepted round that changed the lowering; each lowering names the revision it used | Profile authors and accepted rounds; never the lowerer's defaults (`persona-drives-register`) |
| Language ontology entry (cut `language-ontology`) | Key `onto:<language>:<n>`; type `weksa.ontology_entry`: conditions over feature values (stance, speaker, scene, packet features) and community ids, and the surface choice they select (layer: lexicon, grammar, discourse, phonology, orthography; a bounded descriptor and at most three bounded examples); introduced and retired by a profile revision | Added or retired by a profile revision, never edited; converted from `data/target-language-ontology/*.yaml`, which is deleted in the same cut so the YAML is not a second owner | The language's owner (the operator for her settings' languages) and accepted rounds; an ontology-minting call is assembled from Train items only, like an inventory-minting call, and its text is in the scorer's leak diagnostic. Selection is deterministic code (`select(profile, stance, speaker, scene)`); the lowerer realizes the selected choices and the trace lists their ids (`lowering-traced`) |
| Community (cut `language-ontology`) | Key `community:<slug>`; type `weksa.community`, converted from `data/cultural-ontology/cultures/` and `referents/` (YAML deleted in the same cut) | Revised by commit; a speaker names the communities it belongs to by id | The cultural ontology's author (owner-attested evidence only; Discord-only content stays deleted, `discord-only-content-deleted`) |
| Target profile | Locale or language-project id (`en-US`, `ja-JP`, a conlang's slug). Until entries grow in rounds there is no profile revision document: a run names the profile it lowered with by the ontology store's path and digest in its `RunRecord` (cut `language-ontology`); `profile:<language>:<n>` revision documents arrive with follow-up `language-ontology-growth` | The ontology store is revised by commit; a run pins it and refuses drift | Profile authors and accepted rounds; never the lowerer's defaults (`persona-drives-register`) |
| Language ontology entry (cut `language-ontology`) | Key `onto:<language>:<n>`, `n` from 1 per language; type `weksa.ontology_entry` in `data/ontology/ontology.cc`: the language, the inventory revision its conditions are read at, a layer (lexicon, grammar, discourse, phonology, orthography), at most four conditions (a feature id of slot `Stance` or `Discourse` with `AtLeast`/`AtMost` a grade or `Is` an option or flag; slots `Speaker` and `Scene` and community ids join with follow-up `language-ontology-view-assembly`), a choice of 1 to 200 bytes, at most three examples of 1 to 120 bytes, and an `Annotator` | Admitted in all-or-nothing batches through `OntologyStore::admit`, immutable, never edited; retire arrives with `language-ontology-growth`. The run pins the store like the entity store (path and digest, `Drift(Ontology)`), never copies it, because no loop step writes it yet. Cut `language-ontology-seed` converts by drafting, not by parse: a fresh model drafts each language's seed from that language's YAML and the inventory's feature lines with no corpus text, then deletes `data/target-language-ontology/` and `.weksa/cultural-ontology/` in the same cut | Seed: a labelled model (`Annotator::Model`), no operator gate, never Test evidence (`ja-ontology-owner-answer`); attestation (operator, native reader) appends later. Accepted rounds later, through a mint call from Train items only. Selection is `ontology::select(ontology, language, view)`, called only inside `Prompt::lower`, so no caller picks entries; the prompt prints each selected entry's id, layer, choice and examples. Its text is minted text for the leak diagnostic, which today has no runtime caller (follow-up `leak-diagnostic-in-round`) |
| Community (cut `language-ontology`, follow-up `language-ontology-communities`) | Key `community:<slug>`; type `weksa.community`, converted from `data/cultural-ontology/cultures/` and `referents/` (YAML deleted in the same cut) | Revised by commit; a speaker names the communities it belongs to by id | The cultural ontology's author (owner-attested evidence only; Discord-only content stays deleted, `discord-only-content-deleted`) |
| Speaker (cut `speaker-scene`) | Key `speaker:<slug>@<n>`; type `weksa.speaker`: stable axes as feature values of slot `Speaker` (origin, socioeconomic standing, age cohort, education and whatever the loop adds), at an inventory revision, and an `Annotator` | A new revision appends, never edits; a corpus item and every prior turn name a speaker slug, and a round records the speaker revisions it read | The annotator it names: `PersonaSnapshot` (a projection of a `gamecult.persona_state.v0` document, naming its source document and `updatedAt`), `Operator`, `WorkMetadata` (a public-domain work's own cast list and narration), or `Model { family, generator }` (the 2b `ModelFamily`) |
| Scene (cut `speaker-scene`) | Key `scene:<group>@<n>`, the item's split group; type `weksa.scene`: participants (speaker slugs) and, per ordered pair, relation features of slot `Scene` (relative status, intimacy, in-group, audience and whatever the loop adds), at an inventory revision, and an `Annotator` | As Speaker. Keyed by the group, so a scene never straddles the split | As Speaker |
| Corpus item | Key `item:<source-slug>:<n>`, `n` canonical decimal >= 1 (no sign, no leading zero, from cut 2b); type `weksa.corpus_item` with text, language, speaker, split group, prior turns, source. Source `ModelWritten` carries a `ModelFamily` enum, the generator string as the fixture wrote it, and a prompt digest (cut 2b). The item carries no speaker state of its own: its speaker slug and group name the Speaker and Scene documents | Admitted once; the whole item is immutable (re-adding an identical item is a no-op, any difference is refused; a correction is a new item that `supersedes` the old); text already held under another key is refused unless the new item supersedes that key (cut 2b); withdrawn, never deleted, so old scores stay explainable | Importers and `weksa corpus add`, under `corpus-cast-plus-public-domain`; the source enum has no variant for Discord or any training-forbidden source (`corpus-clean`) |
@ -309,6 +309,27 @@ line two", "その　制御" against the
or socioeconomic standing; `extensions` are non-authoritative by the
schema's own description.
- Probes of 2026-11-12 at `d9ebbcf` (map pass for cut 5): no code reads
`data/target-language-ontology/`, `data/cultural-ontology/` or
`.weksa/cultural-ontology/starter-targets.cc` (a third, typed copy of en,
ja-JP and pt-BR profiles); `README.md`, `docs/multilingual-utterance-lowering.md`,
`examples/cultcache-lowering/` and `prompts/nibu-cultcache-target-*.md` cite
them. `LoweringInput::with_scene` has no caller in `run.rs` or `main.rs`, the
run pins no `SpeakerStateStore`, and no Speaker or Scene document is committed
(`speaker-scene-seeds` open), so only the packet's `Stance` and `Discourse`
features reach a loop's lowering view. `score::leak_diagnostic` and
`TestReadings::at_checkpoint` take minted text as `&[&str]` and have no
caller outside tests: no round reads glosses or any minted text against
Validation. `data/original/inventory.cc` and `data/daodejing/inventory.cc` are
byte-identical; with `data/interlingua/inventory.cc` they share revision 1,
where features 6 to 8 (hedge, certainty grades, irony flag) sit in
`Discourse` and `Predication` and features 9 to 18 (anger to restraint,
grades) in `Stance`. `PROMPT_SOURCE_DIGEST` covers `context.rs`, `split.rs`,
`lower.rs` and `packet.rs`. 25 `RunPaths {` literals across `run.rs`,
`context.rs` doctests, `main.rs`, `tests/common/mod.rs`, `tests/golf_loop.rs`
and `tests/original_corpus.rs`; 10 `Prompt::lower(` calls. `huginn known`
timed out (Unavailable, 15 s deadline) twice this pass.
## Where things live and run (`rust-single-crate`, `loop-runs-on-yggdrasil`)
- Code: one Cargo workspace at the repo root, one package `crates/weksa` (a
@ -403,7 +424,7 @@ One line per cut; specs exist for the cuts about to run (2d and 4, siblings on `
4. `speaker-state`: packet `stance` (felt affect and restraint, slot `Stance`, packet schema v4); formality, warmth and intensity out of the seed and the standard; only `PacketStore::admit` yields a `LoweringInput`, and it refuses packets for withdrawn items (follow-up `lowering-input-from-packet-store`). (spec r1)
4a. `speaker-relation-input`: slot `Scene` and its first seed features (`relative_status`, `intimacy`, `in_group`; inventory revision 3, now `speaker_scene_revision`); `LoweringInput::with_scene` carries the speaker's relation to the addressee as typed `Scene` feature values. Since 4b it takes the `SpeakerStateStore` and the `CorpusStore`, reads the scene of the view's own item group for the item's speaker, and selects the relation by the addressee referent's entity (V16 when the scene does not list the item's speaker or the speaker of any live item of the group, the rule `put_scene` applies too; V17 refuses an addressee that names a predication; V13 is retired). The English lowerer does not render it. (PR for follow-up `chat-s3352-3352`)
4b. `speaker-scene` (spec r1; `speaker-scene-seeds` is its cut 4c: CLI, inventory revision 3, cast speakers and scenes; `Party::Unnamed` covers an addressee with no entity; Speaker and Scene share one `SpeakerStateStore`): slots `Speaker` and `Scene` with their seed features; `weksa.speaker` (`speaker:<entity-slug>@<n>`) and `weksa.scene` (`scene:<group>@<n>`, participants and ordered-pair relations) documents with `Annotator` `Operator` and `Model { family, generator }`, validated by the packet's shared feature check, append-only per key; their stores, `weksa speaker` and `weksa scene` commands, published schemas; Model-annotated cast speakers and scenes for the committed items. Waits on 2d and 4; `pd-speaker-annotation` blocks only the public-domain annotator. Part 1 landed (branch `weksa-lowering/speaker-scene`): `speaker.rs`, `SpeakerStateStore`, slots `Speaker` and `Scene` exclusive to their own feature lists, `speaker_scene_revision()` (Speaker features 22-24), `with_scene` on the store, schemas. The CLI, the committed inventory revision 3 and the seeds stay with 4c.
5. `language-ontology`: `weksa.community` from `data/cultural-ontology/` (owns follow-up `edda-referent-yaml-parse`), `weksa.ontology_entry` for en-US and ja-JP from `data/target-language-ontology/` (YAML deleted), speaker communities, deterministic `select`, the lowering view (packet without trace, speaker, scene, selected entries). Waits on 4.
5. `language-ontology`, in sequential cuts on `d9ebbcf`. 5a `language-ontology` (spec r1): `weksa.ontology_entry`, `OntologyStore`, `ontology::select` over the view's `Stance` and `Discourse` features, `Prompt::lower` printing the selected entries of the item's language, `RunRecord` pinning the ontology store (keys 15 and 16), `weksa ontology admit|dump`; `ontology.rs` joins `PROMPT_SOURCE_DIGEST`. 5b `language-ontology-seed` (spec r1, after 5a): Model-annotated en-US and ja-JP seed entries drafted by an isolated model, `data/target-language-ontology/`, `.weksa/cultural-ontology/` and `schemas/target-language-cultural-ontology.md` deleted. Lost until 5d: keigo, hierarchy and uchi-soto hang on scene relations, so ja-JP politeness is still the lowerer's guess. 5c `language-ontology-view-assembly` (follow-up): the run pins the `SpeakerStateStore`, the round reads the item's Speaker and Scene through the stores, `select` widens to slots `Speaker` and `Scene`; waits on `speaker-scene-seeds`. 5d `language-ontology-communities` (follow-up): `weksa.community` from `data/cultural-ontology/` (YAML deleted, owns `edda-referent-yaml-parse`), community ids on `weksa.speaker`, community conditions; waits on 5c and question `persona-speech-community-home`.
6. `pd-importer`: public-domain play and fiction dialogue into the corpus with speaker, scene group, edition provenance, and Speaker and Scene annotations by the ruled annotator. Waits on 4, follow-up `gutenberg-terms` and question `pd-speaker-annotation`.
7a. `loop-context-builder`: `context.rs` (`TrainItems`, `EncoderOffer`, `Prompt`: corpus text reaches a model call only through `Prompt::mint` over live Train items or `Prompt::encode` over one offered item; instructions are the consts `MINT_INSTRUCTIONS` and `ENCODE_INSTRUCTIONS` in `context.rs`; `Checkpoint` has no public constructor, offers each item once, and its offers borrow it (the gate is typed `TestReadings` and Test offers on the round path; operator tools such as `weksa corpus dump` and `weksa score` still read corpus text of any split); prior turns print the held Train text), `SplitScheme::live`, `CorpusItem::cited_entities` (one owner of the id set `PacketStore::admit` and the offer share), and checkpoint-only Test readings (`Checkpoint`, `TestReadings`; `Aggregate` and `leak_diagnostic` carry no Test). Library only; since 7b `TrainItems::from_corpus` and `EncoderOffer::for_round` are crate-private and `RunStores` is their one public producer. Merges follow-ups `loop-context-builder-2` and `scorer-test-readings-type`. (spec r1)
7b. `golf-loop-round` (spec r1; branch `weksa-lowering/golf-loop-round`): the smallest loop that scores one round. `run.rs`: `RunStores` pins the corpus and entity stores of a run by canonical path and FNV-1a 64 digest at `create`, reopens only from `<root>/run.cc` and refuses drift (corpus, entities, `PROMPT_SOURCE_DIGEST` of `context.rs`, a second pin on `lower.rs` retired in 7d, model, embedder); `round` encodes every live Train and Validation item through a `ModelPort` (`model.rs`; the one adapter `AnswerFiles` writes `<digest>.prompt` and reads `<digest>.answer`, so the model is the Claude session), reads each answer into a `PacketDraft` (`encode.rs`, the one place a model's JSON is read), admits through `PacketStore::admit` into `<root>/round-<n>/packets.cc`, lowers, scores once, and keeps a `RoundRecord` (`weksa.loop_round`, no text, no Test; Train rows, Validation only as means and refusal counts). `weksa loop open|round`. Not here: the spec-edit step, accept rule and composite, noise floor, Score and lowering documents, a subprocess adapter, the checkpoint door, a model lowerer (follow-ups `golf-loop-spec-edit`, `golf-loop-accept-rule`, `golf-loop-score-documents`, `golf-loop-subprocess-adapter`, `golf-loop-checkpoint-door-bound`, `golf-loop-model-lowerer`). `packet-store-inventory-binding` is met for loop packets only (`RunStores` admits with the run's inventory); `validate`, `packet proof|dao` and `lower` still take caller stores. Waits on 7a, `checkpoint-gate`, 4b.

12
schemas/cultnet/weksa.loop_run.schema.json
View file

@ -49,6 +49,14 @@
"description": "The answering model.",
"type": "string"
},
"ontology": {
"description": "Canonical absolute path of the ontology store: read-only for the run,\npinned by `ontology_digest`.",
"type": "string"
},
"ontology_digest": {
"description": "16 hex of the ontology file's bytes.",
"type": "string"
},
"panel": {
"description": "The languages the run offers, reads and decides on: BCP 47-form tags,\nsorted, unique, at least one, set once at open. An item of any other\nlanguage is never offered, minted from, probed or scored.",
"type": "array",
@ -73,7 +81,9 @@
"chrf_version",
"inventory_digest",
"lowerer_reach",
"panel"
"panel",
"ontology",
"ontology_digest"
],
"$defs": {
"LowererReach": {

264
schemas/cultnet/weksa.ontology_entry.schema.json Normal file
View file

@ -0,0 +1,264 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "OntologyEntry",
"description": "One choice of one language. Key `onto:<language>:<n>`.\n\nThe store judges the form of an entry's text (bounds, normalization, no\ncontrol or format character) and never its meaning: no door judges text\n(ruling `leak-owned-by-flow-and-scorer-3`). No Train-only ontology mint\nexists yet, so a seed or a drafted entry is checked against held-out text by\nnothing today; follow-up `leak-diagnostic-in-round` owns that diagnostic.",
"type": "object",
"properties": {
"annotator": {
"$ref": "#/$defs/Annotator"
},
"choice": {
"type": "string"
},
"examples": {
"type": "array",
"items": {
"type": "string"
}
},
"inventory_rev": {
"description": "The inventory revision the conditions read against.",
"type": "integer",
"format": "uint32",
"minimum": 0
},
"language": {
"type": "string"
},
"layer": {
"$ref": "#/$defs/Layer"
},
"n": {
"description": "The entry's number within its language, from 1.",
"type": "integer",
"format": "uint32",
"minimum": 0
},
"when": {
"description": "Every condition must hold; none means the entry always applies.",
"type": "array",
"items": {
"$ref": "#/$defs/Condition"
}
}
},
"required": [
"language",
"n",
"inventory_rev",
"layer",
"when",
"choice",
"examples",
"annotator"
],
"$defs": {
"Annotator": {
"description": "Who annotated a document. A model's generator is an attestation string, held\nto the same form as every provenance string.",
"oneOf": [
{
"type": "string",
"enum": [
"Operator"
]
},
{
"type": "object",
"properties": {
"Model": {
"type": "object",
"properties": {
"family": {
"$ref": "#/$defs/ModelFamily"
},
"generator": {
"type": "string"
}
},
"required": [
"family",
"generator"
]
}
},
"additionalProperties": false,
"required": [
"Model"
]
}
]
},
"Condition": {
"description": "One feature of one slot of the view, tested. Only `Stance` and `Discourse`\nreach a lowering view today.",
"type": "object",
"properties": {
"feature": {
"$ref": "#/$defs/InvId"
},
"slot": {
"$ref": "#/$defs/Slot"
},
"test": {
"$ref": "#/$defs/Test"
}
},
"required": [
"slot",
"feature",
"test"
]
},
"FeatValue": {
"oneOf": [
{
"type": "object",
"properties": {
"Grade": {
"$ref": "#/$defs/Grade"
}
},
"additionalProperties": false,
"required": [
"Grade"
]
},
{
"description": "A label the feature's inventory entry declares.",
"type": "object",
"properties": {
"Option": {
"$ref": "#/$defs/OptionLabel"
}
},
"additionalProperties": false,
"required": [
"Option"
]
},
{
"type": "object",
"properties": {
"Flag": {
"type": "boolean"
}
},
"additionalProperties": false,
"required": [
"Flag"
]
}
]
},
"Grade": {
"type": "integer",
"maximum": 1000,
"minimum": 0
},
"InvId": {
"type": "string",
"pattern": "^(concept|frame|role|relation|speech_act|feature|referent_kind|extension):[1-9][0-9]*$"
},
"Layer": {
"description": "Which part of the language an entry shapes.",
"type": "string",
"enum": [
"Lexicon",
"Grammar",
"Discourse",
"Phonology",
"Orthography"
]
},
"ModelFamily": {
"description": "The family of the model that wrote a line.",
"oneOf": [
{
"type": "string",
"enum": [
"Anthropic",
"OpenAi",
"Google",
"Alibaba",
"Meta",
"Mistral"
]
},
{
"description": "Nothing records which family wrote the line.",
"type": "string",
"const": "Unrecorded"
}
]
},
"OptionLabel": {
"type": "string",
"pattern": "^[a-z][a-z0-9_]{0,23}$"
},
"Slot": {
"description": "Where a feature may sit in a packet.",
"oneOf": [
{
"type": "string",
"enum": [
"Discourse",
"Referent",
"Predication",
"Relation",
"Stance"
]
},
{
"description": "The speaker's relation to one party of the scene, held by `weksa.scene`\ndocuments; see `LoweringInput::with_scene`.",
"type": "string",
"const": "Scene"
},
{
"description": "The speaker's stable axes, held by `weksa.speaker` documents.",
"type": "string",
"const": "Speaker"
}
]
},
"Test": {
"description": "What a feature's value must be for a condition to hold.",
"oneOf": [
{
"type": "object",
"properties": {
"AtLeast": {
"$ref": "#/$defs/Grade"
}
},
"additionalProperties": false,
"required": [
"AtLeast"
]
},
{
"type": "object",
"properties": {
"AtMost": {
"$ref": "#/$defs/Grade"
}
},
"additionalProperties": false,
"required": [
"AtMost"
]
},
{
"type": "object",
"properties": {
"Is": {
"$ref": "#/$defs/FeatValue"
}
},
"additionalProperties": false,
"required": [
"Is"
]
}
]
}
}
}