Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1e06f00336 | |||
| e7775541fe | |||
| 16b9ef32ac | |||
| 69e1b1f327 | |||
| 1858023248 | |||
| 611460bec3 | |||
| 30afdae55b | |||
| 212bdbaf61 | |||
| d9187aeec1 | |||
| b7501a758d | |||
| 2bb9870338 |
43 changed files with 1604 additions and 1643 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -22,4 +22,3 @@ build/
|
|||
.venv/
|
||||
venv/
|
||||
target/
|
||||
tools/show-working/out/
|
||||
|
|
|
|||
|
|
@ -60,4 +60,4 @@ language-project truth.
|
|||
`CARGO_HOME=/var/lib/mind-weksa-self/cargo-home cargo test --workspace`.
|
||||
- `weksa packet proof <store>` admits the nibu proof packet under the key
|
||||
`packet:6e6962752d303031` (hex of `nibu-001`), not `nibu-proof-001`. Use that key
|
||||
with `weksa lower <store> <key>`.
|
||||
with `weksa pronounce <store> <key>`.
|
||||
|
|
|
|||
|
|
@ -35,8 +35,8 @@ measured.
|
|||
- `settings/`: setting-specific proving grounds for language projects.
|
||||
|
||||
**Live versus planned.** Live: docs, fixtures, examples, ontology data and the
|
||||
packet crate `crates/weksa`, including `weksa lower` (the template lowerer for
|
||||
proof fixtures) and `weksa score`, and the golf loop's round and edit-accept
|
||||
packet crate `crates/weksa`, including `weksa pronounce` (the ruled IPA and phones
|
||||
of the entities a packet cites), `weksa score`, and the golf loop's round and edit-accept
|
||||
cycle (`weksa loop open|round|cycle`), which encodes and lowers through the
|
||||
model (a prompted Claude subagent, per staged-distill).
|
||||
Planned: Score and lowering documents and the other panel languages. No
|
||||
|
|
|
|||
|
|
@ -1,398 +0,0 @@
|
|||
//! Build-time generator for the "Show her working" page. Runs the real
|
||||
//! admission door, lowerer and IPA reader over the committed proof packet and
|
||||
//! variants of it, and prints one JSON document on stdout: the typed slots, the
|
||||
//! inventory entries the packet chose, and the outcome, which is lowered text or
|
||||
//! a named refusal. Scoring is not here: `tools/show-working/build.py` calls the
|
||||
//! `weksa score` CLI on the text this emits.
|
||||
//!
|
||||
//! Run from the repository root: `cargo run --release --example show_working`.
|
||||
//! Every variant is a mutation of `proof_packet()`, spelled in its `source`.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use serde_json::{Value, json};
|
||||
use weksa::packet::{
|
||||
EntityId, FeatValue, FeatureValue, Grade, NodeId, ProvenanceSource, RoleFill, proof_packet,
|
||||
};
|
||||
use weksa::{
|
||||
CorpusStore, EntityStore, InvId, Inventory, InventoryStore, LowerError, Packet, PacketStore,
|
||||
StoreError, lower, pronounce,
|
||||
};
|
||||
|
||||
struct Fixture {
|
||||
id: &'static str,
|
||||
title: &'static str,
|
||||
note: &'static str,
|
||||
source: &'static str,
|
||||
packet: Packet,
|
||||
}
|
||||
|
||||
fn id(s: &str) -> InvId {
|
||||
s.parse().expect("inventory id")
|
||||
}
|
||||
|
||||
fn node(s: &str) -> NodeId {
|
||||
NodeId::parse(s).expect("node id")
|
||||
}
|
||||
|
||||
fn grade(feature: &str, v: u16) -> FeatureValue {
|
||||
FeatureValue {
|
||||
feature: id(feature),
|
||||
value: FeatValue::Grade(Grade::new(v).expect("grade")),
|
||||
}
|
||||
}
|
||||
|
||||
fn fixtures() -> Vec<Fixture> {
|
||||
let proof = proof_packet();
|
||||
let mut out = vec![Fixture {
|
||||
id: "proof",
|
||||
title: "Nibu warns the player off the control",
|
||||
note: "The committed proof packet, unchanged.",
|
||||
source: "proof_packet() in crates/weksa/src/packet.rs",
|
||||
packet: proof.clone(),
|
||||
}];
|
||||
let mut add = |id, title, note, source, f: &dyn Fn(&mut Packet)| {
|
||||
let mut packet = proof.clone();
|
||||
f(&mut packet);
|
||||
out.push(Fixture { id, title, note, source, packet });
|
||||
};
|
||||
add(
|
||||
"no-addressee",
|
||||
"Same meaning, nobody addressed",
|
||||
"With the addressee unset the player is a third party: no \"you\", no reflexive.",
|
||||
"proof_packet() with discourse.addressee = None",
|
||||
&|p| p.discourse.addressee = None,
|
||||
);
|
||||
add(
|
||||
"stance-held-back",
|
||||
"Anger held back, restraint high",
|
||||
"Stance is the speaker's felt affect. The lowerer has no row that reads it, so the text is the proof's text.",
|
||||
"proof_packet() with stance anger=300, restraint=700",
|
||||
&|p| p.stance.features = vec![grade("feature:9", 300), grade("feature:18", 700)],
|
||||
);
|
||||
add(
|
||||
"hedged",
|
||||
"Hedged and uncertain warning",
|
||||
"Hedge and certainty are meaning the lowerer cannot yet say. It refuses rather than drop them.",
|
||||
"proof_packet() with discourse hedge=600, certainty=400",
|
||||
&|p| p.discourse.features = vec![grade("feature:6", 600), grade("feature:7", 400)],
|
||||
);
|
||||
add(
|
||||
"negated",
|
||||
"Negated intervention (polarity: negative)",
|
||||
"Dropping the negation would say the opposite, so the lowerer refuses. This is the shape of a negated Dao line awaiting polarity.",
|
||||
"proof_packet() with predication n5 polarity=negative",
|
||||
&|p| {
|
||||
p.predications[1].features = vec![FeatureValue {
|
||||
feature: id("feature:1"),
|
||||
value: FeatValue::Option("negative".parse().expect("label")),
|
||||
}]
|
||||
},
|
||||
);
|
||||
add(
|
||||
"consent-authored",
|
||||
"Addressee consent, authored source",
|
||||
"Someone who saw the request says the addressee asked for it. No lowering speaks for that request yet.",
|
||||
"proof_packet() with discourse.addressee_consent = n4 (source Authored)",
|
||||
&|p| p.discourse.addressee_consent = Some(node("n4")),
|
||||
);
|
||||
add(
|
||||
"consent-imported",
|
||||
"Addressee consent, imported-corpus source",
|
||||
"A reader of wording cannot have seen a request, so the packet is refused at admission (V15), before any lowering.",
|
||||
"proof_packet() with addressee_consent = n4 and provenance.source = ImportedCorpus",
|
||||
&|p| {
|
||||
p.discourse.addressee_consent = Some(node("n4"));
|
||||
p.provenance.source = ProvenanceSource::ImportedCorpus;
|
||||
},
|
||||
);
|
||||
add(
|
||||
"relation",
|
||||
"A causal relation between the predications",
|
||||
"Relations are not rendered; the lowerer refuses instead of asserting a different meaning.",
|
||||
"proof_packet() with relation:8 (causes) n4 -> n5",
|
||||
&|p| {
|
||||
p.relations = vec![weksa::packet::Relation {
|
||||
relation: id("relation:8"),
|
||||
from: node("n4"),
|
||||
to: node("n5"),
|
||||
features: vec![],
|
||||
}]
|
||||
},
|
||||
);
|
||||
add(
|
||||
"dangling-filler",
|
||||
"A role filled by a node that does not exist",
|
||||
"Refused at admission (V4): the packet is not a packet.",
|
||||
"proof_packet() with predication n5 agent filler n9",
|
||||
&|p| {
|
||||
p.predications[1].roles[0] = RoleFill { role: id("role:1"), filler: node("n9") }
|
||||
},
|
||||
);
|
||||
add(
|
||||
"unknown-entity",
|
||||
"A referent naming an entity nobody holds",
|
||||
"The entity store is the only owner of names; a packet cannot invent one.",
|
||||
"proof_packet() with referent n3 entity = entity:nobody",
|
||||
&|p| p.referents[2].entity = Some(EntityId::parse("entity:nobody").expect("entity")),
|
||||
);
|
||||
add(
|
||||
"common-noun-agent",
|
||||
"The ship mind as a common noun",
|
||||
"No entity: the referent lowers as \"the ship mind\", and there is no name to pronounce.",
|
||||
"proof_packet() with referent n3 entity = None",
|
||||
&|p| p.referents[2].entity = None,
|
||||
);
|
||||
out
|
||||
}
|
||||
|
||||
fn gloss(inv: &Inventory, rev: u32, i: &InvId) -> Value {
|
||||
match inv.resolve(i, rev) {
|
||||
Some(e) => json!({ "id": i.to_string(), "gloss": e.gloss }),
|
||||
None => json!({ "id": i.to_string(), "gloss": null }),
|
||||
}
|
||||
}
|
||||
|
||||
fn feature_json(inv: &Inventory, rev: u32, slot: &str, at: &str, f: &FeatureValue) -> Value {
|
||||
let mut g = gloss(inv, rev, &f.feature);
|
||||
g["slot"] = json!(slot);
|
||||
g["at"] = json!(at);
|
||||
g["value"] = serde_json::to_value(&f.value).expect("value");
|
||||
g
|
||||
}
|
||||
|
||||
/// Every feature the packet carries, with where it sits.
|
||||
fn features(p: &Packet, inv: &Inventory) -> Vec<Value> {
|
||||
let rev = p.inventory_rev;
|
||||
let mut v = Vec::new();
|
||||
for f in &p.discourse.features {
|
||||
v.push(feature_json(inv, rev, "discourse", "discourse", f));
|
||||
}
|
||||
for f in &p.stance.features {
|
||||
v.push(feature_json(inv, rev, "stance", "stance", f));
|
||||
}
|
||||
for r in &p.referents {
|
||||
for f in &r.features {
|
||||
v.push(feature_json(inv, rev, "referent", r.id.as_str(), f));
|
||||
}
|
||||
}
|
||||
for pr in &p.predications {
|
||||
for f in &pr.features {
|
||||
v.push(feature_json(inv, rev, "predication", pr.id.as_str(), f));
|
||||
}
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
/// Ids the packet chose, once each, with the inventory's gloss at its revision.
|
||||
fn chosen(p: &Packet, inv: &Inventory) -> Vec<Value> {
|
||||
let rev = p.inventory_rev;
|
||||
let mut ids: Vec<InvId> = Vec::new();
|
||||
let mut push = |i: &InvId| {
|
||||
if !ids.contains(i) {
|
||||
ids.push(*i)
|
||||
}
|
||||
};
|
||||
p.context.setting.iter().for_each(&mut push);
|
||||
p.context.medium.iter().for_each(&mut push);
|
||||
p.discourse.speech_act.iter().for_each(&mut push);
|
||||
for r in &p.referents {
|
||||
push(&r.kind);
|
||||
push(&r.concept);
|
||||
}
|
||||
for pr in &p.predications {
|
||||
push(&pr.concept);
|
||||
push(&pr.frame);
|
||||
pr.roles.iter().for_each(|f| push(&f.role));
|
||||
}
|
||||
p.relations.iter().for_each(|r| push(&r.relation));
|
||||
features(p, inv)
|
||||
.iter()
|
||||
.filter_map(|f| f["id"].as_str().and_then(|s| s.parse().ok()))
|
||||
.for_each(|i: InvId| push(&i));
|
||||
ids.iter().map(|i| gloss(inv, rev, i)).collect()
|
||||
}
|
||||
|
||||
fn slots(p: &Packet) -> Value {
|
||||
let j = |v: &dyn erased::Ser| v.to_value();
|
||||
json!({
|
||||
"meaning": {
|
||||
"referents": p.referents.iter().map(|r| j(r)).collect::<Vec<_>>(),
|
||||
"predications": p.predications.iter().map(|r| j(r)).collect::<Vec<_>>(),
|
||||
"relations": p.relations.iter().map(|r| j(r)).collect::<Vec<_>>(),
|
||||
},
|
||||
"stance": j(&p.stance),
|
||||
"discourse": j(&p.discourse),
|
||||
"context": j(&p.context),
|
||||
"extensions": p.extensions.iter().map(|r| j(r)).collect::<Vec<_>>(),
|
||||
"constraints": j(&p.constraints),
|
||||
"provenance": { "source": j(&p.provenance.source), "corpus_item": p.provenance.corpus_item.to_string(), "confidence": p.provenance.confidence.get() },
|
||||
"kind": j(&p.kind),
|
||||
"inventory_rev": p.inventory_rev,
|
||||
})
|
||||
}
|
||||
|
||||
mod erased {
|
||||
pub trait Ser {
|
||||
fn to_value(&self) -> serde_json::Value;
|
||||
}
|
||||
impl<T: serde::Serialize> Ser for T {
|
||||
fn to_value(&self) -> serde_json::Value {
|
||||
serde_json::to_value(self).expect("serialize")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn lower_error(e: &LowerError) -> Value {
|
||||
let (variant, reason) = match e {
|
||||
LowerError::Unrendered { what } => ("Unrendered", format!("the packet carries {what}, which the lowerer does not render")),
|
||||
LowerError::NoRow { id } => ("NoRow", format!("no table row for {id}")),
|
||||
LowerError::GlossMismatch { id, row } => ("GlossMismatch", format!("{id} is not the {row} the row speaks for")),
|
||||
LowerError::UnknownRole { frame, role } => ("UnknownRole", format!("{frame} names role {role}, which the inventory lacks")),
|
||||
LowerError::MissingRole { predication, role } => ("MissingRole", format!("{predication} fills no {role} role")),
|
||||
LowerError::UnknownEntity { entity } => ("UnknownEntity", format!("the entity store does not hold {entity}")),
|
||||
LowerError::NoForm { entity } => ("NoForm", format!("{entity} has no en-US form")),
|
||||
LowerError::NoPronunciation { entity } => ("NoPronunciation", format!("{entity} has no ruled en-US pronunciation")),
|
||||
LowerError::NoEntityReferent => ("NoEntityReferent", "the packet cites no entity".to_string()),
|
||||
LowerError::UnknownPhone { entity, at, .. } => ("UnknownPhone", format!("the ruled pronunciation of {entity} holds a symbol at {at} with no listed phone")),
|
||||
LowerError::DanglingStress { entity, at } => ("DanglingStress", format!("the ruled pronunciation of {entity} has a stress mark at {at} with no vowel after it")),
|
||||
LowerError::ConsentMissing { predication } => ("ConsentMissing", format!("{predication} is a request the packet does not carry the addressee's consent to")),
|
||||
LowerError::AskerNotAddressee { predication } => ("AskerNotAddressee", format!("the agent of {predication} is not the addressee")),
|
||||
LowerError::EntityRead { entity, .. } => ("EntityRead", format!("the entity store failed reading {entity}")),
|
||||
LowerError::SceneUnread => ("SceneUnread", "the view's scene was never read".to_string()),
|
||||
LowerError::SpeakerTokenTaken => ("SpeakerTokenTaken", "a referent cites the entity id the speaker token uses".to_string()),
|
||||
};
|
||||
json!({ "variant": variant, "reason": reason })
|
||||
}
|
||||
|
||||
struct World {
|
||||
inventory: InventoryStore,
|
||||
corpus: CorpusStore,
|
||||
entities: EntityStore,
|
||||
inv: Inventory,
|
||||
}
|
||||
|
||||
/// Admit `packet` through a fresh packet store and lower it. `Ok` carries the
|
||||
/// text and the IPA outcome; `Err` is the refusal, as JSON.
|
||||
fn run(w: &World, dir: &Path, packet: &Packet) -> Result<(String, Value), Value> {
|
||||
let mut store = PacketStore::open(&dir.join("p.cc")).expect("packet store");
|
||||
let view = match store.admit(packet.clone(), &w.inventory, &w.corpus, &w.entities) {
|
||||
Ok(v) => v,
|
||||
Err(StoreError::Invalid(rs)) => {
|
||||
return Err(json!({
|
||||
"kind": "admission_refused",
|
||||
"refusals": rs.iter().map(|r| json!({ "rule": r.rule(), "detail": r.to_string() })).collect::<Vec<_>>(),
|
||||
}));
|
||||
}
|
||||
Err(e) => {
|
||||
return Err(json!({ "kind": "admission_refused", "refusals": [{ "rule": "store", "detail": e.to_string() }] }));
|
||||
}
|
||||
};
|
||||
let text = lower(&view, &w.inv, &w.entities).map_err(|e| {
|
||||
json!({ "kind": "lower_error", "error": lower_error(&e) })
|
||||
})?;
|
||||
let ipa = match pronounce(&view, &w.entities) {
|
||||
Ok(v) => json!({ "ok": v }),
|
||||
Err(e) => json!({ "error": lower_error(&e) }),
|
||||
};
|
||||
Ok((text, ipa))
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
|
||||
let inventory = InventoryStore::open(&root.join("data/interlingua/inventory.cc")).expect("inventory");
|
||||
let inv = inventory.inventory().expect("inventory view");
|
||||
let w = World {
|
||||
corpus: CorpusStore::open(&root.join("data/corpus/corpus.cc")).expect("corpus"),
|
||||
entities: EntityStore::open(&root.join("data/entity/entity.cc")).expect("entities"),
|
||||
inventory,
|
||||
inv,
|
||||
};
|
||||
let dir = std::env::temp_dir().join(format!("show-working-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).expect("scratch dir");
|
||||
let mut out = Vec::new();
|
||||
for fx in fixtures() {
|
||||
let p = &fx.packet;
|
||||
let first = run(&w, &dir, p);
|
||||
let outcome = match first {
|
||||
Err(e) if e["kind"] == "admission_refused" => e,
|
||||
first => {
|
||||
// Ablation: run again with each feature removed, against the
|
||||
// baseline outcome. It is how the page says whether the lowerer
|
||||
// read a feature, instead of the page asserting it.
|
||||
let base = first.as_ref().ok().map(|(t, _)| t.clone());
|
||||
let n_disc = p.discourse.features.len();
|
||||
let n_stance = p.stance.features.len();
|
||||
let total = features(p, &w.inv).len();
|
||||
let mut effects = Vec::new();
|
||||
for k in 0..total {
|
||||
let mut q = p.clone();
|
||||
if k < n_disc {
|
||||
q.discourse.features.remove(k);
|
||||
} else if k < n_disc + n_stance {
|
||||
q.stance.features.remove(k - n_disc);
|
||||
} else {
|
||||
let mut at = k - n_disc - n_stance;
|
||||
'o: {
|
||||
for r in &mut q.referents {
|
||||
if at < r.features.len() { r.features.remove(at); break 'o; }
|
||||
at -= r.features.len();
|
||||
}
|
||||
for pr in &mut q.predications {
|
||||
if at < pr.features.len() { pr.features.remove(at); break 'o; }
|
||||
at -= pr.features.len();
|
||||
}
|
||||
}
|
||||
}
|
||||
let _ = std::fs::remove_file(dir.join("p.cc"));
|
||||
effects.push(match (&base, run(&w, &dir, &q)) {
|
||||
(Some(t), Ok((u, _))) if *t == u => "Without it the text is identical: the lowerer did not read it.",
|
||||
(Some(_), Ok(_)) => "Without it the text changes: the lowerer read it.",
|
||||
(Some(_), Err(_)) => "Without it the packet is refused.",
|
||||
(None, Ok(_)) => "Without it the packet lowers: this feature is what the lowerer refuses.",
|
||||
(None, Err(_)) => "Still refused without this feature alone.",
|
||||
});
|
||||
}
|
||||
match first {
|
||||
Ok((text, ipa)) => json!({ "kind": "lowered", "text": text, "ipa": ipa, "feature_effects": effects }),
|
||||
Err(e) => { let mut e = e; e["feature_effects"] = json!(effects); e }
|
||||
}
|
||||
}
|
||||
};
|
||||
let _ = std::fs::remove_file(dir.join("p.cc"));
|
||||
let mut slots = slots(p);
|
||||
slots["features"] = json!(features(p, &w.inv));
|
||||
out.push(json!({
|
||||
"id": fx.id,
|
||||
"title": fx.title,
|
||||
"note": fx.note,
|
||||
"source": fx.source,
|
||||
"packet_id": p.id.to_string(),
|
||||
"corpus_item": p.provenance.corpus_item.to_string(),
|
||||
"slots": slots,
|
||||
"chosen": chosen(p, &w.inv),
|
||||
"outcome": outcome,
|
||||
}));
|
||||
}
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
println!("{}", serde_json::to_string_pretty(&json!({ "fixtures": out })).expect("json"));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn refusal_carries_the_closed_reason_never_a_debug_of_the_error() {
|
||||
// UnknownPhone holds the ruled IPA text; no field of the output may.
|
||||
let e = LowerError::UnknownPhone {
|
||||
entity: EntityId::parse("entity:nibu").expect("entity id"),
|
||||
at: 3,
|
||||
text: "CANARY-IPA".into(),
|
||||
};
|
||||
let v = lower_error(&e);
|
||||
assert!(!v.to_string().contains("CANARY"));
|
||||
let keys: Vec<_> = v.as_object().expect("object").keys().cloned().collect();
|
||||
assert_eq!(keys, ["reason", "variant"]);
|
||||
}
|
||||
}
|
||||
|
|
@ -759,6 +759,9 @@ impl Prompt {
|
|||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod mutants;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
|
|||
14
crates/weksa/src/context/mutants.rs
Normal file
14
crates/weksa/src/context/mutants.rs
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
//! Pins for private reach of `context.rs` that no run can produce.
|
||||
|
||||
use super::*;
|
||||
|
||||
/// A corpus with no live Train item of the panel's languages holds an empty
|
||||
/// train. (A run refuses a panel without Train items at open, so only the
|
||||
/// builder shows it.)
|
||||
#[test]
|
||||
fn no_train_item_of_the_panel_is_an_empty_train() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let corpus = CorpusStore::open(&dir.path().join("c.cc")).unwrap();
|
||||
let train = TrainItems::from_corpus(&corpus, &["en-US".to_string()]).unwrap();
|
||||
assert_eq!((train.len(), train.is_empty()), (0, true));
|
||||
}
|
||||
|
|
@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize};
|
|||
|
||||
use crate::form::{FormError, is_slug, string_form};
|
||||
use crate::packet::EntityId;
|
||||
use crate::speaker::Annotator;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Turn {
|
||||
|
|
@ -103,6 +104,11 @@ pub struct CorpusItem {
|
|||
/// Which entities the text names, and where. Fixed with the text at intake.
|
||||
#[cultcache(key = 10)]
|
||||
pub names: Vec<NameSpan>,
|
||||
/// Who placed `names`: the annotator of the intake that wrote them. `None`
|
||||
/// when nothing records one (the cast seeds, `weksa corpus add`, the Dao De
|
||||
/// Jing bed); it labels the spans, never the line or its speaker.
|
||||
#[cultcache(key = 11, default)]
|
||||
pub names_by: Option<Annotator>,
|
||||
}
|
||||
|
||||
impl CorpusItem {
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@
|
|||
//! provenance, constraints and extensions, and an empty trace, so a model that
|
||||
//! saw a Validation item cannot hand its words to whoever reads packets. The
|
||||
//! editor writes family, gloss, value kind and slots only; `into_revision` sets
|
||||
//! every id and revision number.
|
||||
//! every id and revision number. The public-domain intake file (`WorkDraft`) and
|
||||
//! its name-span answer (`MarkedLine`) are read here too.
|
||||
|
||||
use std::fmt;
|
||||
use std::num::NonZeroU32;
|
||||
|
|
@ -15,7 +16,12 @@ use std::num::NonZeroU32;
|
|||
use serde::de::DeserializeOwned;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::corpus::CorpusItemId;
|
||||
use unicode_normalization::UnicodeNormalization;
|
||||
|
||||
use crate::corpus::{CorpusItem, CorpusItemId, NameSpan, Source, item_key};
|
||||
use crate::entity::{Entity, EntityOrigin, LanguageForms};
|
||||
use crate::form::is_slug;
|
||||
use crate::packet::EntityId;
|
||||
use crate::inventory::{
|
||||
Change, Family, InvId, Inventory, InventoryEntry, InventoryRevision, Slot, ValueKind,
|
||||
};
|
||||
|
|
@ -358,6 +364,196 @@ impl LoweringDraft {
|
|||
}
|
||||
}
|
||||
|
||||
/// One cast member of a work: the entity and the forms the work spells it with,
|
||||
/// in the work's language, citation form first.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct CastDraft {
|
||||
pub entity: EntityId,
|
||||
pub forms: Vec<String>,
|
||||
}
|
||||
|
||||
/// One line of a work: its scene group, the speaker its speech prefix names, text.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct LineDraft {
|
||||
pub group: String,
|
||||
pub speaker: EntityId,
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
/// A transcription of a public-domain work in hand: provenance, cast forms,
|
||||
/// lines, scene groups and the speaker each speech prefix names. It holds no
|
||||
/// distributor header, licence or trademark text, and nothing reads a URL.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct WorkDraft {
|
||||
pub slug: String,
|
||||
pub language: String,
|
||||
pub admitted_on: String,
|
||||
pub source: Source,
|
||||
pub cast: Vec<CastDraft>,
|
||||
pub lines: Vec<LineDraft>,
|
||||
}
|
||||
|
||||
/// One line of a name-span answer: the line number and the line copied with
|
||||
/// `[[entity:<slug>|<form>]]` around each place it names a cast member.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MarkedLine {
|
||||
pub line: u32,
|
||||
pub marked: String,
|
||||
}
|
||||
|
||||
impl MarkedLine {
|
||||
pub fn parse(answer: &str) -> Result<Self, DraftError> {
|
||||
from_json(answer)
|
||||
}
|
||||
}
|
||||
|
||||
/// Why an import was refused before any document was written. The text names
|
||||
/// the kind and the line number only, never a value.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum ImportRefusal {
|
||||
NotPublicDomain,
|
||||
CastRepeated,
|
||||
NoSuchLine { line: u32 },
|
||||
LineMarkedTwice { line: u32 },
|
||||
Group { line: u32 },
|
||||
SpeakerNotInCast { line: u32 },
|
||||
Markup { line: u32 },
|
||||
NotInCast { line: u32 },
|
||||
}
|
||||
|
||||
impl fmt::Display for ImportRefusal {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::NotPublicDomain => write!(f, "the work's source is not public domain"),
|
||||
Self::CastRepeated => write!(f, "a cast entity is listed twice"),
|
||||
Self::NoSuchLine { line } => write!(f, "the answer marks a line that is not there (line {line})"),
|
||||
Self::LineMarkedTwice { line } => write!(f, "line {line} is marked twice"),
|
||||
Self::Group { line } => write!(f, "the group of line {line} is not a slug"),
|
||||
Self::SpeakerNotInCast { line } => write!(f, "the speaker of line {line} is not in the cast"),
|
||||
Self::Markup { line } => write!(f, "the marked text of line {line} is not its text with markup"),
|
||||
Self::NotInCast { line } => write!(f, "line {line} marks an entity outside the cast"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for ImportRefusal {}
|
||||
|
||||
impl WorkDraft {
|
||||
pub fn parse(file: &str) -> Result<Self, DraftError> {
|
||||
from_json(file)
|
||||
}
|
||||
|
||||
/// The cast entities and the corpus items for the lines `marked` covers, in
|
||||
/// line order, each item's spans read from its markup and labelled
|
||||
/// `annotator`. Decides everything before any write; whether the documents
|
||||
/// are admissible is `EntityStore::put` and `CorpusStore::add`'s to refuse.
|
||||
pub fn into_import(
|
||||
self,
|
||||
marked: &[MarkedLine],
|
||||
annotator: &Annotator,
|
||||
) -> Result<(Vec<Entity>, Vec<CorpusItem>), ImportRefusal> {
|
||||
let Source::PublicDomain { work, .. } = &self.source else {
|
||||
return Err(ImportRefusal::NotPublicDomain);
|
||||
};
|
||||
let mut entities = Vec::new();
|
||||
for c in &self.cast {
|
||||
if entities.iter().any(|e: &Entity| e.id == c.entity) {
|
||||
return Err(ImportRefusal::CastRepeated);
|
||||
}
|
||||
entities.push(Entity {
|
||||
id: c.entity.clone(),
|
||||
forms: vec![LanguageForms {
|
||||
language: self.language.clone(),
|
||||
forms: c.forms.iter().map(|f| f.nfc().collect()).collect(),
|
||||
pronunciation: None,
|
||||
}],
|
||||
origin: EntityOrigin::PublicDomain { work: work.clone() },
|
||||
});
|
||||
}
|
||||
let in_cast = |id: &EntityId| entities.iter().any(|e| &e.id == id);
|
||||
let mut answer: Vec<&MarkedLine> = marked.iter().collect();
|
||||
answer.sort_by_key(|m| m.line);
|
||||
let mut items = Vec::new();
|
||||
for (i, m) in answer.iter().enumerate() {
|
||||
let line = m.line;
|
||||
if line == 0 || line as usize > self.lines.len() {
|
||||
return Err(ImportRefusal::NoSuchLine { line });
|
||||
}
|
||||
if i > 0 && answer[i - 1].line == line {
|
||||
return Err(ImportRefusal::LineMarkedTwice { line });
|
||||
}
|
||||
let draft = &self.lines[line as usize - 1];
|
||||
let group = format!("{}-{}", self.slug, draft.group);
|
||||
if !is_slug(&group) {
|
||||
return Err(ImportRefusal::Group { line });
|
||||
}
|
||||
if !in_cast(&draft.speaker) {
|
||||
return Err(ImportRefusal::SpeakerNotInCast { line });
|
||||
}
|
||||
let text: String = draft.text.nfc().collect();
|
||||
if text.contains("[[") || text.contains("]]") {
|
||||
return Err(ImportRefusal::Markup { line });
|
||||
}
|
||||
let marked_nfc: String = m.marked.nfc().collect();
|
||||
let (plain, names) = match unmark(&marked_nfc) {
|
||||
Some((plain, names)) if plain == text => (plain, names),
|
||||
_ => return Err(ImportRefusal::Markup { line }),
|
||||
};
|
||||
if names.iter().any(|n| !in_cast(&n.entity)) {
|
||||
return Err(ImportRefusal::NotInCast { line });
|
||||
}
|
||||
items.push(CorpusItem {
|
||||
id: item_key(&self.slug, line),
|
||||
text: plain,
|
||||
language: self.language.clone(),
|
||||
speaker: draft.speaker.clone(),
|
||||
group,
|
||||
prior_turns: Vec::new(),
|
||||
source: self.source.clone(),
|
||||
admitted_on: self.admitted_on.clone(),
|
||||
supersedes: None,
|
||||
withdrawn: false,
|
||||
names,
|
||||
names_by: Some(annotator.clone()),
|
||||
});
|
||||
}
|
||||
Ok((entities, items))
|
||||
}
|
||||
}
|
||||
|
||||
/// The text of `marked` with `[[entity|form]]` replaced by `form`, and the
|
||||
/// byte span of each form in that text. `None` for any malformed markup. The
|
||||
/// only search is for the markup delimiters.
|
||||
fn unmark(marked: &str) -> Option<(String, Vec<NameSpan>)> {
|
||||
let mut out = String::with_capacity(marked.len());
|
||||
let mut names = Vec::new();
|
||||
let mut rest = marked;
|
||||
while let Some(at) = rest.find("[[") {
|
||||
out.push_str(&rest[..at]);
|
||||
let inner = &rest[at + 2..];
|
||||
let (id, after) = inner.split_once('|')?;
|
||||
let (form, tail) = after.split_once("]]")?;
|
||||
if form.is_empty() || form.contains("[[") || id.contains("]]") {
|
||||
return None;
|
||||
}
|
||||
let entity = EntityId::parse(id).ok()?;
|
||||
let start = u32::try_from(out.len()).ok()?;
|
||||
out.push_str(form);
|
||||
names.push(NameSpan {
|
||||
entity,
|
||||
start,
|
||||
end: u32::try_from(out.len()).ok()?,
|
||||
});
|
||||
rest = tail;
|
||||
}
|
||||
out.push_str(rest);
|
||||
Some((out, names))
|
||||
}
|
||||
|
||||
/// `encoder:<hex16>`, a digest of the model id and the prompt digest.
|
||||
pub fn encoder_id(model: &str, prompt_digest: &str) -> EncoderId {
|
||||
let digest = fnv1a64(format!("weksa-encoder-v1:{model}:{prompt_digest}").as_bytes());
|
||||
|
|
@ -370,6 +566,9 @@ pub fn packet_id(item: &CorpusItemId, rev: u32, encoder: &EncoderId) -> PacketId
|
|||
PacketId::parse(&format!("packet:{}", hex16(digest))).expect("a packet id is well formed")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod mutants;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
@ -746,4 +945,145 @@ mod tests {
|
|||
assert!(entries.iter().all(|e| e.annotator == by));
|
||||
assert_eq!(entries[1].examples, ["x"]);
|
||||
}
|
||||
|
||||
const WORK: &str = r#"{"slug":"lantern-wager","language":"en-US","admitted_on":"2026-11-13",
|
||||
"source":{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}},
|
||||
"cast":[{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:tamsin","forms":["Tamsin","Tam"]},{"entity":"entity:wexmoor","forms":["Wexmoor"]}],
|
||||
"lines":[{"group":"s1","speaker":"entity:orsa","text":"Tamsin, bring the lantern."},
|
||||
{"group":"s1","speaker":"entity:tamsin","text":"Orsa, I will not, said Tam."},
|
||||
{"group":"s2","speaker":"entity:wexmoor","text":"Quiet, both of you."}]}"#;
|
||||
|
||||
fn marked(line: u32, text: &str) -> MarkedLine {
|
||||
MarkedLine { line, marked: text.into() }
|
||||
}
|
||||
|
||||
fn model_annotator() -> Annotator {
|
||||
Annotator::Model {
|
||||
family: crate::corpus::ModelFamily::Anthropic,
|
||||
generator: "g".into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn import(
|
||||
work: &str,
|
||||
answer: &[MarkedLine],
|
||||
) -> Result<(Vec<Entity>, Vec<CorpusItem>), ImportRefusal> {
|
||||
WorkDraft::parse(work).unwrap().into_import(answer, &model_annotator())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_work_imports_its_cast_and_marked_lines() {
|
||||
let answer = [
|
||||
marked(1, "[[entity:tamsin|Tamsin]], bring the lantern."),
|
||||
marked(2, "[[entity:orsa|Orsa]], I will not, said [[entity:tamsin|Tam]]."),
|
||||
];
|
||||
let (entities, items) = import(WORK, &answer).unwrap();
|
||||
assert_eq!(entities.len(), 3);
|
||||
for e in &entities {
|
||||
assert!(matches!(&e.origin, EntityOrigin::PublicDomain { work } if work == "The Lantern Wager"));
|
||||
assert_eq!(e.forms[0].language, "en-US");
|
||||
}
|
||||
assert_eq!(items.len(), 2);
|
||||
assert_eq!(items[0].id, "item:lantern-wager:1");
|
||||
assert_eq!(items[1].id, "item:lantern-wager:2");
|
||||
assert_eq!(items[0].group, "lantern-wager-s1");
|
||||
let span = |e: &str, text: &str, form: &str| {
|
||||
let start = text.find(form).unwrap() as u32;
|
||||
NameSpan { entity: EntityId::parse(e).unwrap(), start, end: start + form.len() as u32 }
|
||||
};
|
||||
let t2 = &items[1].text;
|
||||
assert_eq!(items[0].names, [span("entity:tamsin", &items[0].text, "Tamsin")]);
|
||||
assert_eq!(
|
||||
items[1].names,
|
||||
[span("entity:orsa", t2, "Orsa"), span("entity:tamsin", t2, "Tam")]
|
||||
);
|
||||
// "will" is unmarked and unspanned; the line's speaker is the file's.
|
||||
assert_eq!(items[1].speaker, EntityId::parse("entity:tamsin").unwrap());
|
||||
for i in &items {
|
||||
assert_eq!(i.names_by, Some(model_annotator()));
|
||||
assert!(i.prior_turns.is_empty() && i.supersedes.is_none() && !i.withdrawn);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_import_refuses_before_any_document() {
|
||||
let ok = marked(1, "Tamsin, bring the lantern.");
|
||||
let refused = |work: &str, answer: &[MarkedLine]| import(work, answer).unwrap_err();
|
||||
let cast_authored = WORK.replace(
|
||||
r#"{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}}"#,
|
||||
r#""OperatorWritten""#,
|
||||
);
|
||||
assert_eq!(refused(&cast_authored, std::slice::from_ref(&ok)), ImportRefusal::NotPublicDomain);
|
||||
let repeated = WORK.replace(r#"{"entity":"entity:orsa","forms":["Orsa"]}"#, r#"{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:orsa","forms":["O"]}"#);
|
||||
assert_eq!(refused(&repeated, std::slice::from_ref(&ok)), ImportRefusal::CastRepeated);
|
||||
let stranger = WORK.replace(r#""speaker":"entity:orsa""#, r#""speaker":"entity:stranger""#);
|
||||
assert_eq!(refused(&stranger, std::slice::from_ref(&ok)), ImportRefusal::SpeakerNotInCast { line: 1 });
|
||||
assert_eq!(refused(WORK, &[marked(9, "x")]), ImportRefusal::NoSuchLine { line: 9 });
|
||||
assert_eq!(refused(WORK, &[marked(0, "x")]), ImportRefusal::NoSuchLine { line: 0 });
|
||||
assert_eq!(
|
||||
refused(WORK, &[ok.clone(), ok.clone()]),
|
||||
ImportRefusal::LineMarkedTwice { line: 1 }
|
||||
);
|
||||
for bad in [
|
||||
"[[entity:tamsin|Tamsin, bring the lantern.",
|
||||
"[[entity:tamsin Tamsin]], bring the lantern.",
|
||||
"[[tamsin|Tamsin]], bring the lantern.",
|
||||
"[[entity:tamsin|]]Tamsin, bring the lantern.",
|
||||
"Tamsin, bring the lantern",
|
||||
] {
|
||||
assert_eq!(refused(WORK, &[marked(1, bad)]), ImportRefusal::Markup { line: 1 }, "{bad}");
|
||||
}
|
||||
assert_eq!(
|
||||
refused(WORK, &[marked(1, "[[entity:stranger|Tamsin]], bring the lantern.")]),
|
||||
ImportRefusal::NotInCast { line: 1 }
|
||||
);
|
||||
let group = WORK.replace(r#""group":"s1""#, r#""group":"S 1""#);
|
||||
assert_eq!(refused(&group, &[ok]), ImportRefusal::Group { line: 1 });
|
||||
let bracketed = WORK.replace("Tamsin, bring the lantern.", "Tamsin [[bring]] the lantern.");
|
||||
assert_eq!(
|
||||
refused(&bracketed, &[marked(1, "[[entity:tamsin|Tamsin]] [[entity:orsa|bring]] the lantern.")]),
|
||||
ImportRefusal::Markup { line: 1 }
|
||||
);
|
||||
// The refusal text names the kind and the line, no value.
|
||||
assert_eq!(
|
||||
ImportRefusal::Markup { line: 1 }.to_string(),
|
||||
"the marked text of line 1 is not its text with markup"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn markup_offsets_are_bytes_of_the_nfc_text() {
|
||||
let work = WORK
|
||||
.replace("Tamsin, bring the lantern.", "Caf\u{e9} Zo\u{eb}")
|
||||
.replace(r#""forms":["Orsa"]"#, r#""forms":["Zoe\u0308"]"#);
|
||||
let decomposed = "Cafe\u{301} [[entity:orsa|Zoe\u{308}]]";
|
||||
let (entities, items) = import(&work, &[marked(1, decomposed)]).unwrap();
|
||||
assert_eq!(entities[0].forms[0].forms, ["Zo\u{eb}"]);
|
||||
let item = &items[0];
|
||||
assert_eq!(item.text, "Caf\u{e9} Zo\u{eb}");
|
||||
let n = &item.names[0];
|
||||
assert_eq!(&item.text[n.start as usize..n.end as usize], "Zo\u{eb}");
|
||||
assert_eq!(n.start, 6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stray_close_bracket_and_a_scattered_duplicate_answer_are_refused() {
|
||||
// A text holding only `]]` is not a marked line.
|
||||
let close = WORK.replace("Quiet, both of you.", "Quiet ]] both of you.");
|
||||
assert_eq!(
|
||||
import(&close, &[marked(3, "Quiet ]] both of you.")]).unwrap_err(),
|
||||
ImportRefusal::Markup { line: 3 }
|
||||
);
|
||||
// The answer's order does not matter, and a duplicate need not be adjacent.
|
||||
let (_, items) = import(WORK, &[marked(3, "Quiet, both of you."), marked(1, "Tamsin, bring the lantern.")]).unwrap();
|
||||
assert_eq!(items.iter().map(|i| i.id.as_str()).collect::<Vec<_>>(), ["item:lantern-wager:1", "item:lantern-wager:3"]);
|
||||
assert_eq!(
|
||||
import(
|
||||
WORK,
|
||||
&[marked(1, "Tamsin, bring the lantern."), marked(3, "Quiet, both of you."), marked(1, "Tamsin, bring the lantern.")]
|
||||
)
|
||||
.unwrap_err(),
|
||||
ImportRefusal::LineMarkedTwice { line: 1 }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
13
crates/weksa/src/encode/mutants.rs
Normal file
13
crates/weksa/src/encode/mutants.rs
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
//! Pins for `unmark` that the import door only reaches through a whole work.
|
||||
|
||||
use super::*;
|
||||
|
||||
/// A form that opens another markup is malformed, though its entity id is
|
||||
/// sound: the model wrote a nested or unclosed mark, not a name.
|
||||
#[test]
|
||||
fn a_form_that_opens_a_mark_is_malformed() {
|
||||
assert_eq!(unmark("[[entity:orsa|Or[[sa]] came"), None);
|
||||
// The same line without the nested opener unmarks.
|
||||
let (plain, names) = unmark("[[entity:orsa|Orsa]] came").unwrap();
|
||||
assert_eq!((plain.as_str(), names.len()), ("Orsa came", 1));
|
||||
}
|
||||
|
|
@ -31,7 +31,7 @@ pub use inventory::{
|
|||
Change, Family, InvId, Inventory, InventoryEntry, InventoryRevision, OptionLabel, OptionSet,
|
||||
Slot, ValueKind,
|
||||
};
|
||||
pub use lower::{EntityForms, LowerError, PhoneLine, TARGET_LANGUAGE, lower, phones, pronounce};
|
||||
pub use lower::{EntityForms, LowerError, PhoneLine, TARGET_LANGUAGE, phones, pronounce};
|
||||
pub use model::{AnswerFiles, ModelPort};
|
||||
pub use ontology::{Condition, Layer, Ontology, OntologyEntry, Test, select};
|
||||
pub use packet::{
|
||||
|
|
|
|||
|
|
@ -1,52 +1,28 @@
|
|||
//! The English template lowerer: the validated lowering view of a packet to
|
||||
//! text, by a fixed table keyed by inventory ids. It is the deterministic
|
||||
//! lowerer of `weksa lower`, the proof fixtures and their tests; the golf loop
|
||||
//! lowers through the `ModelPort` (`run.rs`, `Prompt::lower`), and follow-up
|
||||
//! `table-retirement` decides this table's future. It renders predications, referents and the
|
||||
//! speech act, and nothing else: no style, no model, no corpus text. It takes a
|
||||
//! [`LoweringInput`], so it cannot see the trace or the corpus item. A packet
|
||||
//! carrying a feature, relation, extension, role fill or referent it does not
|
||||
//! render is refused, because rendering part of a meaning would assert a different
|
||||
//! one (`polarity: negative` dropped says the opposite). Context, kind and
|
||||
//! constraints steer delivery and are not rendered; the concept of a predication
|
||||
//! must resolve but the frame owns the verb.
|
||||
//! The entity port and the ruled pronunciation of a packet's entities.
|
||||
//!
|
||||
//! A referent that cites an entity renders as that entity's own form, read
|
||||
//! through the [`EntityForms`] port: the first (citation) form stored for
|
||||
//! [`TARGET_LANGUAGE`], verbatim. The choice is a table lookup on the language
|
||||
//! tag and the form's position, not a reading of the form. Forms after the
|
||||
//! first are inflected or particle-bound shapes; no English frame asks for one,
|
||||
//! so none is selected. An entity the port does not hold, or one with no form in
|
||||
//! the target language, is refused: the concept word would say a different
|
||||
//! thing than the name the packet cites.
|
||||
//!
|
||||
//! A request clause ("You asked Nibu to stop.") says what the addressee did, so
|
||||
//! it is gated on the packet's own record of it: `Discourse.addressee_consent`
|
||||
//! (set only by authoring or game telemetry, V15) must name that predication,
|
||||
//! and its agent must be the addressee. Without it the lowerer refuses
|
||||
//! ([`LowerError::ConsentMissing`], [`LowerError::AskerNotAddressee`]); it does
|
||||
//! not read wording or guess who asked. A consent naming any other predication
|
||||
//! is an unrendered slot.
|
||||
//!
|
||||
//! The table is a temporary owner. Each row speaks for an inventory entry and
|
||||
//! names the gloss it speaks for, so a row cannot outlive a change of what its id
|
||||
//! means. `language-ontology` moves the rows into ontology documents.
|
||||
//! Wording of a packet in a target language is the `ModelPort`'s answer to
|
||||
//! `Prompt::lower` (`context.rs`); nothing here produces lowered text. This
|
||||
//! module holds what the entity store owns and the lowering path reads: the
|
||||
//! narrow [`EntityForms`] port, the ruled IPA of every entity a packet cites
|
||||
//! ([`pronounce`]), and its segmentation into phones ([`phones`]). A referent
|
||||
//! that cites an entity is read through the port; an entity the port does not
|
||||
//! hold, one with no form or no ruled pronunciation in [`TARGET_LANGUAGE`], is
|
||||
//! refused, because spelling is not a pronunciation and there is no
|
||||
//! grapheme-to-phoneme guess to offer in its place.
|
||||
|
||||
use std::fmt;
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::entity::Entity;
|
||||
use crate::inventory::{Family, InvId, Inventory};
|
||||
use crate::packet::{EntityId, NodeId, Predication};
|
||||
use crate::packet::EntityId;
|
||||
use crate::phones::{Phone, PhoneInventory, SegmentError};
|
||||
use crate::store::EntityStore;
|
||||
use crate::validate::LoweringInput;
|
||||
|
||||
/// The language tag of the template table and the phones: what `weksa lower`
|
||||
/// says in. The golf loop lowers each item in its own language and does not
|
||||
/// read this. Spelled as an entity's `LanguageForms::language` and a corpus
|
||||
/// item's language are.
|
||||
/// The language tag of [`pronounce`] and [`phones`] only. The golf loop lowers
|
||||
/// each item in its own language and does not read this. Spelled as an
|
||||
/// entity's `LanguageForms::language` and a corpus item's language are.
|
||||
pub const TARGET_LANGUAGE: &str = "en-US";
|
||||
|
||||
/// The lowerer's whole view of the entity store: one entity by id. Narrow so a
|
||||
|
|
@ -62,98 +38,8 @@ impl EntityForms for EntityStore {
|
|||
}
|
||||
}
|
||||
|
||||
/// `(inventory id, the gloss the row speaks for, surface)`.
|
||||
type Row = (&'static str, &'static str, &'static str);
|
||||
|
||||
/// Referent concepts, as the noun phrase after "the".
|
||||
const CONCEPTS: &[Row] = &[
|
||||
("concept:3", "player", "player"),
|
||||
("concept:4", "control", "control"),
|
||||
("concept:5", "ship_mind", "ship mind"),
|
||||
// The Dao De Jing test bed (`daodejing.rs`). Nouns, then the verbs a
|
||||
// `{verb}` frame speaks.
|
||||
("concept:7", "sage", "sage"),
|
||||
("concept:8", "mind", "minds"),
|
||||
("concept:9", "belly", "bellies"),
|
||||
("concept:10", "will", "wills"),
|
||||
("concept:11", "bone", "bones"),
|
||||
("concept:12", "empty", "empties"),
|
||||
("concept:13", "fill", "fills"),
|
||||
("concept:14", "weaken", "weakens"),
|
||||
("concept:15", "strengthen", "strengthens"),
|
||||
("concept:16", "heaven", "heaven"),
|
||||
("concept:17", "earth", "earth"),
|
||||
("concept:18", "is_long_enduring", "is long-enduring"),
|
||||
("concept:19", "continues_long", "continues long"),
|
||||
("concept:20", "highest_excellence", "highest excellence"),
|
||||
("concept:21", "water", "water"),
|
||||
("concept:22", "true_words", "words that are strictly true"),
|
||||
("concept:23", "seem_paradoxical", "seem paradoxical"),
|
||||
(
|
||||
"concept:24",
|
||||
"space_between_heaven_and_earth",
|
||||
"space between heaven and earth",
|
||||
),
|
||||
("concept:25", "bellows", "bellows"),
|
||||
("concept:26", "compare", "compare"),
|
||||
("concept:27", "resemble", "resemble"),
|
||||
];
|
||||
|
||||
/// Frames: the clause a predication of that frame says. `{gloss}` is the filler
|
||||
/// of the role whose inventory gloss it is.
|
||||
const FRAMES: &[Row] = &[
|
||||
(
|
||||
"frame:1",
|
||||
"hazard_warning",
|
||||
"{agent} may endanger {patient} by touching {instrument}",
|
||||
),
|
||||
("frame:2", "intervention", "{agent} will stop {patient}"),
|
||||
(
|
||||
"frame:3",
|
||||
"request_intervention",
|
||||
"{agent} asked {patient} to stop",
|
||||
),
|
||||
// The Dao De Jing test bed. Frame 4 was written from a Train chapter, 5 and
|
||||
// 6 from Validation chapters, 7 and 8 from Test chapters.
|
||||
("frame:4", "act", "{agent} {verb} {patient}"),
|
||||
("frame:5", "intransitive", "{experiencer} {verb}"),
|
||||
("frame:6", "likeness", "{experiencer} is like {goal}"),
|
||||
(
|
||||
"frame:7",
|
||||
"seems_paradoxical",
|
||||
"{experiencer} seem paradoxical",
|
||||
),
|
||||
(
|
||||
"frame:8",
|
||||
"may_be_compared",
|
||||
"{experiencer} may be compared to {goal}",
|
||||
),
|
||||
];
|
||||
|
||||
/// The frame of "{agent} asked {patient} to stop": the one frame that reads
|
||||
/// the discourse's addressee consent.
|
||||
const REQUEST_FRAME: &str = "frame:3";
|
||||
|
||||
/// Speech acts: the opener of the first sentence.
|
||||
const SPEECH_ACTS: &[Row] = &[
|
||||
("speech_act:1", "assert", ""),
|
||||
("speech_act:2", "ask", ""),
|
||||
("speech_act:5", "warn", "Warning: "),
|
||||
];
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum LowerError {
|
||||
/// The packet carries a slot the lowerer does not render.
|
||||
Unrendered { what: &'static str },
|
||||
/// No table row for an id the packet uses.
|
||||
NoRow { id: InvId },
|
||||
/// The row's gloss is not the gloss the inventory gives its id at the
|
||||
/// packet's revision, or the id does not resolve there.
|
||||
GlossMismatch { id: InvId, row: &'static str },
|
||||
/// A frame template names a role the inventory has no live entry for.
|
||||
UnknownRole { frame: InvId, role: String },
|
||||
/// The predication fills no role its frame's template needs.
|
||||
MissingRole { predication: NodeId, role: String },
|
||||
/// A referent cites an entity the entity port does not hold.
|
||||
UnknownEntity { entity: EntityId },
|
||||
/// The entity has no form in [`TARGET_LANGUAGE`].
|
||||
|
|
@ -165,7 +51,7 @@ pub enum LowerError {
|
|||
NoEntityReferent,
|
||||
/// Phones were requested for an entity whose ruled IPA holds a symbol the
|
||||
/// language's phone inventory does not list (or the language has no
|
||||
/// inventory). The lowerer does not guess a phone.
|
||||
/// inventory). No phone is guessed.
|
||||
UnknownPhone {
|
||||
entity: EntityId,
|
||||
at: usize,
|
||||
|
|
@ -174,12 +60,6 @@ pub enum LowerError {
|
|||
/// Phones were requested for an entity whose ruled IPA has a stress mark
|
||||
/// with no vowel after it.
|
||||
DanglingStress { entity: EntityId, at: usize },
|
||||
/// A request predication the packet does not carry the addressee's consent
|
||||
/// to: no consent, or a consent that names another predication. The
|
||||
/// lowerer does not guess who asked.
|
||||
ConsentMissing { predication: NodeId },
|
||||
/// A request predication whose agent is not the discourse addressee.
|
||||
AskerNotAddressee { predication: NodeId },
|
||||
/// The entity port failed to read.
|
||||
EntityRead { entity: EntityId, why: String },
|
||||
/// A lowering prompt was asked of a view whose scene was never read.
|
||||
|
|
@ -192,11 +72,6 @@ pub enum LowerError {
|
|||
impl fmt::Display for LowerError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(match self {
|
||||
LowerError::Unrendered { .. } => "packet carries a slot the lowerer does not render",
|
||||
LowerError::NoRow { .. } => "no table row for an inventory id the packet uses",
|
||||
LowerError::GlossMismatch { .. } => "table row does not match the inventory's gloss",
|
||||
LowerError::UnknownRole { .. } => "frame template names a role the inventory lacks",
|
||||
LowerError::MissingRole { .. } => "predication fills no role its frame needs",
|
||||
LowerError::UnknownEntity { .. } => "referent cites an entity the port does not hold",
|
||||
LowerError::NoForm { .. } => "entity has no form in the target language",
|
||||
LowerError::NoPronunciation { .. } => "entity has no ruled pronunciation",
|
||||
|
|
@ -205,149 +80,17 @@ impl fmt::Display for LowerError {
|
|||
"ruled pronunciation holds a symbol with no listed phone"
|
||||
}
|
||||
LowerError::DanglingStress { .. } => "stress mark with no vowel after it",
|
||||
LowerError::ConsentMissing { .. } => "request lacks the addressee's consent",
|
||||
LowerError::AskerNotAddressee { .. } => "request agent is not the addressee",
|
||||
LowerError::EntityRead { .. } => "entity port failed to read",
|
||||
LowerError::SceneUnread => "the view's scene was never read",
|
||||
LowerError::SpeakerTokenTaken => "a referent cites the entity id the speaker token uses",
|
||||
LowerError::SpeakerTokenTaken => {
|
||||
"a referent cites the entity id the speaker token uses"
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for LowerError {}
|
||||
|
||||
fn row(
|
||||
table: &'static [Row],
|
||||
id: InvId,
|
||||
inv: &Inventory,
|
||||
rev: u32,
|
||||
) -> Result<&'static str, LowerError> {
|
||||
let (_, gloss, surface) = table
|
||||
.iter()
|
||||
.find(|(key, _, _)| *key == id.to_string())
|
||||
.ok_or(LowerError::NoRow { id })?;
|
||||
match inv.resolve(&id, rev) {
|
||||
Some(e) if e.gloss == *gloss => Ok(surface),
|
||||
_ => Err(LowerError::GlossMismatch { id, row: gloss }),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn lower(
|
||||
input: &LoweringInput,
|
||||
inv: &Inventory,
|
||||
entities: &dyn EntityForms,
|
||||
) -> Result<String, LowerError> {
|
||||
let rev = input.inventory_rev();
|
||||
let unrendered = |what| Err(LowerError::Unrendered { what });
|
||||
if input.predications().is_empty() {
|
||||
return unrendered("no predication");
|
||||
}
|
||||
if !input.discourse().features.is_empty()
|
||||
|| input.referents().iter().any(|r| !r.features.is_empty())
|
||||
|| input.predications().iter().any(|p| !p.features.is_empty())
|
||||
{
|
||||
return unrendered("features");
|
||||
}
|
||||
request_gate(input, inv)?;
|
||||
if !input.relations().is_empty() {
|
||||
return unrendered("relations");
|
||||
}
|
||||
if !input.extensions().is_empty() {
|
||||
return unrendered("extensions");
|
||||
}
|
||||
let opener = match input.discourse().speech_act {
|
||||
Some(act) => row(SPEECH_ACTS, act, inv, rev)?,
|
||||
None => "",
|
||||
};
|
||||
let mut sentences = Vec::new();
|
||||
for p in input.predications() {
|
||||
sentences.push(clause(input, inv, entities, p)?);
|
||||
}
|
||||
let mentioned = |r: &NodeId| {
|
||||
input
|
||||
.predications()
|
||||
.iter()
|
||||
.any(|p| p.roles.iter().any(|f| &f.filler == r))
|
||||
};
|
||||
if input.referents().iter().any(|r| !mentioned(&r.id)) {
|
||||
return unrendered("a referent no role mentions");
|
||||
}
|
||||
let mut out = String::new();
|
||||
for (i, (clause, name_first)) in sentences.iter().enumerate() {
|
||||
if i > 0 {
|
||||
out.push(' ');
|
||||
}
|
||||
if i == 0 {
|
||||
out.push_str(opener);
|
||||
}
|
||||
let mut chars = clause.chars();
|
||||
if let Some(first) = chars.next() {
|
||||
// A name opens its clause as written; only template words are cased.
|
||||
if (i == 0 && !opener.is_empty()) || *name_first {
|
||||
out.push(first);
|
||||
} else {
|
||||
out.extend(first.to_uppercase());
|
||||
}
|
||||
out.push_str(chars.as_str());
|
||||
}
|
||||
out.push('.');
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// The one place consent is read. A request clause ([`REQUEST_FRAME`]) says what
|
||||
/// the addressee did, so it needs the packet's own record of it: the discourse
|
||||
/// `addressee_consent`, which validation lets only authoring or game telemetry
|
||||
/// set (V15), must name that predication, and its agent must be the addressee.
|
||||
/// Nothing here reads wording. A consent naming any other predication is a slot
|
||||
/// the lowerer does not render.
|
||||
fn request_gate(input: &LoweringInput, inv: &Inventory) -> Result<(), LowerError> {
|
||||
let consent = input.discourse().addressee_consent.as_ref();
|
||||
let requests = input
|
||||
.predications()
|
||||
.iter()
|
||||
.filter(|p| p.frame.to_string() == REQUEST_FRAME);
|
||||
for p in requests {
|
||||
if consent != Some(&p.id) {
|
||||
return Err(LowerError::ConsentMissing {
|
||||
predication: p.id.clone(),
|
||||
});
|
||||
}
|
||||
let agent = inv
|
||||
.entries()
|
||||
.find(|e| {
|
||||
e.family() == Family::Role && e.gloss == "agent" && e.live_at(input.inventory_rev())
|
||||
})
|
||||
.ok_or_else(|| LowerError::UnknownRole {
|
||||
frame: p.frame,
|
||||
role: "agent".to_string(),
|
||||
})?;
|
||||
let asker = p
|
||||
.roles
|
||||
.iter()
|
||||
.find(|r| r.role == agent.id)
|
||||
.map(|r| &r.filler);
|
||||
if asker.is_none() || asker != input.discourse().addressee.as_ref() {
|
||||
return Err(LowerError::AskerNotAddressee {
|
||||
predication: p.id.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
match consent {
|
||||
Some(_)
|
||||
if !input
|
||||
.predications()
|
||||
.iter()
|
||||
.any(|p| p.frame.to_string() == REQUEST_FRAME) =>
|
||||
{
|
||||
Err(LowerError::Unrendered {
|
||||
what: "addressee consent",
|
||||
})
|
||||
}
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// The ruled IPA of every entity a packet cites, one per entity referent in
|
||||
/// the packet's referent order, as stored: bare, stress-marked, nothing
|
||||
/// derived. A rejected pronunciation is never read. An entity the port does not
|
||||
|
|
@ -424,91 +167,6 @@ fn ruled(
|
|||
Ok(out)
|
||||
}
|
||||
|
||||
fn clause(
|
||||
input: &LoweringInput,
|
||||
inv: &Inventory,
|
||||
entities: &dyn EntityForms,
|
||||
p: &Predication,
|
||||
) -> Result<(String, bool), LowerError> {
|
||||
let rev = input.inventory_rev();
|
||||
let template = row(FRAMES, p.frame, inv, rev)?;
|
||||
let mut said: Vec<&NodeId> = Vec::new();
|
||||
let mut out = String::new();
|
||||
let mut name_first = false;
|
||||
let mut rest = template;
|
||||
while let Some(open) = rest.find('{') {
|
||||
out.push_str(&rest[..open]);
|
||||
let close = rest[open..].find('}').map_or(rest.len(), |c| open + c);
|
||||
let name = &rest[open + 1..close];
|
||||
rest = &rest[(close + 1).min(rest.len())..];
|
||||
if name == "verb" {
|
||||
out.push_str(row(CONCEPTS, p.concept, inv, rev)?);
|
||||
continue;
|
||||
}
|
||||
let role = inv
|
||||
.entries()
|
||||
.find(|e| e.family() == Family::Role && e.gloss == name && e.live_at(rev))
|
||||
.ok_or_else(|| LowerError::UnknownRole {
|
||||
frame: p.frame,
|
||||
role: name.to_string(),
|
||||
})?;
|
||||
let fill =
|
||||
p.roles
|
||||
.iter()
|
||||
.find(|r| r.role == role.id)
|
||||
.ok_or_else(|| LowerError::MissingRole {
|
||||
predication: p.id.clone(),
|
||||
role: name.to_string(),
|
||||
})?;
|
||||
let (phrase, is_name) = filler(input, inv, entities, &fill.filler, &said)?;
|
||||
name_first |= is_name && out.is_empty();
|
||||
out.push_str(&phrase);
|
||||
said.push(&fill.filler);
|
||||
}
|
||||
out.push_str(rest);
|
||||
if p.roles.len() != said.len() {
|
||||
return Err(LowerError::Unrendered {
|
||||
what: "a role the frame does not name",
|
||||
});
|
||||
}
|
||||
Ok((out, name_first))
|
||||
}
|
||||
|
||||
/// A role filler as a phrase: the addressee is "you", a predication is "that",
|
||||
/// another referent is its entity's citation form when it cites one, else "the"
|
||||
/// and its concept word. The addressee said twice in
|
||||
/// one predication is reflexive; no other referent is, because the packet has no
|
||||
/// animacy to choose "themself" or "itself" by.
|
||||
fn filler(
|
||||
input: &LoweringInput,
|
||||
inv: &Inventory,
|
||||
entities: &dyn EntityForms,
|
||||
node: &NodeId,
|
||||
said: &[&NodeId],
|
||||
) -> Result<(String, bool), LowerError> {
|
||||
let again = said.contains(&node);
|
||||
if let Some(r) = input.referents().iter().find(|r| &r.id == node) {
|
||||
if input.discourse().addressee.as_ref() == Some(node) {
|
||||
return Ok((if again { "yourself" } else { "you" }.to_string(), false));
|
||||
}
|
||||
if let Some(id) = &r.entity {
|
||||
return Ok((name(entities, id)?, true));
|
||||
}
|
||||
let word = row(CONCEPTS, r.concept, inv, input.inventory_rev())?;
|
||||
return Ok((format!("the {word}"), false));
|
||||
}
|
||||
Ok(("that".to_string(), false))
|
||||
}
|
||||
|
||||
/// The entity's citation form in [`TARGET_LANGUAGE`].
|
||||
fn name(entities: &dyn EntityForms, id: &EntityId) -> Result<String, LowerError> {
|
||||
entity(entities, id)?
|
||||
.forms_in(TARGET_LANGUAGE)
|
||||
.and_then(<[String]>::first)
|
||||
.cloned()
|
||||
.ok_or_else(|| LowerError::NoForm { entity: id.clone() })
|
||||
}
|
||||
|
||||
pub(crate) fn entity(entities: &dyn EntityForms, id: &EntityId) -> Result<Entity, LowerError> {
|
||||
entities
|
||||
.entity(id)
|
||||
|
|
|
|||
|
|
@ -1,14 +1,14 @@
|
|||
//! The `weksa` command line: the same validator and store admission the library
|
||||
//! exposes, plus text projections (`dump`) that are for review and are not stores.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use clap::error::{ContextKind, ContextValue, ErrorKind};
|
||||
use clap::{Args, Parser, Subcommand, ValueEnum};
|
||||
use unicode_normalization::UnicodeNormalization;
|
||||
use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn};
|
||||
use weksa::encode::{EditDraft, OntologyDraft};
|
||||
use weksa::encode::{EditDraft, MarkedLine, OntologyDraft, WorkDraft};
|
||||
use weksa::entity::{Entity, EntityOrigin, LanguageForms, Pronunciation};
|
||||
use weksa::inventory::{proof_revision, seed, speaker_scene_revision};
|
||||
use weksa::packet::{FeatValue, FeatureValue, Grade};
|
||||
|
|
@ -20,7 +20,7 @@ use weksa::run::{
|
|||
use weksa::score::ScoreInput;
|
||||
use weksa::{
|
||||
Annotator, CorpusStore, EntityId, EntityStore, Family, InventoryStore, OllamaEmbedder, OntologyStore, OptionLabel,
|
||||
PacketStore, PairRelation, Party, Scene, Slot, SpeakerStateStore, ValueKind, Scorer, Split, SplitScheme, StoreError, lower, pronounce, validate,
|
||||
PacketStore, PairRelation, Party, Scene, Slot, SpeakerStateStore, ValueKind, Scorer, Split, SplitScheme, StoreError, StoreRefusal, pronounce, validate,
|
||||
};
|
||||
|
||||
#[derive(Parser)]
|
||||
|
|
@ -64,17 +64,14 @@ enum Command {
|
|||
Scene(SceneCommand),
|
||||
#[command(subcommand)]
|
||||
Packet(PacketCommand),
|
||||
/// Lower a stored packet to English text.
|
||||
Lower {
|
||||
/// Print the ruled IPA of each entity a stored packet cites, one per line;
|
||||
/// refused for an entity with no ruled pronunciation.
|
||||
Pronounce {
|
||||
store: PathBuf,
|
||||
key: String,
|
||||
/// Print the ruled IPA of each entity the packet cites, one per line,
|
||||
/// instead of the text; refused for an entity with no ruled pronunciation.
|
||||
#[arg(long, conflicts_with = "phones")]
|
||||
ipa: bool,
|
||||
/// Print the phones line of each entity the packet cites, one JSON
|
||||
/// object per line ({language, source, phones: [{ipa, stress}]}),
|
||||
/// instead of the text; refused for a phone the inventory lacks.
|
||||
/// Print the phones line of each entity instead, one JSON object per
|
||||
/// line ({language, source, phones: [{ipa, stress}]}); refused for a
|
||||
/// phone the inventory lacks.
|
||||
#[arg(long)]
|
||||
phones: bool,
|
||||
},
|
||||
|
|
@ -242,6 +239,22 @@ enum CorpusCommand {
|
|||
Withdraw {
|
||||
key: String,
|
||||
},
|
||||
/// Import a transcribed public-domain work: mint its cast as entities and
|
||||
/// add each line a name-span answer covers (`docs/name-span-annotation.md`).
|
||||
Import {
|
||||
/// The work file: provenance, cast, lines.
|
||||
work: PathBuf,
|
||||
/// The annotator's answer: one `{"line": N, "marked": "..."}` per line.
|
||||
names: PathBuf,
|
||||
#[arg(long, requires = "generator", conflicts_with = "operator")]
|
||||
model_family: Option<ModelFamilyArg>,
|
||||
/// The attestation string naming the annotating model.
|
||||
#[arg(long, requires = "model_family")]
|
||||
generator: Option<String>,
|
||||
/// Annotated by the operator, not a model.
|
||||
#[arg(long)]
|
||||
operator: bool,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
|
|
@ -681,6 +694,7 @@ impl AddArgs {
|
|||
supersedes: self.supersedes,
|
||||
withdrawn: false,
|
||||
names,
|
||||
names_by: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -698,6 +712,22 @@ fn split_text(item: &CorpusItem) -> String {
|
|||
.map_or_else(|| "excluded".to_string(), |s| format!("{s:?}"))
|
||||
}
|
||||
|
||||
/// The annotator the flags name: a model family with its generator, or the operator.
|
||||
fn annotator(
|
||||
model_family: Option<ModelFamilyArg>,
|
||||
generator: Option<String>,
|
||||
operator: bool,
|
||||
) -> Result<Annotator> {
|
||||
match (model_family, generator, operator) {
|
||||
(Some(family), Some(generator), false) => Ok(Annotator::Model {
|
||||
family: family.into(),
|
||||
generator,
|
||||
}),
|
||||
(None, None, true) => Ok(Annotator::Operator),
|
||||
_ => bail!("name the annotator: --model-family with --generator, or --operator"),
|
||||
}
|
||||
}
|
||||
|
||||
fn refused(e: StoreError) -> anyhow::Error {
|
||||
anyhow::anyhow!("{e}")
|
||||
}
|
||||
|
|
@ -737,13 +767,78 @@ fn main() -> Result<()> {
|
|||
.map_err(refused)?;
|
||||
println!("{outcome:?} {key}");
|
||||
}
|
||||
Command::Corpus(CorpusCommand::Import {
|
||||
work,
|
||||
names,
|
||||
model_family,
|
||||
generator,
|
||||
operator,
|
||||
}) => {
|
||||
let annotator = annotator(model_family, generator, operator)?;
|
||||
let draft = WorkDraft::parse(&std::fs::read_to_string(&work)?)?;
|
||||
let mut marked = Vec::new();
|
||||
for (i, line) in std::fs::read_to_string(&names)?.lines().enumerate() {
|
||||
if line.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
marked.push(
|
||||
MarkedLine::parse(line)
|
||||
.map_err(|e| anyhow::anyhow!("names line {}: {e}", i + 1))?,
|
||||
);
|
||||
}
|
||||
let (new_entities, items) = draft.into_import(&marked, &annotator)?;
|
||||
// Every document passes the real doors on scratch copies of the
|
||||
// stores before the first write, so a refusal leaves them untouched.
|
||||
let scratch = std::env::temp_dir().join(format!("weksa-import-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&scratch)?;
|
||||
let dry = (|| -> Result<()> {
|
||||
let copy = |from: &Path, name: &str| -> Result<PathBuf> {
|
||||
let to = scratch.join(name);
|
||||
if from.exists() {
|
||||
std::fs::copy(from, &to)?;
|
||||
}
|
||||
Ok(to)
|
||||
};
|
||||
let mut entities = EntityStore::open(©(&cli.entities, "e.cc")?)?;
|
||||
let mut store = CorpusStore::open(©(&cli.corpus, "c.cc")?)?;
|
||||
for e in new_entities.clone() {
|
||||
entities
|
||||
.put(e)
|
||||
.map_err(|e| anyhow::anyhow!("a cast entity: {e}"))?;
|
||||
}
|
||||
for item in items.clone() {
|
||||
let key = item.id.clone();
|
||||
store.add(item, &entities).map_err(|e| match e {
|
||||
StoreError::Refused(StoreRefusal::S6TextHeld { held_by, .. }) => {
|
||||
anyhow::anyhow!("{key} repeats the text of {held_by}")
|
||||
}
|
||||
e => anyhow::anyhow!("{key}: {e}"),
|
||||
})?;
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
let _ = std::fs::remove_dir_all(&scratch);
|
||||
dry?;
|
||||
let mut entities = EntityStore::open(&cli.entities)?;
|
||||
for e in new_entities {
|
||||
let id = e.id.clone();
|
||||
let outcome = entities.put(e).map_err(refused)?;
|
||||
println!("{outcome:?} {id}");
|
||||
}
|
||||
let mut store = CorpusStore::open(&cli.corpus)?;
|
||||
for item in items {
|
||||
let (key, split) = (item.id.clone(), split_text(&item));
|
||||
let outcome = store.add(item, &entities).map_err(refused)?;
|
||||
println!("{outcome:?} {key} {split}");
|
||||
}
|
||||
}
|
||||
Command::Corpus(CorpusCommand::Dump) => {
|
||||
for i in CorpusStore::open(&cli.corpus)?.all()? {
|
||||
let split = split_text(&i);
|
||||
let withdrawn = if i.withdrawn { " withdrawn" } else { "" };
|
||||
println!(
|
||||
"{} [{} {} group={} {split}{withdrawn}]\n {:?}\n names: {:?}\n source: {:?}",
|
||||
i.id, i.language, i.speaker, i.group, i.text, i.names, i.source
|
||||
"{} [{} {} group={} {split}{withdrawn}]\n {:?}\n names: {:?} by {:?}\n source: {:?}",
|
||||
i.id, i.language, i.speaker, i.group, i.text, i.names, i.names_by, i.source
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -880,14 +975,7 @@ fn main() -> Result<()> {
|
|||
generator,
|
||||
operator,
|
||||
}) => {
|
||||
let annotator = match (model_family, generator, operator) {
|
||||
(Some(family), Some(generator), false) => Annotator::Model {
|
||||
family: family.into(),
|
||||
generator,
|
||||
},
|
||||
(None, None, true) => Annotator::Operator,
|
||||
_ => bail!("name the annotator: --model-family with --generator, or --operator"),
|
||||
};
|
||||
let annotator = annotator(model_family, generator, operator)?;
|
||||
let inventory_store = InventoryStore::open(&cli.inventory)?;
|
||||
let inventory = inventory_store.inventory()?;
|
||||
let corpus = CorpusStore::open(&cli.corpus)?;
|
||||
|
|
@ -1029,14 +1117,8 @@ fn main() -> Result<()> {
|
|||
.map_err(refused)?;
|
||||
println!("admitted {key}");
|
||||
}
|
||||
Command::Lower {
|
||||
store,
|
||||
key,
|
||||
ipa,
|
||||
phones,
|
||||
} => {
|
||||
Command::Pronounce { store, key, phones } => {
|
||||
let inventory_store = InventoryStore::open(&cli.inventory)?;
|
||||
let inventory = inventory_store.inventory()?;
|
||||
let mut packets = PacketStore::open(&store)?;
|
||||
let packet = packets.get(&key)?.context("no such packet")?;
|
||||
// Admission is the only source of a LoweringInput; re-admitting the
|
||||
|
|
@ -1050,16 +1132,14 @@ fn main() -> Result<()> {
|
|||
)
|
||||
.map_err(refused)?;
|
||||
let entities = EntityStore::open(&cli.entities)?;
|
||||
if ipa {
|
||||
for line in pronounce(&view, &entities)? {
|
||||
println!("{line}");
|
||||
}
|
||||
} else if phones {
|
||||
if phones {
|
||||
for line in weksa::phones(&view, &entities)? {
|
||||
println!("{}", serde_json::to_string(&line)?);
|
||||
}
|
||||
} else {
|
||||
println!("{}", lower(&view, &inventory, &entities)?);
|
||||
for line in pronounce(&view, &entities)? {
|
||||
println!("{line}");
|
||||
}
|
||||
}
|
||||
}
|
||||
Command::Score(args) => {
|
||||
|
|
|
|||
|
|
@ -19,8 +19,7 @@
|
|||
//! take caller stores (`validate`, `packet proof|dao`, `lower`) are unchanged.
|
||||
//!
|
||||
//! A round encodes, admits, then lowers each admitted view through the
|
||||
//! `ModelPort` (`Prompt::lower`), in two answer phases; the CLI's `lower` still
|
||||
//! uses the template table. The prompt source pin covers `context.rs`,
|
||||
//! `ModelPort` (`Prompt::lower`), in two answer phases; no other path produces lowered text. The prompt source pin covers `context.rs`,
|
||||
//! `split.rs` and `lower.rs`.
|
||||
//!
|
||||
//! `RunStores::stance_ablation`, like `recall`, is a diagnostic in its own
|
||||
|
|
@ -1485,13 +1484,11 @@ impl RunStores {
|
|||
p.stance = Stance::default();
|
||||
variants[0] = Some(prompt_of(1, p)?);
|
||||
}
|
||||
if held.len() > 1 {
|
||||
let mut p = packet.clone();
|
||||
p.stance = held[(i + 1) % held.len()].1.stance.clone();
|
||||
let prompt = prompt_of(2, p)?;
|
||||
if prompt.digest() != base.digest() {
|
||||
variants[1] = Some(prompt);
|
||||
}
|
||||
let mut p = packet.clone();
|
||||
p.stance = held[(i + 1) % held.len()].1.stance.clone();
|
||||
let prompt = prompt_of(2, p)?;
|
||||
if prompt.digest() != base.digest() {
|
||||
variants[1] = Some(prompt);
|
||||
}
|
||||
// The round's own answer must exist; a missing one is not a gap to ask for.
|
||||
let recorded_text = match asked(recorded, &base)? {
|
||||
|
|
@ -3112,6 +3109,9 @@ fn discard_inventory(dir: &Path) {
|
|||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod mutants;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
@ -3972,7 +3972,7 @@ mod tests {
|
|||
);
|
||||
}
|
||||
if prompt.text().starts_with(crate::context::LOWER_INSTRUCTIONS) {
|
||||
// The table's English for the Dao items, the lowerer's answer.
|
||||
// The fake lowerer's answer for the Dao items.
|
||||
let text = match prompt.item_keys()[0].as_str() {
|
||||
"item:daodejing:1" => "The sage empties the minds. The sage fills the bellies. The sage weakens the wills. The sage strengthens the bones.",
|
||||
"item:daodejing:2" => "The heaven is long-enduring. The earth continues long.",
|
||||
|
|
|
|||
21
crates/weksa/src/run/mutants.rs
Normal file
21
crates/weksa/src/run/mutants.rs
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
//! Pins for private functions of `run.rs` that the doors reach only through a
|
||||
//! whole run.
|
||||
|
||||
use super::*;
|
||||
|
||||
/// A line is probed only when it has at least four words, and then it is
|
||||
/// split in two near its byte middle, the pieces trimmed.
|
||||
#[test]
|
||||
fn a_recall_line_splits_from_four_words_and_not_before() {
|
||||
assert_eq!(recall_split("one two three"), None);
|
||||
assert_eq!(recall_split(""), None);
|
||||
assert_eq!(
|
||||
recall_split("one two three four"),
|
||||
Some(("one two".to_string(), "three four".to_string()))
|
||||
);
|
||||
// No spaces: each ideograph is a word, the split falls at the middle.
|
||||
assert_eq!(
|
||||
recall_split("天地不仁以万物为刍狗"),
|
||||
Some(("天地不仁以".to_string(), "万物为刍狗".to_string()))
|
||||
);
|
||||
}
|
||||
|
|
@ -172,6 +172,9 @@ impl Checkpoint {
|
|||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod mutants;
|
||||
|
||||
/// Item builders for the unit tests that live in `src`: the twins of
|
||||
/// `tests/common`'s `group_in_bucket`, `item_in`, `cast` and `model`.
|
||||
#[cfg(test)]
|
||||
|
|
@ -208,6 +211,7 @@ pub(crate) mod fixture {
|
|||
language: "en-US".into(),
|
||||
speaker: EntityId::parse("entity:nibu").unwrap(),
|
||||
names: vec![],
|
||||
names_by: None,
|
||||
group: group.into(),
|
||||
prior_turns: vec![],
|
||||
source,
|
||||
|
|
|
|||
11
crates/weksa/src/split/mutants.rs
Normal file
11
crates/weksa/src/split/mutants.rs
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
//! Pins of private items of `split` found unpinned by mutation testing.
|
||||
|
||||
use super::Checkpoint;
|
||||
|
||||
/// A checkpoint reports the round it was declared at, whatever the round.
|
||||
#[test]
|
||||
fn a_checkpoint_reports_its_round() {
|
||||
assert_eq!(Checkpoint::for_test(7).round(), 7);
|
||||
assert_eq!(Checkpoint::for_test(0).round(), 0);
|
||||
assert_eq!(Checkpoint::for_test(1).round(), 1);
|
||||
}
|
||||
|
|
@ -344,6 +344,9 @@ impl CorpusStore {
|
|||
if let Some(why) = empty_attestation(&item.source) {
|
||||
return Err(malformed(&item.id, why));
|
||||
}
|
||||
if let Some(by) = &item.names_by {
|
||||
check_annotator(&item.id, by)?;
|
||||
}
|
||||
check_entities(&item, entities)?;
|
||||
for held in self.all()? {
|
||||
if normalized(&held.text) == text && item.supersedes.as_deref() != Some(&held.id) {
|
||||
|
|
@ -487,13 +490,13 @@ pub(crate) fn normalized(text: &str) -> String {
|
|||
}
|
||||
|
||||
/// The one-to-one folds `normalized` adds to NFKC and case folding, applied
|
||||
/// before the case fold (U+0130 would otherwise fold to i plus a combining dot,
|
||||
/// which the key keeps as a mark).
|
||||
/// before the case fold. (U+0130 needs no arm: it folds to i plus a combining
|
||||
/// dot, and the `dotless` loop in `normalized` drops that dot after an `i`, so
|
||||
/// the key is plain `i`.)
|
||||
fn fold_variant(c: char) -> impl Iterator<Item = char> {
|
||||
let one = |c: char| (c, None);
|
||||
let (first, second) = match c {
|
||||
'\u{131}' => one('i'),
|
||||
'\u{130}' => one('I'),
|
||||
// Voiced katakana wa, wi, we, wo: the hiragana (U+308F-3092) plus the
|
||||
// combining voiced mark, which is how NFKC spells the hiragana forms.
|
||||
'\u{30f7}'..='\u{30fa}' => (
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
//! lowerer sees, comes from `PacketStore::admit` alone, which builds it from
|
||||
//! the admitted packet and its corpus item through the crate-private
|
||||
//! `LoweringInput::of`. The stance is checked like every feature
|
||||
//! list, in slot `Stance`. The rules are V1..V8, V10..V12 and V14..V17: the
|
||||
//! list, in slot `Stance`. The rules are V1..V8, V10..V12 and V14..V18: the
|
||||
//! shape of an id or a grade is its type's parse, not a rule, so V9 is retired
|
||||
//! and not renumbered; V13 is retired too. [`LoweringInput::with_scene`] reads the
|
||||
//! scene of the view's own item group from the speaker-state store, and
|
||||
|
|
@ -120,6 +120,12 @@ pub enum Refusal {
|
|||
V17AddresseeNotAReferent {
|
||||
id: NodeId,
|
||||
},
|
||||
/// V18: a predication whose frame is the inventory's `request_intervention`,
|
||||
/// with no `addressee_consent` naming it. It says what the addressee did, so
|
||||
/// only the packet's own record may license it.
|
||||
V18RequestWithoutConsent {
|
||||
predication: NodeId,
|
||||
},
|
||||
}
|
||||
|
||||
impl Refusal {
|
||||
|
|
@ -144,6 +150,7 @@ impl Refusal {
|
|||
Refusal::V15ConsentSource { .. } => "V15",
|
||||
Refusal::V16SpeakerNotInScene => "V16",
|
||||
Refusal::V17AddresseeNotAReferent { .. } => "V17",
|
||||
Refusal::V18RequestWithoutConsent { .. } => "V18",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -173,6 +180,9 @@ impl fmt::Display for Refusal {
|
|||
"the scene does not cover the speaker of every live item of its group"
|
||||
}
|
||||
Refusal::V17AddresseeNotAReferent { .. } => "the addressee names no referent",
|
||||
Refusal::V18RequestWithoutConsent { .. } => {
|
||||
"a request frame needs the addressee's consent naming it"
|
||||
}
|
||||
};
|
||||
write!(f, "{}: {what}", self.rule())
|
||||
}
|
||||
|
|
@ -221,6 +231,7 @@ pub fn validate(packet: &Packet, inventory: &Inventory) -> Result<(), Vec<Refusa
|
|||
out: &mut out,
|
||||
}
|
||||
.check(packet);
|
||||
check_requests(packet, inventory, &mut out);
|
||||
} else {
|
||||
out.push(Refusal::V10UnknownRevision {
|
||||
rev: packet.inventory_rev,
|
||||
|
|
@ -303,6 +314,25 @@ fn check_consent(packet: &Packet, nodes: &BTreeSet<NodeId>, out: &mut Vec<Refusa
|
|||
}
|
||||
}
|
||||
|
||||
/// The gloss of the inventory frame that reports an addressee's request.
|
||||
const REQUEST_FRAME_GLOSS: &str = "request_intervention";
|
||||
|
||||
/// V18: a predication whose frame, resolved at the packet's revision, is the
|
||||
/// request frame is named by `addressee_consent`. An unresolved frame is V5's.
|
||||
fn check_requests(packet: &Packet, inventory: &Inventory, out: &mut Vec<Refusal>) {
|
||||
for p in &packet.predications {
|
||||
if inventory
|
||||
.resolve(&p.frame, packet.inventory_rev)
|
||||
.is_some_and(|e| e.gloss == REQUEST_FRAME_GLOSS)
|
||||
&& packet.discourse.addressee_consent.as_ref() != Some(&p.id)
|
||||
{
|
||||
out.push(Refusal::V18RequestWithoutConsent {
|
||||
predication: p.id.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn node_lists(packet: &Packet) -> [(&Vec<NodeId>, &'static str); 3] {
|
||||
[
|
||||
(&packet.context.participants, "participant"),
|
||||
|
|
|
|||
|
|
@ -904,7 +904,7 @@ fn fake_embedder() -> String {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn proof_packet_lowers_and_the_lowering_scores_against_items() {
|
||||
fn proof_packet_pronounces_and_the_item_scores_against_its_own_text() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
cast_entities(d.path(), &["nibu"]);
|
||||
for args in [&["inventory", "seed"][..], &["inventory", "proof"]] {
|
||||
|
|
@ -939,21 +939,11 @@ fn proof_packet_lowers_and_the_lowering_scores_against_items() {
|
|||
"{}",
|
||||
out(&admitted)
|
||||
);
|
||||
let lowered = weksa(d.path(), &["lower", "./p.cc", "packet:6e6962752d303031"]);
|
||||
assert!(lowered.status.success(), "{}", out(&lowered));
|
||||
let text = String::from_utf8(lowered.stdout).unwrap();
|
||||
assert!(
|
||||
text.starts_with("Warning: you may endanger yourself"),
|
||||
"{text}"
|
||||
);
|
||||
// The referent cites entity:nibu: its stored en-US form (NIBU here) reaches the text as stored.
|
||||
assert!(text.contains("NIBU will stop"), "{text}");
|
||||
|
||||
let missing = weksa(d.path(), &["lower", "./p.cc", "packet:0000000000000000"]);
|
||||
let missing = weksa(d.path(), &["pronounce", "./p.cc", "packet:0000000000000000"]);
|
||||
assert!(!missing.status.success() && out(&missing).contains("no such packet"));
|
||||
|
||||
// IPA is the ruled pronunciation or nothing: spelling is not one.
|
||||
let args = ["lower", "./p.cc", "packet:6e6962752d303031", "--ipa"];
|
||||
let args = ["pronounce", "./p.cc", "packet:6e6962752d303031"];
|
||||
let unruled = weksa(d.path(), &args);
|
||||
assert!(
|
||||
!unruled.status.success() && out(&unruled).contains("entity has no ruled pronunciation"),
|
||||
|
|
@ -993,11 +983,10 @@ fn proof_packet_lowers_and_the_lowering_scores_against_items() {
|
|||
let ipa = weksa(d.path(), &args);
|
||||
assert!(ipa.status.success(), "{}", out(&ipa));
|
||||
assert_eq!(String::from_utf8(ipa.stdout).unwrap(), "ˈnibu\n");
|
||||
// The phones line is the committed fixture, byte for byte, and is a different
|
||||
// output from --ipa, so the two flags cannot be combined.
|
||||
// The phones line is the committed fixture, byte for byte.
|
||||
let phones = weksa(
|
||||
d.path(),
|
||||
&["lower", "./p.cc", "packet:6e6962752d303031", "--phones"],
|
||||
&["pronounce", "./p.cc", "packet:6e6962752d303031", "--phones"],
|
||||
);
|
||||
assert!(phones.status.success(), "{}", out(&phones));
|
||||
let fixture = std::fs::read_to_string(
|
||||
|
|
@ -1006,21 +995,13 @@ fn proof_packet_lowers_and_the_lowering_scores_against_items() {
|
|||
)
|
||||
.unwrap();
|
||||
assert_eq!(String::from_utf8(phones.stdout).unwrap(), fixture);
|
||||
let both = weksa(
|
||||
d.path(),
|
||||
&[
|
||||
"lower",
|
||||
"./p.cc",
|
||||
"packet:6e6962752d303031",
|
||||
"--phones",
|
||||
"--ipa",
|
||||
],
|
||||
);
|
||||
assert!(!both.status.success(), "{}", out(&both));
|
||||
let again = weksa(d.path(), &["lower", "./p.cc", "packet:6e6962752d303031"]);
|
||||
assert_eq!(String::from_utf8(again.stdout).unwrap(), text);
|
||||
// The old lowering subcommand is gone: clap's usage error, exit 2.
|
||||
let gone = weksa(d.path(), &["lower", "./p.cc", "packet:6e6962752d303031"]);
|
||||
assert_eq!(gone.status.code(), Some(2), "{}", out(&gone));
|
||||
|
||||
std::fs::write(d.path().join("out.txt"), &text).unwrap();
|
||||
// The item's own text, as the scorer's output.
|
||||
let text = "Warning: you may endanger yourself by touching the control. NIBU will stop that.";
|
||||
std::fs::write(d.path().join("out.txt"), text).unwrap();
|
||||
let url = fake_embedder();
|
||||
let scored = weksa(
|
||||
d.path(),
|
||||
|
|
@ -1180,7 +1161,7 @@ fn cli_errors_never_echo_identifiers() {
|
|||
]),
|
||||
"--model-family",
|
||||
);
|
||||
// Keys on withdraw, split, validate, lower and score.
|
||||
// Keys on withdraw, split, validate, pronounce and score.
|
||||
refuses_without_echo(p, &["corpus", "withdraw", "item:canary:7"], "no such item");
|
||||
refuses_without_echo(p, &["split", "item:canary:7"], "no such item");
|
||||
std::fs::write(p.join("out.txt"), "text").unwrap();
|
||||
|
|
@ -1198,7 +1179,7 @@ fn cli_errors_never_echo_identifiers() {
|
|||
&["validate", "p.cc", "packet:canary:7"],
|
||||
"no such packet",
|
||||
);
|
||||
refuses_without_echo(p, &["lower", "p.cc", "packet:canary:7"], "no such packet");
|
||||
refuses_without_echo(p, &["pronounce", "p.cc", "packet:canary:7"], "no such packet");
|
||||
// S6 names no held key; S5 names no group.
|
||||
let held = weksa(
|
||||
p,
|
||||
|
|
@ -2164,3 +2145,130 @@ fn inventory_admit_cap_is_256_and_a_refusal_writes_nothing() {
|
|||
assert!(admitted.status.success(), "{}", out(&admitted));
|
||||
assert!(out(&admitted).contains("admitted rev:2 with 256 entries"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corpus_import_labels_spans_and_is_idempotent() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
let work = d.path().join("work.json");
|
||||
std::fs::write(
|
||||
&work,
|
||||
r#"{"slug":"lantern-wager","language":"en-US","admitted_on":"2026-11-13",
|
||||
"source":{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}},
|
||||
"cast":[{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:tamsin","forms":["Tamsin","Tam"]}],
|
||||
"lines":[{"group":"s1","speaker":"entity:orsa","text":"Tamsin, bring the lantern."},
|
||||
{"group":"s1","speaker":"entity:tamsin","text":"Orsa, I will not."}]}"#,
|
||||
)
|
||||
.unwrap();
|
||||
let names = d.path().join("names.jsonl");
|
||||
std::fs::write(
|
||||
&names,
|
||||
concat!(
|
||||
r#"{"line":1,"marked":"[[entity:tamsin|Tamsin]], bring the lantern."}"#, "\n\n",
|
||||
r#"{"line":2,"marked":"[[entity:orsa|Orsa]], I will not."}"#, "\n",
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
let (w, n) = (work.to_str().unwrap(), names.to_str().unwrap());
|
||||
let bare = weksa(d.path(), &["corpus", "import", w, n]);
|
||||
assert!(!bare.status.success());
|
||||
let flags = ["--model-family", "anthropic", "--generator", "a name annotator"];
|
||||
let run = || weksa(d.path(), &[&["corpus", "import", w, n][..], &flags].concat());
|
||||
let first = run();
|
||||
assert!(first.status.success(), "{}", out(&first));
|
||||
let text = out(&first);
|
||||
assert!(text.contains("Added item:lantern-wager:1 ") && text.contains("Added item:lantern-wager:2 "), "{text}");
|
||||
assert!(text.contains("Added entity:orsa"), "{text}");
|
||||
let second = out(&run());
|
||||
assert!(second.contains("Unchanged item:lantern-wager:1 ") && !second.contains("Added"), "{second}");
|
||||
let dump = out(&weksa(d.path(), &["corpus", "dump"]));
|
||||
assert!(dump.contains("by Some(Model"), "{dump}");
|
||||
}
|
||||
|
||||
const LANTERN_CAST: &str = r#""cast":[{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:tamsin","forms":["Tamsin","Tam"]}]"#;
|
||||
const LANTERN_SOURCE: &str = r#""source":{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}}"#;
|
||||
|
||||
/// Writes a work of `lines` (text per line, all said by Orsa) and an answer of
|
||||
/// `marked` per line, returns both paths.
|
||||
fn lantern(dir: &std::path::Path, lines: &[&str], marked: &[&str]) -> (String, String) {
|
||||
let ls: Vec<String> = lines
|
||||
.iter()
|
||||
.map(|t| format!(r#"{{"group":"s1","speaker":"entity:orsa","text":"{t}"}}"#))
|
||||
.collect();
|
||||
let work = dir.join("w.json");
|
||||
std::fs::write(
|
||||
&work,
|
||||
format!(
|
||||
r#"{{"slug":"lantern-wager","language":"en-US","admitted_on":"2026-11-13",{LANTERN_SOURCE},{LANTERN_CAST},"lines":[{}]}}"#,
|
||||
ls.join(",")
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
let names = dir.join("n.jsonl");
|
||||
let rows: Vec<String> = marked
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, m)| format!(r#"{{"line":{},"marked":"{m}"}}"#, i + 1))
|
||||
.collect();
|
||||
std::fs::write(&names, rows.join("\n")).unwrap();
|
||||
(work.to_str().unwrap().into(), names.to_str().unwrap().into())
|
||||
}
|
||||
|
||||
const MODEL: [&str; 4] = ["--model-family", "anthropic", "--generator", "a name annotator"];
|
||||
|
||||
#[test]
|
||||
fn a_refused_import_writes_nothing_and_a_redraw_imports() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
let lines = ["Tamsin, bring the lantern.", "Tamsin's lantern, said Orsa."];
|
||||
let (w, n) = lantern(d.path(), &lines, &["[[entity:tamsin|Tamsin]], bring the lantern.", "[[entity:tamsin|Tamsin's]] lantern, said Orsa."]);
|
||||
let import = |w: &str, n: &str| weksa(d.path(), &[&["corpus", "import", w, n][..], &MODEL].concat());
|
||||
let bad = import(&w, &n);
|
||||
assert!(!bad.status.success());
|
||||
assert_eq!(out(&weksa(d.path(), &["corpus", "dump"])).trim(), "");
|
||||
assert!(!d.path().join("e.cc").exists() || EntityStore::open(&d.path().join("e.cc")).unwrap().all().unwrap().is_empty());
|
||||
// The redrawn answer, different on line 1 too, imports.
|
||||
let (w, n) = lantern(d.path(), &lines, &["Tamsin, bring the lantern.", "[[entity:tamsin|Tamsin]]'s lantern, said Orsa."]);
|
||||
let good = import(&w, &n);
|
||||
assert!(good.status.success(), "{}", out(&good));
|
||||
assert!(out(&good).contains("Added item:lantern-wager:2"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_work_repeating_a_line_is_refused_naming_both_lines() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
let (w, n) = lantern(d.path(), &["No, my lord.", "No, my lord!", "Quiet, both of you."], &["No, my lord.", "No, my lord!", "Quiet, both of you."]);
|
||||
let o = weksa(d.path(), &[&["corpus", "import", &w, &n][..], &MODEL].concat());
|
||||
assert!(!o.status.success());
|
||||
let err = String::from_utf8_lossy(&o.stderr).to_string();
|
||||
assert!(err.contains("item:lantern-wager:2") && err.contains("item:lantern-wager:1"), "{err}");
|
||||
assert_eq!(out(&weksa(d.path(), &["corpus", "dump"])).trim(), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_annotator_flags_decide_the_label() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
let (w, n) = lantern(d.path(), &["Quiet, both of you."], &["Quiet, both of you."]);
|
||||
let none = weksa(d.path(), &["corpus", "import", &w, &n]);
|
||||
assert!(!none.status.success());
|
||||
let blank = weksa(d.path(), &["corpus", "import", &w, &n, "--model-family", "anthropic", "--generator", " "]);
|
||||
assert!(!blank.status.success());
|
||||
assert_eq!(out(&weksa(d.path(), &["corpus", "dump"])).trim(), "");
|
||||
let op = weksa(d.path(), &["corpus", "import", &w, &n, "--operator"]);
|
||||
assert!(op.status.success(), "{}", out(&op));
|
||||
let dump = out(&weksa(d.path(), &["corpus", "dump"]));
|
||||
assert!(dump.contains("by Some(Operator)"), "{dump}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_import_refused_on_a_cast_entity_leaves_both_stores_untouched() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
// entity:tamsin is already held by a cast origin, so the work's PublicDomain
|
||||
// claim on it (cast position 2, after orsa) is refused by EntityStore::put.
|
||||
cast_entities(d.path(), &["tamsin"]);
|
||||
let (w, n) = lantern(d.path(), &["Quiet, both of you."], &["Quiet, both of you."]);
|
||||
let before = std::fs::read(d.path().join("e.cc")).unwrap();
|
||||
let o = weksa(d.path(), &[&["corpus", "import", &w, &n][..], &MODEL].concat());
|
||||
assert!(!o.status.success());
|
||||
assert!(String::from_utf8_lossy(&o.stderr).contains("a cast entity"), "{}", String::from_utf8_lossy(&o.stderr));
|
||||
assert_eq!(std::fs::read(d.path().join("e.cc")).unwrap(), before);
|
||||
assert!(!d.path().join("c.cc").exists() || CorpusStore::open(&d.path().join("c.cc")).unwrap().all().unwrap().is_empty());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -205,6 +205,7 @@ pub fn item_in(key: &str, text: &str, group: &str, source: Source) -> CorpusItem
|
|||
language: "en-US".into(),
|
||||
speaker: eid("nibu"),
|
||||
names: vec![],
|
||||
names_by: None,
|
||||
group: group.into(),
|
||||
prior_turns: vec![],
|
||||
source,
|
||||
|
|
|
|||
|
|
@ -593,6 +593,41 @@ mod lowering {
|
|||
assert!(Prompt::lower(&read, &item, &away(), "en-US", &inv, &held, &empty).is_ok());
|
||||
}
|
||||
|
||||
/// The packet the encoder emits for a request: frame:3, no consent, source
|
||||
/// ModelDecomposition. Admission refuses it, so no lowering view exists.
|
||||
#[test]
|
||||
fn an_encoder_request_never_reaches_a_lowerer() {
|
||||
let cast = InventoryStore::open(&data("interlingua/inventory.cc")).unwrap();
|
||||
let entities = EntityStore::open(&data("entity/entity.cc")).unwrap();
|
||||
let corpus = CorpusStore::open(&data("corpus/corpus.cc")).unwrap();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut store = PacketStore::open(&dir.path().join("p.cc")).unwrap();
|
||||
let mut packet = proof_packet();
|
||||
packet.provenance.source = weksa::packet::ProvenanceSource::ModelDecomposition;
|
||||
packet.predications[0].frame = weksa::InvId::parse("frame:3").unwrap();
|
||||
match store.admit(packet.clone(), &cast, &corpus, &entities) {
|
||||
Err(weksa::StoreError::Invalid(r)) => assert_eq!(
|
||||
r,
|
||||
vec![weksa::Refusal::V18RequestWithoutConsent {
|
||||
predication: weksa::packet::NodeId::parse("n4").unwrap()
|
||||
}]
|
||||
),
|
||||
other => panic!("expected V18, got {:?}", other.map(|_| ())),
|
||||
}
|
||||
// Not blanket: telemetry that records the consent admits and lowers.
|
||||
packet.provenance.source = weksa::packet::ProvenanceSource::GameTelemetry;
|
||||
packet.discourse.addressee_consent = Some(weksa::packet::NodeId::parse("n4").unwrap());
|
||||
drop(store);
|
||||
let view = admitted(packet, &cast, "corpus/corpus.cc", &entities);
|
||||
let item = CorpusItemId::parse("item:nibu:1").unwrap();
|
||||
let held = Held(vec![nibu(&[("en-US", &["Nibu"])])]);
|
||||
let empty = weksa::Ontology::empty();
|
||||
assert!(
|
||||
Prompt::lower(&view, &item, &away(), "en-US", &cast.inventory().unwrap(), &held, &empty)
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
|
||||
fn dao(index: usize) -> (LoweringInput, Inventory, EntityStore, CorpusItemId) {
|
||||
let inv = InventoryStore::open(&data("daodejing/inventory.cc")).unwrap();
|
||||
let entities = EntityStore::open(&data("daodejing/entity.cc")).unwrap();
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
//! The Dao De Jing test bed (`data/daodejing`): its Legge items carry full
|
||||
//! public-domain provenance, and its packets are admitted and lower to the
|
||||
//! English the first golf-loop run scored (`docs/targets/dao-de-jing-test-bed.md`).
|
||||
//! public-domain provenance, and its packets are admitted
|
||||
//! (`docs/targets/dao-de-jing-test-bed.md`).
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use weksa::daodejing::packets;
|
||||
use weksa::{CorpusStore, EntityStore, InventoryStore, PacketStore, Source, lower};
|
||||
use weksa::{CorpusStore, EntityStore, InventoryStore, PacketStore, Source};
|
||||
|
||||
fn root() -> std::path::PathBuf {
|
||||
Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/daodejing")
|
||||
|
|
@ -39,30 +39,15 @@ fn every_legge_item_names_licence_jurisdiction_translator_and_edition_year() {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn the_slice_packets_are_admitted_and_lower_to_the_scored_english() {
|
||||
fn the_slice_packets_are_admitted_and_the_committed_store_holds_them() {
|
||||
let inventory = InventoryStore::open(&root().join("inventory.cc")).unwrap();
|
||||
let corpus = CorpusStore::open(&root().join("corpus.cc")).unwrap();
|
||||
let entities = EntityStore::open(&root().join("entity.cc")).unwrap();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut store = PacketStore::open(&dir.path().join("p.cc")).unwrap();
|
||||
let inv = inventory.inventory().unwrap();
|
||||
let lowered: Vec<String> = packets()
|
||||
.into_iter()
|
||||
.map(|p| {
|
||||
let view = store.admit(p, &inventory, &corpus, &entities).unwrap();
|
||||
lower(&view, &inv, &entities).unwrap()
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(
|
||||
lowered,
|
||||
[
|
||||
"The sage empties the minds. The sage fills the bellies. The sage weakens the wills. The sage strengthens the bones.",
|
||||
"The heaven is long-enduring. The earth continues long.",
|
||||
"The highest excellence is like the water.",
|
||||
"The words that are strictly true seem paradoxical.",
|
||||
"The space between heaven and earth may be compared to the bellows.",
|
||||
]
|
||||
);
|
||||
for p in packets() {
|
||||
store.admit(p, &inventory, &corpus, &entities).unwrap();
|
||||
}
|
||||
// The committed packet store holds exactly what the module builds.
|
||||
let stored = PacketStore::open(&root().join("packets.cc")).unwrap();
|
||||
for p in packets() {
|
||||
|
|
|
|||
|
|
@ -33,20 +33,6 @@ fn assert_clean_and_distinct<T: std::fmt::Display + std::fmt::Debug>(all: Vec<T>
|
|||
#[test]
|
||||
fn lower_error_display_names_the_variant_and_no_identifier() {
|
||||
assert_clean_and_distinct(vec![
|
||||
LowerError::Unrendered { what: SENTINEL },
|
||||
LowerError::NoRow { id: inv() },
|
||||
LowerError::GlossMismatch {
|
||||
id: inv(),
|
||||
row: SENTINEL,
|
||||
},
|
||||
LowerError::UnknownRole {
|
||||
frame: inv(),
|
||||
role: SENTINEL.into(),
|
||||
},
|
||||
LowerError::MissingRole {
|
||||
predication: node(),
|
||||
role: SENTINEL.into(),
|
||||
},
|
||||
LowerError::UnknownEntity { entity: entity() },
|
||||
LowerError::NoForm { entity: entity() },
|
||||
LowerError::NoPronunciation { entity: entity() },
|
||||
|
|
@ -60,12 +46,6 @@ fn lower_error_display_names_the_variant_and_no_identifier() {
|
|||
entity: entity(),
|
||||
at: 4242,
|
||||
},
|
||||
LowerError::ConsentMissing {
|
||||
predication: node(),
|
||||
},
|
||||
LowerError::AskerNotAddressee {
|
||||
predication: node(),
|
||||
},
|
||||
LowerError::EntityRead {
|
||||
entity: entity(),
|
||||
why: SENTINEL.into(),
|
||||
|
|
@ -124,6 +104,7 @@ fn refusal_display_names_the_rule_and_no_identifier() {
|
|||
},
|
||||
Refusal::V16SpeakerNotInScene,
|
||||
Refusal::V17AddresseeNotAReferent { id: node() },
|
||||
Refusal::V18RequestWithoutConsent { predication: node() },
|
||||
];
|
||||
assert_eq!(
|
||||
Refusal::V16SpeakerNotInScene.to_string(),
|
||||
|
|
|
|||
|
|
@ -502,3 +502,96 @@ fn a_malformed_pronunciation_is_refused() {
|
|||
}
|
||||
assert_eq!(store.get(&eid("kay")).unwrap(), None);
|
||||
}
|
||||
|
||||
const LANTERN: &str = r#"{"slug":"lantern-wager","language":"en-US","admitted_on":"2026-11-13",
|
||||
"source":{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}},
|
||||
"cast":[{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:tamsin","forms":["Tamsin","Tam"]}],
|
||||
"lines":[{"group":"s1","speaker":"entity:orsa","text":"Tamsin, bring the lantern."},
|
||||
{"group":"s1","speaker":"entity:tamsin","text":"Tamsin's lantern, said Orsa."},
|
||||
{"group":"s2","speaker":"entity:orsa","text":"Quiet, both of you."}]}"#;
|
||||
|
||||
fn marked(line: u32, text: &str) -> weksa::encode::MarkedLine {
|
||||
weksa::encode::MarkedLine { line, marked: text.into() }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_imported_work_passes_the_store_doors() {
|
||||
use weksa::Annotator;
|
||||
use weksa::corpus::ModelFamily;
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
let mut entities = EntityStore::open(&d.path().join("e.cc")).unwrap();
|
||||
let mut corpus = CorpusStore::open(&d.path().join("c.cc")).unwrap();
|
||||
let model = Annotator::Model { family: ModelFamily::Anthropic, generator: "g".into() };
|
||||
let work = || weksa::encode::WorkDraft::parse(LANTERN).unwrap();
|
||||
let (cast, items) = work()
|
||||
.into_import(&[marked(1, "[[entity:tamsin|Tamsin]], bring the lantern.")], &model)
|
||||
.unwrap();
|
||||
for e in cast {
|
||||
assert_eq!(entities.put(e).unwrap(), AddOutcome::Added);
|
||||
}
|
||||
for i in items.clone() {
|
||||
assert_eq!(corpus.add(i, &entities).unwrap(), AddOutcome::Added);
|
||||
}
|
||||
drop(corpus);
|
||||
let mut corpus = CorpusStore::open(&d.path().join("c.cc")).unwrap();
|
||||
let stored = corpus.get("item:lantern-wager:1").unwrap().unwrap();
|
||||
assert_eq!(stored.names_by, Some(model.clone()));
|
||||
assert_eq!(stored.names.len(), 1);
|
||||
assert_eq!(corpus.add(items[0].clone(), &entities).unwrap(), AddOutcome::Unchanged);
|
||||
|
||||
// A form the entity does not list is the store's refusal, not the importer's.
|
||||
let (_, bad) = work()
|
||||
.into_import(&[marked(2, "[[entity:tamsin|Tamsin's]] lantern, said Orsa.")], &model)
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
refusal(corpus.add(bad[0].clone(), &entities)),
|
||||
StoreRefusal::NameSpanNotAForm { .. }
|
||||
));
|
||||
|
||||
// An operator run over another line labels its own spans.
|
||||
let (_, op) = work()
|
||||
.into_import(&[marked(3, "Quiet, both of you.")], &Annotator::Operator)
|
||||
.unwrap();
|
||||
assert_eq!(corpus.add(op[0].clone(), &entities).unwrap(), AddOutcome::Added);
|
||||
assert_eq!(
|
||||
corpus.get("item:lantern-wager:3").unwrap().unwrap().names_by,
|
||||
Some(Annotator::Operator)
|
||||
);
|
||||
// The first run's line keeps its label.
|
||||
assert_eq!(corpus.get("item:lantern-wager:1").unwrap().unwrap().names_by, Some(model));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn committed_items_read_with_no_names_label() {
|
||||
let data = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data");
|
||||
for sub in ["daodejing", "original"] {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
let copy = d.path().join("c.cc");
|
||||
std::fs::copy(data.join(sub).join("corpus.cc"), ©).unwrap();
|
||||
let items = CorpusStore::open(©).unwrap().all().unwrap();
|
||||
assert!(!items.is_empty(), "{sub}");
|
||||
assert!(items.iter().all(|i| i.names_by.is_none()), "{sub}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_blank_names_by_generator_is_refused_by_the_corpus_door() {
|
||||
use weksa::Annotator;
|
||||
use weksa::corpus::ModelFamily;
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
let mut entities = EntityStore::open(&d.path().join("e.cc")).unwrap();
|
||||
let mut corpus = CorpusStore::open(&d.path().join("c.cc")).unwrap();
|
||||
let blank = Annotator::Model { family: ModelFamily::Anthropic, generator: " ".into() };
|
||||
let (cast, items) = weksa::encode::WorkDraft::parse(LANTERN)
|
||||
.unwrap()
|
||||
.into_import(&[marked(3, "Quiet, both of you.")], &blank)
|
||||
.unwrap();
|
||||
for e in cast {
|
||||
entities.put(e).unwrap();
|
||||
}
|
||||
assert!(matches!(
|
||||
refusal(corpus.add(items[0].clone(), &entities)),
|
||||
StoreRefusal::Malformed { .. }
|
||||
));
|
||||
assert!(corpus.all().unwrap().is_empty());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2742,15 +2742,6 @@ fn source_tripwires() {
|
|||
.collect();
|
||||
let count = |needle: &str| code.iter().filter(|l| l.contains(needle)).count();
|
||||
assert!(!run.contains("pub fn new"));
|
||||
// The round does not call the template table: the model lowers.
|
||||
assert_eq!(
|
||||
code.iter()
|
||||
.filter(|l| l.contains("lower(&") && !l.contains("Prompt::lower("))
|
||||
.count(),
|
||||
0,
|
||||
"run.rs calls the table"
|
||||
);
|
||||
assert_eq!(count("crate::lower::lower"), 0, "run.rs imports the table");
|
||||
assert_eq!(
|
||||
count("Prompt::lower("),
|
||||
2,
|
||||
|
|
|
|||
|
|
@ -1,22 +1,16 @@
|
|||
//! The English lowerer against the committed inventory (revision 2) and the
|
||||
//! `nibu-proof-001` packet, and the rules that keep a table row from speaking
|
||||
//! for a meaning it does not carry.
|
||||
//! The entity port, `pronounce` and `phones` against the committed stores and
|
||||
//! the `nibu-proof-001` packet.
|
||||
|
||||
mod common;
|
||||
use std::path::Path;
|
||||
|
||||
use common::{iv, nd};
|
||||
use weksa::entity::{Entity, EntityOrigin, LanguageForms};
|
||||
use weksa::inventory::proof_revision;
|
||||
use weksa::packet::{EntityId, FeatValue, FeatureValue, Packet, proof_packet};
|
||||
use weksa::packet::{EntityId, Packet, proof_packet};
|
||||
use weksa::{
|
||||
CorpusStore, EntityForms, EntityStore, Inventory, InventoryStore, LowerError, LoweringInput,
|
||||
PacketStore, PhoneInventory, PhoneLine, Pronunciation, Stress, lower, pronounce, validate,
|
||||
PacketStore, PhoneInventory, PhoneLine, Pronunciation, Stress, pronounce,
|
||||
};
|
||||
|
||||
const PROOF: &str = "Warning: you may endanger yourself by touching the control. \
|
||||
Nibu will stop that.";
|
||||
|
||||
fn committed() -> (Inventory, tempfile::TempDir) {
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
|
||||
let inv = InventoryStore::open(&root.join("data/interlingua/inventory.cc")).unwrap();
|
||||
|
|
@ -48,181 +42,6 @@ fn committed_entities() -> EntityStore {
|
|||
EntityStore::open(&root.join("data/entity/entity.cc")).unwrap()
|
||||
}
|
||||
|
||||
fn lowered(packet: &Packet, inv: &Inventory) -> Result<String, LowerError> {
|
||||
lower(&view(packet, inv), inv, &committed_entities())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_proof_packet_is_admitted_by_the_store_and_lowers_to_its_sentences() {
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
|
||||
let (inv, dir) = committed();
|
||||
let inventory = InventoryStore::open(&root.join("data/interlingua/inventory.cc")).unwrap();
|
||||
let corpus = CorpusStore::open(&root.join("data/corpus/corpus.cc")).unwrap();
|
||||
let entities = EntityStore::open(&root.join("data/entity/entity.cc")).unwrap();
|
||||
let mut packets = PacketStore::open(&dir.path().join("p.cc")).unwrap();
|
||||
packets
|
||||
.admit(proof_packet(), &inventory, &corpus, &entities)
|
||||
.unwrap();
|
||||
let stored = packets.get("packet:6e6962752d303031").unwrap().unwrap();
|
||||
assert_eq!(lowered(&stored, &inv).unwrap(), PROOF);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_the_addressee_is_you_and_is_reflexive() {
|
||||
let (inv, _dir) = committed();
|
||||
let mut p = proof_packet();
|
||||
p.discourse.addressee = None;
|
||||
assert_eq!(
|
||||
lowered(&p, &inv).unwrap(),
|
||||
"Warning: the player may endanger the player by touching the control. \
|
||||
Nibu will stop that."
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_speech_act_means_no_opener_and_a_capital() {
|
||||
let (inv, _dir) = committed();
|
||||
let mut p = proof_packet();
|
||||
p.discourse.speech_act = None;
|
||||
assert!(lowered(&p, &inv).unwrap().starts_with("You may endanger"));
|
||||
p.discourse.speech_act = Some(iv("speech_act:1"));
|
||||
assert!(lowered(&p, &inv).unwrap().starts_with("You may endanger"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_packet_carrying_what_the_lowerer_does_not_render_is_refused() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut store = InventoryStore::open(&dir.path().join("i.cc")).unwrap();
|
||||
let (rev1, entries) = weksa::inventory::seed();
|
||||
store.admit_revision(rev1, entries).unwrap();
|
||||
let (rev2, entries) = proof_revision();
|
||||
store.admit_revision(rev2, entries).unwrap();
|
||||
let mut ext = common::entry("extension:1", 3);
|
||||
ext.value_kind = Some(weksa::ValueKind::Grade);
|
||||
common::admit(&mut store, 3, vec![ext]);
|
||||
let inv = store.inventory().unwrap();
|
||||
let discourse = |p: &mut Packet| {
|
||||
p.discourse.features.push(FeatureValue {
|
||||
feature: iv("feature:6"),
|
||||
value: FeatValue::Grade(common::grade(300)),
|
||||
})
|
||||
};
|
||||
let predication = |p: &mut Packet| {
|
||||
p.predications[0].features.push(FeatureValue {
|
||||
feature: iv("feature:1"),
|
||||
value: FeatValue::Option(common::lab("negative")),
|
||||
})
|
||||
};
|
||||
let relation = |p: &mut Packet| {
|
||||
p.relations.push(weksa::packet::Relation {
|
||||
relation: iv("relation:8"),
|
||||
from: nd("n4"),
|
||||
to: nd("n5"),
|
||||
features: vec![],
|
||||
})
|
||||
};
|
||||
let extension = |p: &mut Packet| {
|
||||
p.inventory_rev = 3;
|
||||
p.extensions.push(weksa::packet::Extension {
|
||||
key: iv("extension:1"),
|
||||
value: FeatValue::Grade(common::grade(3)),
|
||||
})
|
||||
};
|
||||
let referent_feature = |p: &mut Packet| {
|
||||
p.referents[0].features.push(FeatureValue {
|
||||
feature: iv("feature:6"),
|
||||
value: FeatValue::Grade(common::grade(1)),
|
||||
})
|
||||
};
|
||||
let no_predication = |p: &mut Packet| p.predications.clear();
|
||||
let extra_role = |p: &mut Packet| {
|
||||
let filler = p.predications[0].roles[0].filler.clone();
|
||||
p.predications[0].roles.push(weksa::packet::RoleFill {
|
||||
role: iv("role:3"),
|
||||
filler,
|
||||
})
|
||||
};
|
||||
let consent = |p: &mut Packet| p.discourse.addressee_consent = Some(nd("n5"));
|
||||
let lone_referent = |p: &mut Packet| {
|
||||
let mut r = p.referents[0].clone();
|
||||
r.id = nd("n99");
|
||||
p.referents.push(r)
|
||||
};
|
||||
for (what, edit) in [
|
||||
("features", &discourse as &dyn Fn(&mut Packet)),
|
||||
("features", &predication),
|
||||
("addressee consent", &consent),
|
||||
("relations", &relation),
|
||||
("extensions", &extension),
|
||||
("no predication", &no_predication),
|
||||
("a role the frame does not name", &extra_role),
|
||||
("a referent no role mentions", &lone_referent),
|
||||
] {
|
||||
let mut p = proof_packet();
|
||||
edit(&mut p);
|
||||
assert_eq!(
|
||||
lower(&admitted(&p, &store), &inv, &committed_entities()),
|
||||
Err(LowerError::Unrendered { what }),
|
||||
"{what}"
|
||||
);
|
||||
}
|
||||
// The seed lets no feature sit on a referent, so that arm is refused by validate.
|
||||
let mut p = proof_packet();
|
||||
referent_feature(&mut p);
|
||||
assert!(validate(&p, &inv).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unknown_row_and_an_unfilled_role_are_refused_by_name() {
|
||||
let (inv, _dir) = committed();
|
||||
let mut p = proof_packet();
|
||||
p.discourse.speech_act = Some(iv("speech_act:3"));
|
||||
assert_eq!(
|
||||
lowered(&p, &inv),
|
||||
Err(LowerError::NoRow {
|
||||
id: iv("speech_act:3")
|
||||
})
|
||||
);
|
||||
let mut p = proof_packet();
|
||||
p.predications[0].roles.pop();
|
||||
assert_eq!(
|
||||
lowered(&p, &inv),
|
||||
Err(LowerError::MissingRole {
|
||||
predication: nd("n4"),
|
||||
role: "patient".into()
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_row_cannot_speak_for_an_id_whose_gloss_differs() {
|
||||
// Revision 2 with concept:4 glossed "hatch" in place of "control": the row
|
||||
// for concept:4 says "control" and must not render it.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut store = InventoryStore::open(&dir.path().join("i.cc")).unwrap();
|
||||
let (rev1, entries) = weksa::inventory::seed();
|
||||
store.admit_revision(rev1, entries).unwrap();
|
||||
let (rev2, mut entries) = proof_revision();
|
||||
for e in &mut entries {
|
||||
if e.id == iv("concept:4") {
|
||||
e.gloss = "hatch".into();
|
||||
}
|
||||
}
|
||||
store.admit_revision(rev2, entries).unwrap();
|
||||
let inv = store.inventory().unwrap();
|
||||
assert_eq!(
|
||||
lower(
|
||||
&admitted(&proof_packet(), &store),
|
||||
&inv,
|
||||
&committed_entities()
|
||||
),
|
||||
Err(LowerError::GlossMismatch {
|
||||
id: iv("concept:4"),
|
||||
row: "control"
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
/// An entity port a test controls: ids to entities, and an optional read fault.
|
||||
#[derive(Default)]
|
||||
struct Entities {
|
||||
|
|
@ -257,109 +76,6 @@ fn nibu(forms: &[(&str, &[&str])]) -> Entity {
|
|||
}
|
||||
}
|
||||
|
||||
fn through(entities: &Entities) -> Result<String, LowerError> {
|
||||
let (inv, _dir) = committed();
|
||||
lower(&view(&proof_packet(), &inv), &inv, entities)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_name_survives_lowering_verbatim_where_the_concept_word_was() {
|
||||
// The entity's own form, not "the ship mind", and not re-cased or inflected:
|
||||
// the form is lowercase and stays so mid-sentence.
|
||||
let mut e = Entities::default();
|
||||
let n = nibu(&[("en-US", &["nibu-of-the-deep"])]);
|
||||
e.held.push(n);
|
||||
// It opens the second sentence, where a template word would be capitalised.
|
||||
assert_eq!(
|
||||
through(&e).unwrap(),
|
||||
"Warning: you may endanger yourself by touching the control. \
|
||||
nibu-of-the-deep will stop that."
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_the_target_languages_citation_form_is_chosen() {
|
||||
// Forms after the first are inflected shapes no English frame asks for; a
|
||||
// form under another language tag is never read, whatever order it is in.
|
||||
let mut e = Entities::default();
|
||||
let n = nibu(&[
|
||||
("sa-Latn", &["Nibuwa"]),
|
||||
("en-US", &["Nibu", "Nibu's", "Nibus"]),
|
||||
]);
|
||||
e.held.push(n);
|
||||
assert_eq!(through(&e).unwrap(), PROOF);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_entity_the_port_does_not_hold_is_refused_not_replaced_by_its_concept_word() {
|
||||
let id = EntityId::parse("entity:nibu").unwrap();
|
||||
assert_eq!(
|
||||
through(&Entities::default()),
|
||||
Err(LowerError::UnknownEntity { entity: id })
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_entity_with_no_target_language_form_is_refused() {
|
||||
let mut e = Entities::default();
|
||||
let n = nibu(&[("sa-Latn", &["Nibuwa"])]);
|
||||
let id = n.id.clone();
|
||||
e.held.push(n);
|
||||
assert_eq!(through(&e), Err(LowerError::NoForm { entity: id }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failing_entity_port_is_refused_by_name() {
|
||||
let e = Entities {
|
||||
fault: true,
|
||||
..Entities::default()
|
||||
};
|
||||
assert_eq!(
|
||||
through(&e),
|
||||
Err(LowerError::EntityRead {
|
||||
entity: EntityId::parse("entity:nibu").unwrap(),
|
||||
why: "store unreadable".into()
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_addressee_entity_still_lowers_to_you_and_is_never_looked_up() {
|
||||
// Naming the addressee would break "you"/"yourself": the port is not asked,
|
||||
// so a port that faults on every read changes nothing.
|
||||
let (inv, _dir) = committed();
|
||||
let mut p = proof_packet();
|
||||
p.referents[0].entity = Some(EntityId::parse("entity:nibu").unwrap());
|
||||
p.referents[2].entity = None;
|
||||
let faulting = Entities {
|
||||
fault: true,
|
||||
..Entities::default()
|
||||
};
|
||||
assert_eq!(
|
||||
lower(&view(&p, &inv), &inv, &faulting).unwrap(),
|
||||
"Warning: you may endanger yourself by touching the control. \
|
||||
The ship mind will stop that."
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_purpose_the_frame_does_not_say_is_refused_not_dropped() {
|
||||
// The intervention row says "will stop"; a packet that also fills the
|
||||
// beneficiary role would assert a purpose the text does not carry.
|
||||
let (inv, _dir) = committed();
|
||||
let mut p = proof_packet();
|
||||
p.predications[1].roles.push(weksa::packet::RoleFill {
|
||||
role: iv("role:8"),
|
||||
filler: nd("n1"),
|
||||
});
|
||||
assert_eq!(
|
||||
lowered(&p, &inv),
|
||||
Err(LowerError::Unrendered {
|
||||
what: "a role the frame does not name"
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
fn ruled(ipa: &str, rejected: &[&str]) -> Pronunciation {
|
||||
Pronunciation {
|
||||
ipa: ipa.into(),
|
||||
|
|
@ -375,6 +91,21 @@ fn say(entities: &dyn EntityForms) -> Result<Vec<String>, LowerError> {
|
|||
pronounce(&view(&proof_packet(), &inv), entities)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failing_entity_port_is_refused_by_name() {
|
||||
let e = Entities {
|
||||
fault: true,
|
||||
..Entities::default()
|
||||
};
|
||||
assert_eq!(
|
||||
say(&e),
|
||||
Err(LowerError::EntityRead {
|
||||
entity: EntityId::parse("entity:nibu").unwrap(),
|
||||
why: "store unreadable".into()
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nibu_round_trips_from_the_committed_seed_to_its_ruled_ipa() {
|
||||
let seed = committed_entities();
|
||||
|
|
@ -573,160 +304,3 @@ fn the_inventory_parser_refuses_what_it_cannot_read() {
|
|||
assert!(PhoneInventory::parse("# c\n\ni\tvowel\ttense\ta\n").is_ok());
|
||||
assert!(PhoneInventory::for_language("sa-Latn").is_none());
|
||||
}
|
||||
|
||||
/// `nibu-proof-001` re-cut as the request: the addressee asked Nibu to stop,
|
||||
/// one frame:3 predication (n4), consent naming it, and no other claim.
|
||||
fn asked() -> Packet {
|
||||
let mut p = proof_packet();
|
||||
p.referents.remove(1);
|
||||
let mut ask = p.predications[1].clone();
|
||||
ask.id = nd("n4");
|
||||
ask.concept = iv("concept:6");
|
||||
ask.frame = iv("frame:3");
|
||||
ask.roles = vec![
|
||||
weksa::packet::RoleFill {
|
||||
role: iv("role:1"),
|
||||
filler: nd("n1"),
|
||||
},
|
||||
weksa::packet::RoleFill {
|
||||
role: iv("role:2"),
|
||||
filler: nd("n3"),
|
||||
},
|
||||
];
|
||||
p.predications = vec![ask];
|
||||
p.discourse.speech_act = Some(iv("speech_act:1"));
|
||||
p.discourse.addressee_consent = Some(nd("n4"));
|
||||
p.constraints.preserve = vec![nd("n1")];
|
||||
p
|
||||
}
|
||||
|
||||
/// The request plus a frame:2 predication (n5) the consent could wrongly name.
|
||||
fn asked_and_stopped() -> Packet {
|
||||
let mut p = asked();
|
||||
let mut stop = proof_packet().predications.remove(1);
|
||||
stop.roles[1].filler = nd("n4");
|
||||
p.predications.push(stop);
|
||||
p
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_consented_request_lowers_to_what_the_addressee_asked() {
|
||||
let (inv, _dir) = committed();
|
||||
assert_eq!(lowered(&asked(), &inv).unwrap(), "You asked Nibu to stop.");
|
||||
let mut p = asked();
|
||||
p.provenance.source = weksa::packet::ProvenanceSource::GameTelemetry;
|
||||
assert_eq!(lowered(&p, &inv).unwrap(), "You asked Nibu to stop.");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_request_without_its_consent_is_refused_not_invented() {
|
||||
let (inv, _dir) = committed();
|
||||
let mut p = asked();
|
||||
p.discourse.addressee_consent = None;
|
||||
assert_eq!(
|
||||
lowered(&p, &inv),
|
||||
Err(LowerError::ConsentMissing {
|
||||
predication: nd("n4")
|
||||
})
|
||||
);
|
||||
// Consent that names a different predication is no consent to the request.
|
||||
let mut p = asked_and_stopped();
|
||||
p.discourse.addressee_consent = Some(nd("n5"));
|
||||
assert_eq!(
|
||||
lowered(&p, &inv),
|
||||
Err(LowerError::ConsentMissing {
|
||||
predication: nd("n4")
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn consent_to_one_of_two_requests_leaves_the_other_unconsented() {
|
||||
let (inv, _dir) = committed();
|
||||
let mut p = asked();
|
||||
let mut second = p.predications[0].clone();
|
||||
second.id = nd("n5");
|
||||
p.predications.push(second);
|
||||
assert_eq!(
|
||||
lowered(&p, &inv),
|
||||
Err(LowerError::ConsentMissing {
|
||||
predication: nd("n5")
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_asker_is_the_role_the_inventory_glosses_agent_not_the_first_role() {
|
||||
// role:1 and role:2 swap glosses: "agent" is now role:2, so the asker of
|
||||
// `asked()` (role:1 filled by the addressee) is the one role:2 names, Nibu.
|
||||
// A lookup that took the first role, or the first live entry, would read the
|
||||
// addressee and let the request through.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut store = InventoryStore::open(&dir.path().join("i.cc")).unwrap();
|
||||
let (rev1, mut entries) = weksa::inventory::seed();
|
||||
for e in &mut entries {
|
||||
if e.id == iv("role:1") {
|
||||
e.gloss = "patient".into();
|
||||
} else if e.id == iv("role:2") {
|
||||
e.gloss = "agent".into();
|
||||
}
|
||||
}
|
||||
store.admit_revision(rev1, entries).unwrap();
|
||||
let (rev2, entries) = proof_revision();
|
||||
store.admit_revision(rev2, entries).unwrap();
|
||||
let inv = store.inventory().unwrap();
|
||||
assert_eq!(
|
||||
lower(&admitted(&asked(), &store), &inv, &committed_entities()),
|
||||
Err(LowerError::AskerNotAddressee {
|
||||
predication: nd("n4")
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_request_by_someone_other_than_the_addressee_is_refused() {
|
||||
let (inv, _dir) = committed();
|
||||
let mut p = asked();
|
||||
p.predications[0].roles[0].filler = nd("n3");
|
||||
p.predications[0].roles[1].filler = nd("n1");
|
||||
assert_eq!(
|
||||
lowered(&p, &inv),
|
||||
Err(LowerError::AskerNotAddressee {
|
||||
predication: nd("n4")
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn consent_on_a_claim_that_is_no_request_stays_unrendered() {
|
||||
let (inv, _dir) = committed();
|
||||
let mut p = proof_packet();
|
||||
p.discourse.addressee_consent = Some(nd("n5"));
|
||||
assert_eq!(
|
||||
lowered(&p, &inv),
|
||||
Err(LowerError::Unrendered {
|
||||
what: "addressee consent"
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_decomposition_or_import_cannot_reach_the_lowerer_with_consent() {
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
|
||||
let inventory = InventoryStore::open(&root.join("data/interlingua/inventory.cc")).unwrap();
|
||||
let corpus = CorpusStore::open(&root.join("data/corpus/corpus.cc")).unwrap();
|
||||
let entities = EntityStore::open(&root.join("data/entity/entity.cc")).unwrap();
|
||||
for source in [
|
||||
weksa::packet::ProvenanceSource::ModelDecomposition,
|
||||
weksa::packet::ProvenanceSource::ImportedCorpus,
|
||||
] {
|
||||
let mut p = asked();
|
||||
p.provenance.source = source;
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let refused = PacketStore::open(&dir.path().join("p.cc"))
|
||||
.unwrap()
|
||||
.admit(p, &inventory, &corpus, &entities)
|
||||
.unwrap_err();
|
||||
assert!(format!("{refused:?}").contains("V15"), "{source:?}");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
25
crates/weksa/tests/mutants_a.rs
Normal file
25
crates/weksa/tests/mutants_a.rs
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
//! Pins for behaviour of the small and middle weksa files that mutation testing
|
||||
//! found unpinned. Each test states the behaviour in domain terms.
|
||||
|
||||
use weksa::form::is_calendar_date;
|
||||
|
||||
/// A calendar date is exactly `YYYY-MM-DD`: both hyphens sit where the format
|
||||
/// puts them, so a ten-byte string with a digit in the second hyphen's place is
|
||||
/// no date.
|
||||
#[test]
|
||||
fn a_calendar_date_needs_both_hyphens_in_place() {
|
||||
assert!(is_calendar_date("2026-01-01"));
|
||||
assert!(!is_calendar_date("2026-01201"));
|
||||
assert!(!is_calendar_date("2026101-01"));
|
||||
}
|
||||
|
||||
/// February has 29 days in a Gregorian leap year and 28 otherwise.
|
||||
#[test]
|
||||
fn february_has_29_days_only_in_gregorian_leap_years() {
|
||||
assert!(is_calendar_date("2024-02-29"));
|
||||
assert!(is_calendar_date("2000-02-29"));
|
||||
assert!(!is_calendar_date("1900-02-29"));
|
||||
assert!(!is_calendar_date("2023-02-29"));
|
||||
assert!(is_calendar_date("2023-02-28"));
|
||||
assert!(!is_calendar_date("2024-02-30"));
|
||||
}
|
||||
297
crates/weksa/tests/mutants_b.rs
Normal file
297
crates/weksa/tests/mutants_b.rs
Normal file
|
|
@ -0,0 +1,297 @@
|
|||
//! Pins found by mutation testing store.rs, inventory.rs and main.rs. Each test
|
||||
//! states what the production code must do, in domain terms.
|
||||
mod common;
|
||||
use common::*;
|
||||
use std::process::{Command, Output};
|
||||
use weksa::packet::Grade;
|
||||
use weksa::store::EntityBound;
|
||||
use weksa::{
|
||||
Annotator, Condition, EntityStore, Layer, OntologyEntry, OntologyStore, Pronunciation, Refusal,
|
||||
Slot, StoreError, StoreRefusal, Test,
|
||||
};
|
||||
|
||||
fn refusal<T: std::fmt::Debug>(r: Result<T, StoreError>) -> StoreRefusal {
|
||||
match r {
|
||||
Err(StoreError::Refused(r)) => r,
|
||||
other => panic!("expected a store refusal, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn kay_saying(p: Pronunciation) -> weksa::Entity {
|
||||
let mut e = ent("kay", "en-US", &["Kay"]);
|
||||
e.forms[0].pronunciation = Some(p);
|
||||
e
|
||||
}
|
||||
|
||||
fn ruling(ipa: &str, rejected: Vec<String>) -> Pronunciation {
|
||||
Pronunciation {
|
||||
ipa: ipa.into(),
|
||||
ruled_by: "Kay".into(),
|
||||
date: "2026-11-07".into(),
|
||||
source: "test".into(),
|
||||
rejected,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_pronunciation_with_a_control_or_invisible_format_character_is_refused() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
let mut store = EntityStore::open(&d.path().join("e.cc")).unwrap();
|
||||
// Either kind alone is refused: a control character, and a zero-width space.
|
||||
for ipa in ["ˈk\u{7}eɪ", "ˈk\u{200b}eɪ"] {
|
||||
assert!(
|
||||
matches!(
|
||||
store.put(kay_saying(ruling(ipa, vec![]))),
|
||||
Err(StoreError::Refused(StoreRefusal::Malformed { .. }))
|
||||
),
|
||||
"{ipa:?}"
|
||||
);
|
||||
}
|
||||
// The same for a rejected pronunciation.
|
||||
for bad in ["ˈk\u{7}aɪ", "ˈk\u{200b}aɪ"] {
|
||||
assert!(
|
||||
matches!(
|
||||
store.put(kay_saying(ruling("ˈkeɪ", vec![bad.into()]))),
|
||||
Err(StoreError::Refused(StoreRefusal::Malformed { .. }))
|
||||
),
|
||||
"{bad:?}"
|
||||
);
|
||||
}
|
||||
assert_eq!(store.get(&eid("kay")).unwrap(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_pronunciation_keeps_at_most_eight_rejected_variants() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
let mut store = EntityStore::open(&d.path().join("e.cc")).unwrap();
|
||||
let variants = |n: usize| -> Vec<String> { (0..n).map(|i| format!("ˈkaɪ{i}")).collect() };
|
||||
assert_eq!(
|
||||
refusal(store.put(kay_saying(ruling("ˈkeɪ", variants(9))))),
|
||||
StoreRefusal::EntityBounds {
|
||||
id: eid("kay"),
|
||||
bound: EntityBound::Forms
|
||||
}
|
||||
);
|
||||
store
|
||||
.put(kay_saying(ruling("ˈkeɪ", variants(8))))
|
||||
.expect("eight rejected variants are the bound, not beyond it");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_ontology_entry_read_at_an_unknown_revision_is_refused_with_one_refusal() {
|
||||
let w = world();
|
||||
let mut onto = OntologyStore::open(&w.dir.path().join("o2.cc")).unwrap();
|
||||
let entry = OntologyEntry {
|
||||
language: "en-US".into(),
|
||||
n: 1,
|
||||
inventory_rev: 99,
|
||||
layer: Layer::Discourse,
|
||||
when: vec![Condition {
|
||||
slot: Slot::Stance,
|
||||
feature: iv(ANGER),
|
||||
test: Test::AtLeast(Grade::new(200).unwrap()),
|
||||
}],
|
||||
choice: "WARM".into(),
|
||||
examples: vec![],
|
||||
annotator: Annotator::Operator,
|
||||
};
|
||||
match onto.admit(vec![entry], &w.inventory) {
|
||||
Err(StoreError::Invalid(r)) => assert_eq!(r, vec![Refusal::V10UnknownRevision { rev: 99 }]),
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn weksa(dir: &std::path::Path, args: &[&str]) -> Output {
|
||||
Command::new(env!("CARGO_BIN_EXE_weksa"))
|
||||
.current_dir(dir)
|
||||
.args([
|
||||
"--corpus",
|
||||
"./c.cc",
|
||||
"--inventory",
|
||||
"./i.cc",
|
||||
"--entities",
|
||||
"./e.cc",
|
||||
"--ontology",
|
||||
"./o.cc",
|
||||
"--speaker-state",
|
||||
"./s.cc",
|
||||
])
|
||||
.args(args)
|
||||
.output()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn out(o: &Output) -> String {
|
||||
String::from_utf8_lossy(&o.stdout).into_owned() + &String::from_utf8_lossy(&o.stderr)
|
||||
}
|
||||
|
||||
fn ok(o: Output) -> String {
|
||||
assert!(o.status.success(), "{}", out(&o));
|
||||
out(&o)
|
||||
}
|
||||
|
||||
/// A scene bed: entities nibu and brokkr, the inventory through the
|
||||
/// speaker-scene revision (rev 3).
|
||||
fn scene_bed() -> tempfile::TempDir {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
for slug in ["nibu", "brokkr"] {
|
||||
let id = format!("entity:{slug}");
|
||||
let form = format!("en-US={}", slug.to_uppercase());
|
||||
ok(weksa(
|
||||
d.path(),
|
||||
&[
|
||||
"entity",
|
||||
"put",
|
||||
&id,
|
||||
"--form",
|
||||
&form,
|
||||
"--origin",
|
||||
"cast",
|
||||
"--persona",
|
||||
slug,
|
||||
"--fixture",
|
||||
"f.yaml",
|
||||
],
|
||||
));
|
||||
}
|
||||
for step in ["seed", "proof", "speaker-scene"] {
|
||||
ok(weksa(d.path(), &["inventory", step]));
|
||||
}
|
||||
d
|
||||
}
|
||||
|
||||
/// Adds a cast-authored line; returns the key it printed ("Added <key>").
|
||||
fn add_line(d: &std::path::Path, slug: &str, speaker: &str, group: &str, text: &str) -> String {
|
||||
let speaker = format!("entity:{speaker}");
|
||||
let printed = ok(weksa(
|
||||
d,
|
||||
&[
|
||||
"corpus",
|
||||
"add",
|
||||
"--slug",
|
||||
slug,
|
||||
"--text",
|
||||
text,
|
||||
"--speaker",
|
||||
&speaker,
|
||||
"--group",
|
||||
group,
|
||||
"--admitted-on",
|
||||
"2026-10-29",
|
||||
"--source",
|
||||
"cast-authored",
|
||||
"--language",
|
||||
"en-US",
|
||||
"--persona",
|
||||
"nibu",
|
||||
"--fixture",
|
||||
"f.yaml",
|
||||
"--author",
|
||||
"Ada Writer",
|
||||
],
|
||||
));
|
||||
printed.split_whitespace().nth(1).unwrap().to_string()
|
||||
}
|
||||
|
||||
fn scene_put(d: &std::path::Path, group: &str, to: &str) -> Output {
|
||||
weksa(
|
||||
d,
|
||||
&[
|
||||
"scene",
|
||||
"put",
|
||||
"--group",
|
||||
group,
|
||||
"--to",
|
||||
to,
|
||||
"--inventory-rev",
|
||||
"3",
|
||||
"--model-family",
|
||||
"anthropic",
|
||||
"--generator",
|
||||
"a scene annotator",
|
||||
"--set",
|
||||
"intimacy=200",
|
||||
],
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_scene_group_holds_one_live_speaker_among_its_own_live_items() {
|
||||
let d = scene_bed();
|
||||
add_line(d.path(), "nibu", "nibu", "scene-1", "Mind the hatch.");
|
||||
// Another group's speaker and a withdrawn line of this group are not the
|
||||
// scene's speakers.
|
||||
add_line(d.path(), "brokkr", "brokkr", "scene-2", "Brokkr hums.");
|
||||
let gone = add_line(
|
||||
d.path(),
|
||||
"brokkr",
|
||||
"brokkr",
|
||||
"scene-1",
|
||||
"Brokkr leaves early.",
|
||||
);
|
||||
ok(weksa(d.path(), &["corpus", "withdraw", &gone]));
|
||||
let put = ok(scene_put(d.path(), "scene-1", "unnamed"));
|
||||
assert!(put.contains("scene:scene-1@1"), "{put}");
|
||||
assert!(ok(weksa(d.path(), &["scene", "dump"])).contains("entity:nibu->unnamed"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_scene_names_its_addressee_as_a_participant_unless_it_is_the_speaker() {
|
||||
let d = scene_bed();
|
||||
add_line(d.path(), "nibu", "nibu", "scene-1", "Mind the hatch.");
|
||||
add_line(d.path(), "nibu", "nibu", "scene-2", "Mind the ladder.");
|
||||
ok(scene_put(d.path(), "scene-1", "entity:brokkr"));
|
||||
let dump = ok(weksa(d.path(), &["scene", "dump"]));
|
||||
assert!(dump.contains("scene:scene-1@1 rev:3"), "{dump}");
|
||||
assert!(dump.contains("[entity:nibu,entity:brokkr]"), "{dump}");
|
||||
// A speaker addressing themself is a one-participant scene, whose
|
||||
// relation to a named party is refused as the store reads it.
|
||||
let own = scene_put(d.path(), "scene-2", "entity:nibu");
|
||||
assert!(!own.status.success());
|
||||
assert!(
|
||||
out(&own).contains("a relation's named to is another participant"),
|
||||
"{}",
|
||||
out(&own)
|
||||
);
|
||||
assert_eq!(ok(weksa(d.path(), &["scene", "dump"])), dump);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relower_refuses_malformed_arm_and_floor_labels_before_reading_the_run() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
let relower = |arm: &str, floor: &[&str]| {
|
||||
let mut args = vec![
|
||||
"loop",
|
||||
"relower",
|
||||
"no-such-run",
|
||||
"--round",
|
||||
"1",
|
||||
"--arm",
|
||||
arm,
|
||||
"--lowerer",
|
||||
"m",
|
||||
];
|
||||
for f in floor {
|
||||
args.extend(["--floor", f]);
|
||||
}
|
||||
weksa(d.path(), &args)
|
||||
};
|
||||
for (arm, floor) in [
|
||||
("BAD", vec!["b"]),
|
||||
("a", vec!["Bad"]),
|
||||
("a", vec!["a", "b"]),
|
||||
("a", vec!["b", "b"]),
|
||||
] {
|
||||
let o = relower(arm, &floor);
|
||||
assert!(!o.status.success());
|
||||
assert!(
|
||||
out(&o).contains("arm and floor labels must match"),
|
||||
"{arm} {floor:?}: {}",
|
||||
out(&o)
|
||||
);
|
||||
}
|
||||
// Well-formed, distinct labels get past the label check (to the missing run).
|
||||
let o = relower("a", &["b", "c"]);
|
||||
assert!(!o.status.success());
|
||||
assert!(out(&o).contains("the root holds no run"), "{}", out(&o));
|
||||
}
|
||||
136
crates/weksa/tests/mutants_c.rs
Normal file
136
crates/weksa/tests/mutants_c.rs
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
//! Pins for behaviour of `context.rs`, `encode.rs` and `run.rs` found unpinned
|
||||
//! by mutation testing. Each test states what the code does in domain terms.
|
||||
|
||||
mod common;
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use weksa::context::Prompt;
|
||||
use weksa::entity::{Entity, EntityOrigin, LanguageForms};
|
||||
use weksa::packet::{EntityId, proof_packet};
|
||||
use weksa::{CorpusItemId, CorpusStore, EntityForms, EntityStore, InventoryStore, PacketStore};
|
||||
|
||||
fn data(sub: &str) -> PathBuf {
|
||||
Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../../data")
|
||||
.join(sub)
|
||||
}
|
||||
|
||||
struct Held(Vec<Entity>);
|
||||
|
||||
impl EntityForms for Held {
|
||||
fn entity(&self, id: &EntityId) -> anyhow::Result<Option<Entity>> {
|
||||
Ok(self.0.iter().find(|e| &e.id == id).cloned())
|
||||
}
|
||||
}
|
||||
|
||||
fn spoken(slug: &str, form: &str) -> Entity {
|
||||
Entity {
|
||||
id: EntityId::parse(&format!("entity:{slug}")).unwrap(),
|
||||
forms: vec![LanguageForms {
|
||||
language: "en-US".into(),
|
||||
forms: vec![form.into()],
|
||||
pronunciation: None,
|
||||
}],
|
||||
origin: EntityOrigin::Cast {
|
||||
persona: slug.into(),
|
||||
fixture: "f.yaml".into(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// A speaker whom no referent cites contributes no entry to the lowering
|
||||
/// prompt, even when another entity is cited: only the cited entity is listed.
|
||||
#[test]
|
||||
fn an_uncited_speaker_is_absent_from_the_lowering_prompt() {
|
||||
let cast_inv = InventoryStore::open(&data("interlingua/inventory.cc")).unwrap();
|
||||
let entities = EntityStore::open(&data("entity/entity.cc")).unwrap();
|
||||
let corpus = CorpusStore::open(&data("corpus/corpus.cc")).unwrap();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let view = PacketStore::open(&dir.path().join("p.cc"))
|
||||
.unwrap()
|
||||
.admit(proof_packet(), &cast_inv, &corpus, &entities)
|
||||
.unwrap();
|
||||
let state = weksa::SpeakerStateStore::open(&dir.path().join("s.cc")).unwrap();
|
||||
let view = view
|
||||
.with_scene(&cast_inv.inventory().unwrap(), &state, &corpus)
|
||||
.unwrap();
|
||||
let held = Held(vec![
|
||||
spoken("nibu", "Nibu"),
|
||||
spoken("other", "SPEAKERCANARY"),
|
||||
]);
|
||||
let item = CorpusItemId::parse("item:nibu:1").unwrap();
|
||||
let speaker = EntityId::parse("entity:other").unwrap();
|
||||
let prompt = Prompt::lower(
|
||||
&view,
|
||||
&item,
|
||||
&speaker,
|
||||
"en-US",
|
||||
&cast_inv.inventory().unwrap(),
|
||||
&held,
|
||||
&weksa::Ontology::empty(),
|
||||
)
|
||||
.unwrap();
|
||||
let text = prompt.text();
|
||||
assert!(
|
||||
text.contains("\n\nentities: {\"entity:nibu\":[\"Nibu\"]}\n\n"),
|
||||
"{text}"
|
||||
);
|
||||
assert!(!text.contains("SPEAKERCANARY"), "{text}");
|
||||
}
|
||||
|
||||
/// A run refuses a panel language that lacks a live Train or a live Validation
|
||||
/// item: one no item carries, one with only a Test item, one with only Train.
|
||||
#[test]
|
||||
fn a_panel_language_no_item_carries_is_refused_at_open() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let d = dir.path();
|
||||
// Builds the corpus, entity and ontology stores the run reads.
|
||||
drop(common::run_over(d));
|
||||
let entities = common::entities_in(d);
|
||||
let mut corpus = CorpusStore::open(&d.join("c.cc")).unwrap();
|
||||
for (key, language, bucket) in [("item:jp:1", "ja-JP", 90), ("item:pt:1", "pt-BR", 10)] {
|
||||
let mut item = common::item_in(
|
||||
key,
|
||||
&format!("line of {key}"),
|
||||
&common::group_in_bucket(bucket),
|
||||
common::cast(),
|
||||
);
|
||||
item.language = language.into();
|
||||
corpus.add(item, &entities).unwrap();
|
||||
}
|
||||
drop(corpus);
|
||||
let create = |panel: &[&str]| {
|
||||
weksa::run::RunStores::create(
|
||||
&d.join(format!("run-{}", panel.join("-"))),
|
||||
"t",
|
||||
weksa::run::RunPaths {
|
||||
corpus: &d.join("c.cc"),
|
||||
inventory: &d.join("i.cc"),
|
||||
entities: &d.join("entities.cc"),
|
||||
ontology: &d.join("ontology.cc"),
|
||||
speaker_state: &common::speaker_state_beside(&d.join("ontology.cc")),
|
||||
},
|
||||
"fake-model",
|
||||
"bag",
|
||||
weksa::run::LowererReach::ToolLessRelay,
|
||||
weksa::corpus::ModelFamily::Unrecorded,
|
||||
&panel.iter().map(|t| (*t).to_string()).collect::<Vec<_>>(),
|
||||
)
|
||||
};
|
||||
assert!(create(&["en-US"]).is_ok());
|
||||
for panel in [&["ja-JP"][..], &["pt-BR"], &["xx-XX"], &["en-US", "ja-JP"]] {
|
||||
assert!(
|
||||
matches!(create(panel), Err(weksa::run::RunError::Panel)),
|
||||
"{panel:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// A run keeps its recall probes in `recall` under its own root.
|
||||
#[test]
|
||||
fn a_run_keeps_its_recall_under_its_root() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let run = common::run_over(dir.path());
|
||||
assert_eq!(run.recall_dir(), dir.path().join("run").join("recall"));
|
||||
}
|
||||
|
|
@ -11,7 +11,7 @@ use weksa::packet::{FeatValue, FeatureValue, Packet, proof_packet};
|
|||
use weksa::speaker::{Annotator, PairRelation, Party, Scene};
|
||||
use weksa::{
|
||||
CorpusStore, EntityStore, Inventory, InventoryStore, LoweringInput, NameSpan, PacketStore,
|
||||
Refusal, SceneRead, SpeakerStateStore, StoreError, lower,
|
||||
Refusal, SceneRead, SpeakerStateStore, StoreError,
|
||||
};
|
||||
|
||||
struct W {
|
||||
|
|
@ -158,7 +158,7 @@ fn selected(rev: u32, features: Vec<FeatureValue>) -> SceneRead {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn a_scene_relation_reaches_the_lowering_view_and_changes_no_english() {
|
||||
fn a_scene_relation_reaches_the_lowering_view() {
|
||||
let mut w = world();
|
||||
put(&mut w, standard("g", 1));
|
||||
let plain = view_of(&w, 3, Some("n3"));
|
||||
|
|
@ -170,10 +170,6 @@ fn a_scene_relation_reaches_the_lowering_view_and_changes_no_english() {
|
|||
.unwrap();
|
||||
assert_eq!(with.scene(), status().as_slice());
|
||||
assert_eq!(with.scene_read(), &selected(1, status()));
|
||||
assert_eq!(
|
||||
lower(&with, &inv(&w), &w.entities).unwrap(),
|
||||
lower(&plain, &inv(&w), &w.entities).unwrap()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ fn item(group: &str, source: Source) -> CorpusItem {
|
|||
language: "en-US".into(),
|
||||
speaker: weksa::EntityId::parse("entity:s").unwrap(),
|
||||
names: vec![],
|
||||
names_by: None,
|
||||
group: group.into(),
|
||||
prior_turns: vec![],
|
||||
source,
|
||||
|
|
|
|||
|
|
@ -740,6 +740,53 @@ fn a_source_that_only_read_wording_cannot_set_consent() {
|
|||
}
|
||||
}
|
||||
|
||||
// V18: a request frame is licensed by the packet's own consent record.
|
||||
|
||||
/// The committed inventory, where `frame:3` is `request_intervention`.
|
||||
fn cast_refusals(p: &Packet) -> Vec<Refusal> {
|
||||
let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/interlingua/inventory.cc");
|
||||
let inv = weksa::InventoryStore::open(&path).unwrap().inventory().unwrap();
|
||||
validate(p, &inv).err().unwrap_or_default()
|
||||
}
|
||||
|
||||
fn requesting() -> Packet {
|
||||
let mut p = proof_packet();
|
||||
p.predications[0].frame = iv("frame:3");
|
||||
p.discourse.addressee_consent = Some(nd("n4"));
|
||||
p
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_request_frame_needs_consent_naming_its_predication() {
|
||||
assert_eq!(cast_refusals(&requesting()), vec![]);
|
||||
let mut p = requesting();
|
||||
p.discourse.addressee_consent = None;
|
||||
assert_eq!(
|
||||
cast_refusals(&p),
|
||||
vec![Refusal::V18RequestWithoutConsent { predication: nd("n4") }]
|
||||
);
|
||||
p.discourse.addressee_consent = Some(nd("n5"));
|
||||
assert_eq!(
|
||||
cast_refusals(&p),
|
||||
vec![Refusal::V18RequestWithoutConsent { predication: nd("n4") }]
|
||||
);
|
||||
let mut p = requesting();
|
||||
p.provenance.source = ProvenanceSource::GameTelemetry;
|
||||
assert_eq!(cast_refusals(&p), vec![]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_source_consent_is_v15_alone() {
|
||||
let mut p = requesting();
|
||||
p.provenance.source = ProvenanceSource::ModelDecomposition;
|
||||
assert_eq!(
|
||||
cast_refusals(&p),
|
||||
vec![Refusal::V15ConsentSource {
|
||||
source: ProvenanceSource::ModelDecomposition
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
// V17: the addressee is someone spoken to, a referent.
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -85,8 +85,8 @@ thing.
|
|||
An entity's citation form in a language may carry a ruled pronunciation
|
||||
(`LanguageForms.pronunciation`): bare IPA with stress marks, who ruled it, the
|
||||
date, the source, and an optional list of rejected variants kept so nobody
|
||||
re-proposes them. `weksa lower <store> <key> --ipa` prints the stored IPA of each
|
||||
entity the packet cites, one per line, and refuses with `NoPronunciation` for an
|
||||
re-proposes them. `weksa pronounce <store> <key>` prints the stored IPA of each
|
||||
entity the packet cites, one per line (`--phones` prints the phones line), and refuses with `NoPronunciation` for an
|
||||
entity with none; rejected variants are never read. This is not the plan below:
|
||||
it carries no timing, contour, length or invented prosody, and Weksa has no
|
||||
English G2P, so an unruled name has no IPA.
|
||||
|
|
|
|||
|
|
@ -164,6 +164,7 @@ names the rule.
|
|||
`GameTelemetry`; a decomposition of text or imported corpus text cannot mint
|
||||
it.
|
||||
- V17: an `addressee` names a referent, not a predication.
|
||||
- V18: a predication whose frame is the inventory's `request_intervention` is named by `addressee_consent`.
|
||||
|
||||
V9 is retired and not renumbered: the form of a key or a grade is its type, and
|
||||
an extension key's family is V6. `PacketStore::admit` returns the lowering view of a
|
||||
|
|
@ -196,10 +197,24 @@ and `features` carry source-side nuance on content (hedge, certainty, irony and
|
|||
whatever the inventory grows). This is source-side communicative pressure, not
|
||||
target-language register. `addressee_consent` is no feature: it names the
|
||||
predication the addressee asked for or agreed to, a fact about what the
|
||||
addressee did. The English lowerer's request row (`frame:3`, "{agent} asked
|
||||
{patient} to stop") reads it: the consent must name that predication and its
|
||||
agent must be the addressee, else `ConsentMissing` or `AskerNotAddressee`. A
|
||||
consent naming any other predication is refused as unrendered.
|
||||
addressee did. `V14` and `V15` check the field: it may be set only by authoring or game
|
||||
telemetry, and it must name a predication of the packet. `V18` refuses a predication whose frame resolves, at the packet's revision, to
|
||||
the gloss `request_intervention` (`frame:3` in the committed inventories)
|
||||
unless `addressee_consent` names that predication. The encoder can never set
|
||||
consent (`V15`), so an encoder-produced `request_intervention` predication (`frame:3`) is always refused.
|
||||
Limits, tracked by `weksa-lowering:follow_up:request-frame-consent-residual`:
|
||||
`V18` keys on the gloss `request_intervention` and sees no other frame or
|
||||
concept, so a request-shaped meaning built from another frame or concept (a
|
||||
`frame:12` communication with `concept:6` request validates) or from a renamed
|
||||
or newly minted frame is not seen by `validate`, which does not read meaning.
|
||||
The inventory carries no marker for request shape, so classifying it would be
|
||||
an inventory-shape decision for Imagination and language-ontology. `addressee_consent`
|
||||
is one field, so a packet carries one request predication, and a compound
|
||||
request is two packets until a packet schema revision makes consent a set. The
|
||||
asker is not checked to be the addressee: a consented `frame:3` whose agent is
|
||||
not the addressee validates, and no code outside `validate` and the encoder
|
||||
prompt text reads `addressee_consent`. That check is an owed obligation with no
|
||||
owner yet.
|
||||
|
||||
### `stance`
|
||||
|
||||
|
|
|
|||
52
docs/name-span-annotation.md
Normal file
52
docs/name-span-annotation.md
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
# Name-span annotation
|
||||
|
||||
How the name spans of an imported public-domain work are placed. A span has a
|
||||
committed rule that produced it; this file is that rule.
|
||||
|
||||
## Why
|
||||
|
||||
- `pd-annotation-model-then-attest`: a labelled model places the spans of a
|
||||
public-domain work; the operator may attest them afterwards.
|
||||
- `leak-owned-by-flow-and-scorer-3`: spans are fixed with the text at intake and
|
||||
written by no loop writer. The offsets are derived from markup the annotator
|
||||
writes, never typed by a model (inline marking, as in GPT-NER, Wang et al.
|
||||
2023); `CorpusStore::add` judges range, grapheme boundary, form equality and
|
||||
overlap.
|
||||
|
||||
The work file (`weksa corpus import <work>`) is a transcription of a work in
|
||||
hand: provenance, cast forms, lines, scene groups and the speaker each speech
|
||||
prefix names. It holds no distributor header, licence or trademark text, and the
|
||||
binary fetches nothing.
|
||||
|
||||
## Rubric r1
|
||||
|
||||
You are given a work's cast: each member's entity id and the forms the work spells that member with. For each numbered line, copy the line exactly, character for character, and wrap each place where the line names a cast member by one of that member's listed forms as [[<entity id>|<form>]]. Mark a form only where it refers to that cast member; a listed form used as an ordinary word, or naming someone else, stays unmarked. Do not mark pronouns, titles or descriptions unless the exact text is a listed form. When a form is listed for two members, mark the one the line refers to; when you cannot tell, leave it unmarked. Answer one JSON line per input line, in the order given, and nothing else: {"line": N, "marked": "..."}.
|
||||
|
||||
## Procedure
|
||||
|
||||
- One fresh model agent per work. Its brief is rubric r1, the cast as
|
||||
`<entity id>: <form>, <form>` lines and the work's lines as `<n><TAB><text>`;
|
||||
nothing else (no store, no loop, no other answer).
|
||||
- Import with `--model-family` and `--generator '<annotator model id> (name-span
|
||||
rubric r1, docs/name-span-annotation.md)'`. An operator-written or
|
||||
operator-reviewed answer imports with `--operator`.
|
||||
- Import is all or nothing: every entity and item passes the real store doors on
|
||||
scratch copies before the first write, so a refusal (a parse failure, a span
|
||||
the store judges, a work that repeats a line under case or width folding)
|
||||
leaves the stores untouched. Re-draw the whole answer once, log it, and import
|
||||
again. A work file repeating a line's text is refused naming both lines; leave
|
||||
that line out of the answer file (the rest imports with its numbering intact),
|
||||
since a repeated line cannot be two corpus items.
|
||||
- A line the answer does not cover is not imported by that run; a later run over
|
||||
other lines of the work does not collide (keys are `item:<slug>:<n>`).
|
||||
- A line whose own text holds `[[` or `]]` cannot be marked and is refused; it
|
||||
can still enter by hand through `weksa corpus add --name`.
|
||||
|
||||
## Label scope
|
||||
|
||||
`names_by` labels the spans only, not the line or its speaker. One label per
|
||||
item: a later attestation would be a superseding item with the same text and
|
||||
group, but no command writes one yet; question `pd-name-span-attestation` owns
|
||||
that door.
|
||||
`None` means unrecorded (the cast seeds, `weksa corpus add`, the Dao De Jing
|
||||
bed), not attested.
|
||||
|
|
@ -65,7 +65,7 @@ Verified licence facts are few. Discord data excluded by policy; not read.
|
|||
- PersonaChat (Zhang et al. 2018): ParlAI project page, archived 2020-08-01, states no licence, only "You may continue to use the data without issue". [V] https://parl.ai/projects/personachat/ ; paper https://arxiv.org/abs/1801.07243 says data released open source via ParlAI; licence text [GAP]. Crowdworker-written, persona-statement-conditioned chit-chat [B].
|
||||
- LIGHT: ParlAI page states no licence; contents: 663 locations, 3,462 objects, 1,755 character types, 11,000 episodes; WILD 41,131 training episodes from game deployment (human players, so a different provenance than crowdwork). [V] https://parl.ai/projects/light/ . Licence [GAP: check ParlAI repo].
|
||||
- Cornell Movie-Dialogs Corpus: licence/terms page fetch failed (certificate rejected) https://www.cs.cornell.edu/~cristian/Cornell_Movie-Dialogs_Corpus.html . [GAP] Underlying scripts are copyrighted film text [B]; dataset licence unverified.
|
||||
- Project Gutenberg: "vast majority" of eBooks public domain in the US, so no one can grant or withhold permission for uses; PG cannot sublicense copyrighted items; subject to its Terms of Use, which the permission page links and does not summarise; no statement on ML/bulk use on that page. [V] https://www.gutenberg.org/policy/permission.html . Terms-of-Use text on bulk/automated access and the trademark licence: [GAP, not read]. Items public domain in the US may not be elsewhere (same page).
|
||||
- Project Gutenberg: "vast majority" of eBooks public domain in the US, so no one can grant or withhold permission for uses; PG cannot sublicense copyrighted items; subject to its Terms of Use, which the permission page links and does not summarise; no statement on ML/bulk use on that page. [V] https://www.gutenberg.org/policy/permission.html . Terms-of-Use text on bulk/automated access and the trademark licence: read 2026-11-13, see `2026-11-13-gutenberg-terms.md`. Items public domain in the US may not be elsewhere (same page).
|
||||
- Google Synthetic-Persona-Chat (model-generated, persona-chat-style): CC BY-NC? listed as cc-by-4.0 on HF. [V search-only] https://huggingface.co/datasets/google/Synthetic-Persona-Chat
|
||||
- Operator-authored or model-generated corpus: no licence question beyond provider terms [B, unread this pass]. Problems with evaluating on model-generated text: see 2 (self-bias, preference leakage, stylistic homogeneity: Xu et al. 2025; Li et al. 2025). The same Xu et al. finding says it extends to open-ended generation.
|
||||
- Expressive-register-annotated English corpora (hedging, humour, affect, politeness labelled), with permissive terms: [GAP] not searched beyond the list above (e.g. Stanford Politeness Corpus, GYAFC formality, Enron etc. are [B] candidates, unverified, GYAFC is known to carry Yahoo Answers terms [B]).
|
||||
|
|
@ -83,6 +83,6 @@ Verified licence facts are few. Discord data excluded by policy; not read.
|
|||
- No per-metric human-correlation numbers for hedging, humour, affect, stance, persona voice; nearest is formality (Briakou 2021). COMET, BLEURT, MAUVE correlations not obtained.
|
||||
- No source on surface-form smuggling through an LLM bottleneck or its defences (vocab restriction, paraphrase invariance).
|
||||
- No source on schema/inventory growth from generation failures, or its overfitting controls.
|
||||
- Cornell, LIGHT, PersonaChat licence texts not obtained; Gutenberg ToU on bulk use not read.
|
||||
- Cornell, LIGHT, PersonaChat licence texts not obtained.
|
||||
- No realiser or evaluation protocol for constructed languages from a meaning representation.
|
||||
- Several arXiv PDFs (Briakou, Zheng, STEL, LUAR/StyleDistance) fetched as binary; numbers above come from the ar5iv/arXiv-HTML versions where marked, otherwise search summaries.
|
||||
|
|
|
|||
59
docs/research/2026-11-13-gutenberg-terms.md
Normal file
59
docs/research/2026-11-13-gutenberg-terms.md
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
# Project Gutenberg terms: bulk access and header stripping
|
||||
|
||||
Read 2026-11-13 for follow-up `gutenberg-terms` (campaign `weksa-lowering`),
|
||||
ahead of cut `pd-importer`. Pages: gutenberg.org `policy/permission.html`,
|
||||
`policy/terms_of_use.html`, `policy/robot_access.html`, `policy/license.html`.
|
||||
The quotes were checked against the live HTML of each page by a second reader.
|
||||
`pd-importer`'s Imagination re-reads the pages before it fixes a fetch path,
|
||||
since they change.
|
||||
|
||||
## Findings
|
||||
|
||||
- **Automated access to the main site is refused.** Terms of Use: "This website
|
||||
is intended for human users only"; "Any perceived use of automated tools to
|
||||
access this website will result in a temporary or permanent block". OPDS
|
||||
feeds: no more requests than a browser user would make; heavy requesters are
|
||||
blocked by IP.
|
||||
- **Bulk has two sanctioned paths.** Terms of Use: take bulk downloads "from
|
||||
one of our mirrors, not from the main site". The robot-access page names the
|
||||
`robot/harvest` wget endpoint (`-w 2`, filterable by file type or language)
|
||||
and recommends a private mirror for a full local copy; it suggests checking
|
||||
periodically for changes. Catalog data are offered as
|
||||
machine-readable offline catalogs, "granted to the public domain".
|
||||
- **Header and licence stripping.** Licence section 1.C: for a work not
|
||||
protected by US copyright, with the user in the US, the Foundation does "not
|
||||
claim a right to prevent you from copying, distributing, performing,
|
||||
displaying or creating derivative works", "as long as all references to
|
||||
Project Gutenberg are removed", the condition holding for a user in the US.
|
||||
Section 1.E opens "Unless you have removed all references to Project
|
||||
Gutenberg", and 1.E.1 to 1.E.7 (including 1.E.4, do not detach the licence)
|
||||
attach to a work on which the phrase "Project Gutenberg" appears or with which
|
||||
it is associated, so a text with every reference removed is outside 1.E.
|
||||
1.E.2 is not that grant: it says a PG eBook derived from texts not restricted
|
||||
by US copyright may be copied and distributed in the US without fee. The
|
||||
trademark is registered; non-commercial use of the name needs no permission,
|
||||
commercial use does.
|
||||
- **No statement on machine learning or AI use** on any of the four pages.
|
||||
- **US public domain only.** The Foundation makes no claim about copyright
|
||||
outside the US (1.D). A work free in the US may not be free elsewhere, and
|
||||
Gutenberg's own catalog is not a per-work clearance.
|
||||
|
||||
## What this decides for `pd-importer`
|
||||
|
||||
- Fetch from a mirror or an offline dump. The Terms of Use footnote counts "many
|
||||
books" as more than about 100 a day and sends those to mirrors, while
|
||||
`robot/harvest` is a `www.gutenberg.org` URL: use it (with `-w 2`) only for a
|
||||
handful of works. Never crawl `gutenberg.org` pages or OPDS from the loop host.
|
||||
- Strip the whole header and footer, every Project Gutenberg reference and the
|
||||
licence text from the stored text; keep provenance as the ebook number and
|
||||
edition in the item's source field, which is Weksa's own record and not a use
|
||||
of the mark. This is the path `docs/targets/dao-de-jing-test-bed.md` already
|
||||
takes. The remaining open question there, what counts as a "reference", stays
|
||||
open: the safe reading is that the ebook number in a provenance field is a
|
||||
catalogue identifier, not a reference inside the work, but nothing read here
|
||||
rules on it.
|
||||
- Per-work check stays with the importer: the work's author and first
|
||||
publication date against US public-domain rules, recorded on the item. The
|
||||
Foundation's catalogue does not supply it. A work Gutenberg posts with the
|
||||
copyright holder's permission (1.E.3) is out of scope.
|
||||
- No importer code in this note.
|
||||
|
|
@ -23,9 +23,10 @@ plus a separately recorded chosen gloss per lowering).
|
|||
`corpus add ... --language en-US --source public-domain` per item, then `packet dao <store>`.
|
||||
- Source: Project Gutenberg eBook 216, "The Tao Teh King, or the Tao and its
|
||||
Characteristics", Laozi, tr. James Legge (Sacred Books of the East 39, 1891).
|
||||
Gutenberg's licence (1.E.2) lets a work derived from texts not protected by US
|
||||
copyright be used with no fee provided every Project Gutenberg reference is
|
||||
removed from the work. The items hold only Legge's words, one numbered section
|
||||
Gutenberg's licence (1.C) does not claim a right to prevent derivative works of
|
||||
a work not protected by US copyright, for a user in the US, provided every
|
||||
Project Gutenberg reference is removed (see
|
||||
`docs/research/2026-11-13-gutenberg-terms.md`). The items hold only Legge's words, one numbered section
|
||||
or sentence each; no Gutenberg header, licence text or trademark is stored,
|
||||
and the eBook file itself is not committed. `edition_url` names the page the
|
||||
text was retrieved from, as a citation of provenance and not as part of any
|
||||
|
|
@ -80,6 +81,8 @@ ceiling of an author who saw the reference:
|
|||
frame minted from Train, scored lowest (chrF 44.18). The scores rise with
|
||||
how bespoke the frame is, the reverse of generalisation. The Validation and
|
||||
Test numbers measure the lowerer table, not what the interlingua can carry.
|
||||
That table is deleted: the Dao numbers below were measured on it, and the
|
||||
loop now lowers through the model (`Prompt::lower`).
|
||||
- n = 5, one translation as reference, one embedding model, no variance.
|
||||
|
||||
## What the scorer cannot see
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ means no store holds it and nothing writes it.
|
|||
| Community (cut `language-ontology`, follow-up `language-ontology-communities`) | Key `community:<slug>`; type `weksa.community`, built fresh when the cut is mapped: `data/cultural-ontology/` (draft YAML profiles, read by no code) is deleted from main (cut 5e `delete-cultural-ontology`) and stays in git history before that cut's commit | Revised by commit; a speaker names the communities it belongs to by id | The cultural ontology's author (owner-attested evidence only; Discord-only content stays deleted, `discord-only-content-deleted`) |
|
||||
| Speaker (cut `speaker-scene`) | Key `speaker:<slug>@<n>`; type `weksa.speaker`: stable axes as feature values of slot `Speaker` (origin, socioeconomic standing, age cohort, education and whatever the loop adds), at an inventory revision, and an `Annotator` | A new revision appends, never edits; a corpus item and every prior turn name a speaker slug, and a round records the speaker revisions it read | The annotator it names: `PersonaSnapshot` (a projection of a `gamecult.persona_state.v0` document, naming its source document and `updatedAt`), `Operator`, `WorkMetadata` (a public-domain work's own cast list and narration), or `Model { family, generator }` (the 2b `ModelFamily`) |
|
||||
| Scene (cut `speaker-scene`) | Key `scene:<group>@<n>`, the item's split group; type `weksa.scene`: participants (speaker slugs) and, per ordered pair, relation features of slot `Scene` (relative status, intimacy, in-group, audience and whatever the loop adds), at an inventory revision (provenance; a run reads the features at its own head), and an `Annotator` | As Speaker. Keyed by the group, so a scene never straddles the split. `data/original/speaker-state.cc` holds Model-annotated keeper scenes, one per Train or Validation group of both languages (`weksa scene put|dump`); a Test group has none and reads `NoScene` | As Speaker; a model-annotated scene is never Test evidence |
|
||||
| Corpus item | Key `item:<source-slug>:<n>`, `n` canonical decimal >= 1 (no sign, no leading zero, from cut 2b); type `weksa.corpus_item` with text, language, speaker, split group, prior turns, source. Source `ModelWritten` carries a `ModelFamily` enum, the generator string as the fixture wrote it, and a prompt digest (cut 2b). The item carries no speaker state of its own: its speaker slug and group name the Speaker and Scene documents | Admitted once; the whole item is immutable (re-adding an identical item is a no-op, any difference is refused; a correction is a new item that `supersedes` the old); text already held under another key is refused unless the new item supersedes that key (cut 2b); withdrawn, never deleted, so old scores stay explainable | Importers and `weksa corpus add`, under `corpus-cast-plus-public-domain`; the source enum has no variant for Discord or any training-forbidden source (`corpus-clean`) |
|
||||
| Corpus item | Key `item:<source-slug>:<n>`, `n` canonical decimal >= 1 (no sign, no leading zero, from cut 2b); type `weksa.corpus_item` with text, language, speaker, split group, prior turns, source and who placed its name spans (`names_by`, an `Annotator`, absent when unrecorded; cut `pd-name-spans`). Source `ModelWritten` carries a `ModelFamily` enum, the generator string as the fixture wrote it, and a prompt digest (cut 2b). The item carries no speaker state of its own: its speaker slug and group name the Speaker and Scene documents | Admitted once; the whole item is immutable (re-adding an identical item is a no-op, any difference is refused; a correction is a new item that `supersedes` the old); text already held under another key is refused unless the new item supersedes that key (cut 2b); withdrawn, never deleted, so old scores stay explainable | Importers and `weksa corpus add`, under `corpus-cast-plus-public-domain`; the source enum has no variant for Discord or any training-forbidden source (`corpus-clean`) |
|
||||
| Split assignment | Derived: `split(scheme, item.group)` | Fixed for a scheme; a new scheme is a new constant, never a reassignment. A model-written item whose group derives test is in no split; a superseding item keeps its predecessor's group | The hash function. Nobody assigns by hand, so there is no writer to forbid |
|
||||
| Loop round | Key `run:<label>:round:<n>`; type `weksa.loop_round` with spec version, profile revisions, model ids and prompt digests, sampled train items, Validation means and refusal counts (from cut `ja-panel-loop`: one set per language of the run's panel, no pooled mean), per-item Validation scores in corpus order (key 22, numbers only, no item key or text, private to `run.rs` for the paired floor of the cycle (rounds 1 to 4) and of relower; a run from before has none and opens), the digest of the inventory file it was scored under and, for a cycle's candidate round, its cycle `k` | Scored and recorded whole; immutable. A candidate round is put in one batch with its Decision. Lives in the run store on Yggdrasil | `run::round` and `run::cycle` (both through `score_round`) |
|
||||
| Loop decision | Key `run:<label>:cycle:<k>`; type `weksa.loop_decision` with the base round and revision, the edit prompt's digest and Train keys, the editor's answer digest, the candidate revision, inventory digest and round, the base, candidate and floor readings (chrF, meaning, composite; the floor is the paired floor of round 1 against rounds 2 to 4), the threshold and the verdict (Accepted, or Rejected for edit-parse, edit-admit, no-gain, regressed chrF, meaning or composite); digests and numbers, no text | Immutable. The latest Accepted one names the run's head inventory (`<root>/cycle-<k>/inventory.cc`, its candidate digest, its candidate round); with none the head is the copy `create` made at `<root>/inventory.cc`. Lives in the run store on Yggdrasil | `run::cycle`'s accept rule, `run::decide`: the validation composite gains more than max(noise floor, `LADDER_STEP`), no metric falls past max(its floor, one step), the composite does not fall past max(its floor, one step), and a non-finite reading rejects (the one regression test, `run::regression`); differences compared in billionths so a gain of exactly one step is a tie. From cut `ja-panel-loop` the readings, floor and threshold are kept per panel language and `run::regression` runs per language and is the one regression test (`decide` adds the gain test for a one-language panel): any language regressing rejects (verdicts `regressed-chrf`, `regressed-meaning`, `regressed-composite`), otherwise a one-language panel accepts as `decide` does and a panel of two or more accepts when the sum of composite deltas exceeds one `LADDER_STEP` (billionths). Given up: strict Pareto non-regression; a language may lose up to its own noise floor if the panel nets more than a step, because zero tolerance ratchets every base to a favourable noise draw and strangles the loop. Termination: each accept raises the composite sum by more than a step and composites lie in -0.5 to 1, so a run accepts at most 1.5 * languages / `LADDER_STEP` edits |
|
||||
|
|
@ -340,9 +340,14 @@ line two", "その 制御" against the
|
|||
- Stores: `data/interlingua/inventory.cc` and `data/corpus/corpus.cc` in the
|
||||
repo (small, reviewed with `weksa dump`); run stores (rounds, lowerings,
|
||||
scores) on Yggdrasil under the loop's state root, not in git.
|
||||
- The loop runs on Yggdrasil as a detached unit with a log and a status record.
|
||||
Starfire runs none of it. Model calls go through a `ModelPort`; its first
|
||||
adapter drives `claude -p` on the operator's subscription. Never OpenAI
|
||||
- The loop is a batch command driven from a Claude Code session on Yggdrasil
|
||||
(`golf-loop-in-claude`), not a detached unit. Model calls go through a
|
||||
`ModelPort` whose one adapter, `AnswerFiles`, writes a `.prompt` file and reads
|
||||
a `.answer` file: Claude agents answer them, one fresh subagent per encode
|
||||
prompt (`encoder-isolation-subagent-per-prompt`) and a tool-less relay for
|
||||
lowering prompts (`lowerer-call-reach-toolless-relay`). Builds, stores, scoring
|
||||
and embeddings run on Yggdrasil; Starfire runs none of it, and Weksa needs no
|
||||
claude binary or token there. Never OpenAI
|
||||
(`eureka-body:ruling:no-return-to-openai-supplier`), never Discord data
|
||||
(`gamecult-chat:ruling:discord-taints-training`).
|
||||
|
||||
|
|
@ -601,57 +606,25 @@ measurable with this corpus either.
|
|||
|
||||
Training pairs are Train only, at a frozen revision; the export is 12b.
|
||||
|
||||
## Lowerer first cut (`golf-loop-lowerer-cut`)
|
||||
## Lowering authorities (`golf-loop-lowerer-cut`, `table-retirement`)
|
||||
|
||||
Authority map for the smallest lowerer, written before its code (Imagination
|
||||
pass by the Weksa Self run, 2026-11-01). The cut exists to give the golf loop a
|
||||
measurable baseline from a packet: `lower` gives text, `score` gives numbers.
|
||||
Authority map of who decides what about a packet's wording. The first cut gave
|
||||
the loop a baseline from an English template table; cut `table-retirement`
|
||||
deleted that table, `lower()` and the CLI lowering command, so no deterministic path
|
||||
produces lowered text and the model's answer to `Prompt::lower` is the only
|
||||
wording owner. `lower.rs` keeps the entity port, `pronounce` and `phones`
|
||||
(`weksa pronounce`).
|
||||
|
||||
| Decision | Owner | Inputs | Outputs | Not the owner |
|
||||
|---|---|---|---|---|
|
||||
| Whether a packet exists | `validate` / `PacketStore::admit` (unchanged) | packet, inventory, corpus, entities | admitted packet or typed refusal | the lowerer |
|
||||
| Wording of a scored item in English (golf loop, `golf-loop-model-lowerer`) | the ModelPort's answer to `Prompt::lower` (`context.rs`), read as `LoweringDraft` (`encode.rs`) and held in `run::score_round`'s locals | the admitted view, the inventory at its revision, en-US entity forms | `{"text"}`, scored once, never stored | the corpus item, the scorer, the editor |
|
||||
| Wording of a packet in English (`weksa lower`, proof fixtures) | `crates/weksa/src/lower.rs`: `lower(&LoweringInput, &Inventory) -> Result<String, LowerError>` | the validated lowering view and the inventory at its revision | text, or a typed refusal naming the missing template | the corpus item, the scorer, the golf loop |
|
||||
| English template table (concept words, frame templates, speech-act openers) | the same module, as rows `(inventory id, gloss it speaks for, surface)` | none; it is data in code | rows the lowerer looks up | the inventory (it owns the ids and glosses, not the English) |
|
||||
| The inventory's concepts and frames | an inventory revision; here `inventory::proof_revision()` (rev 2), hand-seeded as `seed()` is | none | the 6 concepts and 3 frames `nibu-proof-001` and the consent-gated request packet name | the lowerer |
|
||||
| What a score counts | `Scorer::score` (unchanged) | output text, reference text | `ItemScore` | `weksa score`, which only reads text and calls it |
|
||||
| The proof packet | `packet::proof_packet()`, typed ids of rev 2, admitted by `weksa packet proof <store>` through `PacketStore::admit`. Not JSON: `Packet`'s serde is CultCache's positional tuple, and JSON in is the xenos boundary the cut 7 encoder owns | none | admitted packet | the CLI decides nothing about validity |
|
||||
|
||||
Decisions this map makes, and why:
|
||||
|
||||
- **`lower` takes `&LoweringInput`, not `&Packet`.** A `Packet` carries `trace`
|
||||
and `provenance.corpus_item`; the model page says the lowerer sees the packet
|
||||
without its trace and never the corpus item (`lowering-traced`,
|
||||
`leak-owned-by-flow-and-scorer-3`). `validate` already yields the view, so the
|
||||
CLI runs `validate` and hands it over. Cut 4 narrows the door to
|
||||
`PacketStore::admit`; this cut does not wait on it.
|
||||
- **The frame owns the verb phrase; a predication's concept must resolve but is
|
||||
not rendered.** Referent concepts supply nouns. A frame template names its
|
||||
roles by inventory gloss (`{agent}`), resolved against the inventory at the
|
||||
packet's revision, so a retired or re-glossed role cannot be silently
|
||||
mis-keyed. A row whose gloss differs from the inventory's entry for its id is
|
||||
a `LowerError`, so a table row cannot speak for a different meaning than its
|
||||
id now carries.
|
||||
- **Rendering rules, all deterministic and all from packet fields:** the
|
||||
discourse addressee renders "you" ("yourself" when it already filled a role
|
||||
in the same predication), other referents render "the <concept word>", a
|
||||
predication filling a role renders "that", the speech-act row supplies an
|
||||
optional opener, sentences are capitalised and end in a full stop.
|
||||
Entity-bearing referents render by concept word: the lowerer does not read
|
||||
the entity store in this cut (follow-up: entity forms in the lowerer).
|
||||
- **No style, no register, no features.** Hedge, certainty, modality and
|
||||
polarity are not rendered, so the baseline scores what meaning alone buys.
|
||||
A lowerer that ignores `polarity: negative` would assert the opposite, so
|
||||
`lower` refuses any packet that carries a feature, relation or extension it
|
||||
does not render rather than render it wrong (`packet-owns-meaning`); features arrive as templates
|
||||
learn them.
|
||||
- **Deleted by later cuts, not this one:** the table in `lower.rs` and
|
||||
`proof_revision()` are temporary owners. Cut 5 (`language-ontology`) moves
|
||||
the table to `weksa.ontology_entry` documents selected by the target
|
||||
profile; cut 7 replaces `proof_revision()` with the loop's minting round.
|
||||
Nothing is deleted today because no earlier lowerer exists; the YAML
|
||||
`examples/english-lowering/` hand-lowerings are fixtures, not owners, and
|
||||
stay.
|
||||
- **`weksa score` is a thin reader of `Scorer::score`**: `weksa score <item>...
|
||||
[--output FILE]` embeds the output text once against every named item. It
|
||||
stores nothing (Score documents arrive with cut 7) and gates nothing.
|
||||
|
|
@ -673,11 +646,12 @@ One line per cut; specs exist for the cuts about to run (2d and 4, siblings on `
|
|||
5. `language-ontology`, in sequential cuts on `d9ebbcf`. 5a `language-ontology` (spec r1): `weksa.ontology_entry`, `OntologyStore`, `ontology::select` over the view's `Stance` and `Discourse` features, `Prompt::lower` printing the selected entries of the item's language, `RunRecord` pinning the ontology store (keys 15 and 16), `weksa ontology admit|dump`; `ontology.rs` joins `PROMPT_SOURCE_DIGEST`. 5b `language-ontology-seed` (spec r1, after 5a): Model-annotated en-US and ja-JP seed entries drafted by an isolated model, `data/target-language-ontology/`, `.weksa/cultural-ontology/` and `schemas/target-language-cultural-ontology.md` deleted. Lost until 5c': keigo, hierarchy and uchi-soto hang on the scene relation from 5c', so ja-JP politeness is still the lowerer's guess. 5c' `language-ontology-scene` (spec r1, after 5b): the run pins the `SpeakerStateStore` (keys 17 and 18, `Drift(SpeakerState)`), `RunStores::assembled` is the one door that reads the item's scene into a loop view, `select` reads slot `Scene`, `Prompt::lower` refuses a view whose scene was never read, `weksa scene put|dump` and `weksa inventory speaker-scene` (rev 4 in data/original and data/daodejing, rev 3 in data/interlingua, which holds no Dao revision), Model-annotated keeper scenes and Scene-conditioned entries for `data/original`. 5c `language-ontology-view-assembly` (follow-up, now only the Speaker read): `select` widens to slot `Speaker`; waits on `speaker-scene-seeds`. 5d `language-ontology-communities` (follow-up): `weksa.community` built fresh (`data/cultural-ontology/` is deleted by 5e), community ids on `weksa.speaker`, community conditions; waits on 5c and question `persona-speech-community-home`.
|
||||
5e. `delete-cultural-ontology` (steward ruling, 2026-11-13): `data/cultural-ontology/` (five draft YAML profile files and a README; `old-norse-edda-source-tradition.yaml` did not parse) deleted. Nothing in `crates/`, tests, scripts or CI read it; `examples/persona-lowering/brokkr-edda-voice/` cites its paths as text only (the one test that reads that example reads `lowered-output.yaml`). The typed `weksa.community` of follow-up `language-ontology-communities` is built fresh under cut 5; follow-up `edda-referent-yaml-parse` is moot.
|
||||
6. `pd-importer`: public-domain play and fiction dialogue into the corpus with speaker, scene group, edition provenance, and Speaker and Scene annotations by the ruled annotator. Waits on 4, follow-up `gutenberg-terms` and question `pd-speaker-annotation`.
|
||||
6a. `pd-name-spans` (spec r1): `weksa corpus import <work> <names>` mints a work's cast as `PublicDomain` entities with forms in the work's language and adds each line a name-span answer covers, speaker from the work file, spans from `[[entity|form]]` markup, labelled `names_by` (Model or Operator) per `docs/name-span-annotation.md`. Speaker and Scene documents, prior turns and fiction speaker attribution stay with 6.
|
||||
7a. `loop-context-builder`: `context.rs` (`TrainItems`, `EncoderOffer`, `Prompt`: corpus text reaches a model call only through `Prompt::mint` over live Train items or `Prompt::encode` over one offered item; instructions are the consts `MINT_INSTRUCTIONS` and `ENCODE_INSTRUCTIONS` in `context.rs`; `Checkpoint` has no public constructor, offers each item once, and its offers borrow it (the gate is typed `TestReadings` and Test offers on the round path; operator tools such as `weksa corpus dump` and `weksa score` still read corpus text of any split); prior turns print the held Train text), `SplitScheme::live`, `CorpusItem::cited_entities` (one owner of the id set `PacketStore::admit` and the offer share), and checkpoint-only Test readings (`Checkpoint`, `TestReadings`; `Aggregate` and `leak_diagnostic` carry no Test). Library only; since 7b `TrainItems::from_corpus` and `EncoderOffer::for_round` are crate-private and `RunStores` is their one public producer. Merges follow-ups `loop-context-builder-2` and `scorer-test-readings-type`. (spec r1)
|
||||
7b. `golf-loop-round` (spec r1; branch `weksa-lowering/golf-loop-round`): the smallest loop that scores one round. `run.rs`: `RunStores` pins the corpus and entity stores of a run by canonical path and FNV-1a 64 digest at `create`, reopens only from `<root>/run.cc` and refuses drift (corpus, entities, `PROMPT_SOURCE_DIGEST` of `context.rs`, a second pin on `lower.rs` retired in 7d, model, embedder); `round` encodes every live Train and Validation item through a `ModelPort` (`model.rs`; the one adapter `AnswerFiles` writes `<digest>.prompt` and reads `<digest>.answer`, so the model is the Claude session), reads each answer into a `PacketDraft` (`encode.rs`, the one place a model's JSON is read), admits through `PacketStore::admit` into `<root>/round-<n>/packets.cc`, lowers, scores once, and keeps a `RoundRecord` (`weksa.loop_round`, no text, no Test; Train rows, Validation only as means and refusal counts). `weksa loop open|round`. Not here: the spec-edit step, accept rule and composite, noise floor, Score and lowering documents, a subprocess adapter, the checkpoint door, a model lowerer (follow-ups `golf-loop-spec-edit`, `golf-loop-accept-rule`, `golf-loop-score-documents`, `golf-loop-subprocess-adapter`, `golf-loop-checkpoint-door-bound`, `golf-loop-model-lowerer`). `packet-store-inventory-binding` is met for loop packets only (`RunStores` admits with the run's inventory); `validate`, `packet proof|dao` and `lower` still take caller stores. Waits on 7a, `checkpoint-gate`, 4b.
|
||||
7c. `golf-loop-edit-accept` (spec r1; branch `weksa-lowering/golf-loop-edit-accept`): one edit-accept cycle. The run works on its own copy of the inventory (`create` copies the caller's file to `<root>/inventory.cc`, refusing a root that already holds a file of that name, and pins its digest as `RunRecord` key 11; `RunRecord.inventory` is provenance; a `run.cc` from before has no key 11 and does not open); the head is derived from the latest Accepted `Decision`. `weksa loop cycle` asks the editor for one inventory edit from `Prompt::mint(train, inventory, rev, feedback)` (Train blocks, the inventory listing and an `EditFeedback` line: whether the last edit was kept and the head's Validation composite rounded to `LADDER_STEP`, the Ladder release of Blum and Hardt 2015), reads it as an `EditDraft` (`encode.rs`; ids and revisions are the store's, `into_revision`), admits it through `InventoryStore::admit_revision` into `<root>/cycle-<k>/inventory.cc`, runs a candidate round under it, and `decide`s on Validation against the head's establishing round past a floor (never under one `LADDER_STEP`; from cut `cycle-floor-paired` the paired floor over rounds 1 to `FLOOR_ARMS_MIN` + 1, so a cycle needs four recorded rounds, where 7c measured the spread between rounds 1 and 2). An edit that fails is a rejected `Decision`, not an error. `RoundRecord` gains keys 16 (inventory digest) and 17 (cycle); an encode answer that is not UTF-8 is that item's parse refusal. Not here: Score and lowering documents (`golf-loop-score-docs`), Train rows in the edit prompt (`golf-loop-edit-train-feedback`), a paired floor (`golf-loop-paired-floor`), the checkpoint door and merging to `data/` (`golf-loop-checkpoint-door-bound`, `golf-loop-freeze-merge`), a model lowerer (`golf-loop-model-lowerer-next`; until it lands an entry an edit adds has no lowering row, so a cycle gains only by retiring entries or adding ones the encoder does not cite). Waits on 7b.
|
||||
7c. `golf-loop-edit-accept` (spec r1; branch `weksa-lowering/golf-loop-edit-accept`): one edit-accept cycle. The run works on its own copy of the inventory (`create` copies the caller's file to `<root>/inventory.cc`, refusing a root that already holds a file of that name, and pins its digest as `RunRecord` key 11; `RunRecord.inventory` is provenance; a `run.cc` from before has no key 11 and does not open); the head is derived from the latest Accepted `Decision`. `weksa loop cycle` asks the editor for one inventory edit from `Prompt::mint(train, inventory, rev, feedback)` (Train blocks, the inventory listing and an `EditFeedback` line: whether the last edit was kept and the head's Validation composite rounded to `LADDER_STEP`, the Ladder release of Blum and Hardt 2015), reads it as an `EditDraft` (`encode.rs`; ids and revisions are the store's, `into_revision`), admits it through `InventoryStore::admit_revision` into `<root>/cycle-<k>/inventory.cc`, runs a candidate round under it, and `decide`s on Validation against the head's establishing round past a floor (never under one `LADDER_STEP`; from cut `cycle-floor-paired` the paired floor over rounds 1 to `FLOOR_ARMS_MIN` + 1, so a cycle needs four recorded rounds, where 7c measured the spread between rounds 1 and 2). An edit that fails is a rejected `Decision`, not an error. `RoundRecord` gains keys 16 (inventory digest) and 17 (cycle); an encode answer that is not UTF-8 is that item's parse refusal. Not here: Score and lowering documents (`golf-loop-score-docs`), Train rows in the edit prompt (`golf-loop-edit-train-feedback`), a paired floor (`golf-loop-paired-floor`), the checkpoint door and merging to `data/` (`golf-loop-checkpoint-door-bound`, `golf-loop-freeze-merge`), a model lowerer (landed as 7d: until then an entry an edit added had no lowering row). Waits on 7b.
|
||||
7d. `golf-loop-model-lowerer` (spec r1; branch `weksa-lowering/golf-loop-model-lowerer`): the round lowers through the `ModelPort`. `Prompt::lower` (`context.rs`) takes a `LoweringInput` (no text, key, group or speaker getter), the item's speaker id, the inventory at the view's revision and the item language's entity forms, and prints `LOWER_INSTRUCTIONS`, the meaning of the packet, the entities line and the inventory listing; the item key goes only to `item_keys`, and the speaker id never prints: a referent citing the item's speaker holds the fixed token `entity:speaker` (`SPEAKER_TOKEN`, a constant the same for every speaker and item, never derived from an id) and the entities map holds the speaker's forms under that token. The promise guards the id that names an entity of this item; the speaker's forms and the role token do print, because forms are shared by every item of the language (follow-up `speaker-alias-token-in-lowering-prompt`, which restored the lowerer's ability to name its speaker after #84 removed it). A speaker the language has no form for, or the port lacks, does not refuse: its referents print no entity and the map has no entry. A referent citing the real id `entity:speaker` is refused (`LowerError::SpeakerTokenTaken`); the entity store does not reserve it. The answer is `{"text"}`, read as `LoweringDraft` in `encode.rs`, held in `score_round`'s locals and scored once; it is never put in `run.cc`, a `RoundRecord`, a `Decision` or `EditFeedback`. A round has two answer phases (every encode prompt, then every lowering prompt), all or nothing: while any prompt is unanswered `packets.cc` is discarded and nothing reaches `run.cc`. `lower::lower` and the template table leave the loop (they remain `weksa lower`'s; follow-up `table-retirement`), so an entry an edit adds can be said and a cycle can buy a gain. One source pin remains: `PROMPT_SOURCE_DIGEST` covers `context.rs`, `split.rs` and `lower.rs` (`RunRecord` key 7 and `RoundRecord` key 3 are retired). The run's record pins `LowererReach` (`weksa loop open --lowerer-reach briefed-subagent|tool-less-relay`, `RunRecord` key 12) and every printed round and cycle line says it; a briefed-subagent run says its numbers are smoke-round only (rulings `lowerer-call-reach-toolless-relay`, `dao-round-seed-inventory`). A lowering is refused at `Lower` when blank, over `MAX_LOWERING_BYTES` (4096) or holding a control character other than a line break or tab; `PROMPT_SOURCE_DIGEST` also covers `packet.rs`. Not here: per-item lowering text documents (`golf-loop-score-docs`), the subprocess adapter. Waits on 7c.
|
||||
7. `golf-loop-en` (7b and 7c landed the round, `ModelPort`, encoder prompt, template digest, round records and the edit-accept cycle; the rest remains): the packet store bound to one inventory store everywhere (`packet-store-inventory-binding`), a subprocess adapter, Score documents, `weksa score` in the loop, the detached unit on Yggdrasil. Waits on 3, 5, 6 and follow-up `claude-on-yggdrasil`.
|
||||
7. `golf-loop-en` (7b, 7c and 7d landed the round, `ModelPort` with `AnswerFiles`, encoder and lowering prompts, template digest, round records and the edit-accept cycle; the rest remains): the packet store bound to one inventory store everywhere (`packet-store-inventory-binding`), Score documents (`golf-loop-score-docs`, `golf-loop-score-documents`), the accept rule (`golf-loop-accept-rule`), the checkpoint door and freeze merge (`golf-loop-checkpoint-door-bound`, `golf-loop-freeze-merge`). No subprocess adapter and no detached unit: `golf-loop-in-claude` and `encoder-isolation-subagent-per-prompt` rule them out. Waits on 3, 5 and 6.
|
||||
8. `style-axis`: a style-embedding score behind the scorer port. Waits on 3.
|
||||
9. `ja-panel`: ja-JP into the loop's panel, in two cuts on `e9cbd9b`. 9a `ja-panel-loop` (spec r1): language is the item's and the panel the run's: `Prompt::lower` takes the item's language and prints it on a language line (no prompt names English), entity forms are read in that language, `RunRecord` pins the panel (`weksa loop open --panel`), offers, mint and recall read only panel items, round means, floors and readings are per panel language, the accept rule runs `decide` per language, the recall split works on unspaced scripts, and `weksa corpus add` has no default language. 9b `ja-panel-corpus` (spec r1, after 9a): the ja-JP slice of `data/original`, model-written Japanese Train and Validation passages and eight Aozora Bunko public-domain Test sentences with full provenance, and their entities. Cut 5 is not a prerequisite: neither language lowers with ontology entries until cut 5 gives both theirs.
|
||||
10. `conlang-lexicon`: the first conlang's lexicon placed at concept embeddings, its morphology data, its ontology entries and the gloss reader. Waits on 3, 5 and question `first-conlang`.
|
||||
|
|
|
|||
|
|
@ -24,6 +24,17 @@
|
|||
"$ref": "#/$defs/NameSpan"
|
||||
}
|
||||
},
|
||||
"names_by": {
|
||||
"description": "Who placed `names`: the annotator of the intake that wrote them. `None`\nwhen nothing records one (the cast seeds, `weksa corpus add`, the Dao De\nJing bed); it labels the spans, never the line or its speaker.",
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/Annotator"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"prior_turns": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
|
|
@ -63,6 +74,41 @@
|
|||
"names"
|
||||
],
|
||||
"$defs": {
|
||||
"Annotator": {
|
||||
"description": "Who annotated a document. A model's generator is an attestation string, held\nto the same form as every provenance string.",
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"Operator"
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"Model": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"family": {
|
||||
"$ref": "#/$defs/ModelFamily"
|
||||
},
|
||||
"generator": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"family",
|
||||
"generator"
|
||||
]
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"Model"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"EntityId": {
|
||||
"type": "string",
|
||||
"pattern": "^entity:[a-z0-9]([a-z0-9-]{0,46}[a-z0-9])?$"
|
||||
|
|
|
|||
|
|
@ -1,16 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
# Build the "Show her working" page from the real lowerer and scorer, then publish it as a Mind preview.
|
||||
# Run from the repository root, in a Mind's Self run (it writes only /srv/minds-web/<mind>/).
|
||||
#
|
||||
# scripts/publish-show-working.sh [--no-publish]
|
||||
#
|
||||
# Needs cargo, python3 and an Ollama embedder at 127.0.0.1:11434 (qwen3-embedding:0.6b): the scorer is not stubbed.
|
||||
# publish-mind-preview.sh lives in the gamecult-ops checkout: set GAMECULT_OPS to it (default ../gamecult-ops).
|
||||
set -euo pipefail
|
||||
[ -f Cargo.toml ] && [ -d tools/show-working ] || { echo "run from the Weksa repository root" >&2; exit 2; }
|
||||
out=${SHOW_WORKING_OUT:-tools/show-working/out}
|
||||
rm -rf "$out"
|
||||
python3 -I tools/show-working/build.py "$out"
|
||||
[ "${1:-}" = --no-publish ] && { echo "built $out"; exit 0; }
|
||||
ops=${GAMECULT_OPS:-../gamecult-ops}
|
||||
exec "$ops/scripts/publish-mind-preview.sh" "$out" weksa-show-working
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
# Show her working
|
||||
|
||||
A static page that shows the lowering chain for committed fixture packets: typed slots, inventory entries chosen, lowered English, scorer closeness to the corpus target, with hedge, certainty and restraint annotated. Refusals (admission rules, `LowerError`, addressee consent, negation awaiting polarity) are results, not omissions.
|
||||
|
||||
- `crates/weksa/examples/show_working.rs`: runs the real admission door, `lower` and `pronounce` over `proof_packet()` and its variants and prints JSON. Each feature is removed in turn and the packet re-run, so "the lowerer read this" is measured.
|
||||
- `build.py`: runs that, scores each lowered text with the `weksa score` CLI (needs Ollama with `qwen3-embedding:0.6b`), writes `out/data.json` and one dependency-free `out/index.html`. Fails if anything cannot run; there is no fallback text.
|
||||
- `scripts/publish-show-working.sh`: build, then publish through gamecult-ops `publish-mind-preview.sh`.
|
||||
|
||||
Adding a fixture: add an entry in `fixtures()` in the example. A packet from another store needs a loader there; none exists yet.
|
||||
|
|
@ -1,217 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Build the "Show her working" page from the real lowerer and scorer.
|
||||
|
||||
usage: build.py [out_dir] (run from the repository root)
|
||||
|
||||
1. `cargo run --example show_working` admits each fixture packet through the
|
||||
packet store and lowers it with the English lowerer (crates/weksa/examples).
|
||||
2. `weksa score` scores each lowered text against its corpus item (and the
|
||||
other committed Nibu reference) through the Ollama embedder.
|
||||
3. out_dir/data.json holds the joined result; out_dir/index.html is one file
|
||||
with no scripts to fetch and no network calls.
|
||||
|
||||
Nothing on the page is typed by hand except the layout and the reading notes.
|
||||
If cargo or the scorer cannot run, the build fails: there is no fallback text.
|
||||
Standard library only.
|
||||
"""
|
||||
import json
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from html import escape as e
|
||||
|
||||
root = pathlib.Path.cwd()
|
||||
out_dir = pathlib.Path(sys.argv[1] if len(sys.argv) > 1 else "tools/show-working/out")
|
||||
weksa_bin = None
|
||||
|
||||
|
||||
def run(cmd, **kw):
|
||||
r = subprocess.run(cmd, cwd=root, capture_output=True, text=True, **kw)
|
||||
if r.returncode != 0:
|
||||
sys.exit(f"{' '.join(map(str, cmd))} failed:\n{r.stderr}")
|
||||
return r.stdout
|
||||
|
||||
|
||||
def cargo_target():
|
||||
meta = json.loads(run(["cargo", "metadata", "--format-version", "1", "--no-deps"]))
|
||||
return pathlib.Path(meta["target_directory"])
|
||||
|
||||
|
||||
# -- 1. the real lowerer ------------------------------------------------------
|
||||
run(["cargo", "build", "--release", "--bin", "weksa", "--example", "show_working"])
|
||||
target = cargo_target() / "release"
|
||||
weksa_bin = target / "weksa"
|
||||
data = json.loads(run([str(target / "examples" / "show_working")]))
|
||||
|
||||
# -- 2. the real scorer -------------------------------------------------------
|
||||
REFS_EXTRA = ["item:nibu:2"]
|
||||
# The page publishes only the corpus items its fixtures cite (plus the extra
|
||||
# reference each lowered fixture is scored against). A Validation or Test item
|
||||
# is never published, even when hand-added to REFS_EXTRA or cited by a fixture:
|
||||
# the dump holds every split, and the guard below reads each item's split.
|
||||
cited = {fx["corpus_item"] for fx in data["fixtures"]} | set(REFS_EXTRA)
|
||||
corpus = {}
|
||||
for m in re.finditer(r'^(item:\S+) \[(.*?)\]\n "(.*)"$', run([str(weksa_bin), "corpus", "dump"]), re.M):
|
||||
if m.group(1) in cited:
|
||||
corpus[m.group(1)] = {"text": m.group(3), "meta": m.group(2)}
|
||||
if set(corpus) != cited:
|
||||
sys.exit(f"corpus dump lacks cited items {sorted(cited - set(corpus))}; unexpected {sorted(set(corpus) - cited)}")
|
||||
# The dump's meta ends with the split (`... group=g Validation`, `... excluded`).
|
||||
# Only Train and split-less items publish; any other name, including a split
|
||||
# added later, is held out until it is allowed here.
|
||||
held_out = {k: v["meta"].split()[-1] for k, v in corpus.items() if v["meta"].split()[-1] not in ("Train", "excluded")}
|
||||
if held_out:
|
||||
sys.exit(f"refusing to publish held-out items (nothing written): {held_out}")
|
||||
model = "qwen3-embedding:0.6b"
|
||||
|
||||
|
||||
def score(text, items):
|
||||
out = run([str(weksa_bin), "score", *items], input=text)
|
||||
got = {}
|
||||
for m in re.finditer(r"^(item:\S+) chrf=([\d.]+) meaning=(-?[\d.]+)", out, re.M):
|
||||
got[m.group(1)] = {"chrf": float(m.group(2)), "meaning": float(m.group(3))}
|
||||
if set(got) != set(items):
|
||||
sys.exit(f"scorer answered {sorted(got)} for {items}:\n{out}")
|
||||
return got
|
||||
|
||||
|
||||
for fx in data["fixtures"]:
|
||||
o = fx["outcome"]
|
||||
if o["kind"] == "lowered":
|
||||
refs = [fx["corpus_item"]] + [r for r in REFS_EXTRA if r != fx["corpus_item"]]
|
||||
o["scores"] = [{"item": k, "target": k == fx["corpus_item"], **v} for k, v in score(o["text"], refs).items()]
|
||||
|
||||
data["corpus"] = corpus
|
||||
data["build"] = {
|
||||
"commit": run(["git", "rev-parse", "HEAD"]).strip(),
|
||||
"embedder": model,
|
||||
"built_unix": int(time.time()),
|
||||
}
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
(out_dir / "data.json").write_text(json.dumps(data, indent=1, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
# -- 3. the page --------------------------------------------------------------
|
||||
CSS = """
|
||||
:root{--ink:#1c1b19;--mute:#5d5a54;--line:#cfc9bd;--bg:#f6f3ec;--card:#fff;--ok:#0b6b4b;--warn:#9a4a00;--bad:#9b1c2c;--bar:#365f9c}
|
||||
*{box-sizing:border-box}body{font:16px/1.55 system-ui,sans-serif;max-width:62rem;margin:2rem auto;padding:0 1rem;color:var(--ink);background:var(--bg)}
|
||||
h1{margin:0 0 .2rem}h2{margin:0 0 .4rem}h3{margin:1.2rem 0 .3rem;font-size:.85rem;text-transform:uppercase;letter-spacing:.06em;color:var(--mute)}
|
||||
code,.mono{font:13px/1.4 ui-monospace,Menlo,monospace}code{background:#ece8dd;padding:0 .25rem;border-radius:3px}
|
||||
.lede{max-width:46rem}.meta{color:var(--mute);font-size:.88rem}
|
||||
nav.pick{display:flex;flex-wrap:wrap;gap:.4rem;margin:1.2rem 0}
|
||||
nav.pick a{border:1px solid var(--line);background:var(--card);padding:.25rem .6rem;border-radius:999px;text-decoration:none;color:var(--ink);font-size:.9rem}
|
||||
nav.pick a.refused{border-color:var(--bad)}nav.pick a.lowered{border-color:var(--ok)}
|
||||
article{background:var(--card);border:1px solid var(--line);border-radius:8px;padding:1rem 1.3rem;margin:1.4rem 0}
|
||||
.chain{display:grid;grid-template-columns:repeat(4,1fr);gap:.6rem;margin:.6rem 0 1rem}
|
||||
.chain div{border:1px solid var(--line);border-radius:6px;padding:.3rem .6rem;font-size:.82rem;color:var(--mute)}
|
||||
.chain b{display:block;color:var(--ink)}
|
||||
table{border-collapse:collapse;width:100%;font-size:.9rem}th,td{text-align:left;padding:.25rem .5rem;border-bottom:1px solid #e6e1d5;vertical-align:top}th{color:var(--mute);font-weight:600}
|
||||
.out{font-size:1.15rem;padding:.6rem .9rem;border-left:4px solid var(--ok);background:#eef6f1}
|
||||
.refusal{padding:.6rem .9rem;border-left:4px solid var(--bad);background:#faeef0}.refusal b{color:var(--bad)}
|
||||
.tag{display:inline-block;font-size:.75rem;padding:0 .4rem;border-radius:3px;border:1px solid var(--line);color:var(--mute)}
|
||||
.tag.read{border-color:var(--ok);color:var(--ok)}.tag.unread{border-color:var(--warn);color:var(--warn)}.tag.refuse{border-color:var(--bad);color:var(--bad)}
|
||||
.bar{height:.7rem;background:#e4dfd2;border-radius:3px;min-width:8rem;position:relative}.bar i{display:block;height:100%;background:var(--bar);border-radius:3px}
|
||||
.ref{font-size:.88rem;color:var(--mute);margin:.2rem 0 .6rem}.ref q{color:var(--ink)}
|
||||
details{margin:.5rem 0}summary{cursor:pointer;color:var(--mute);font-size:.88rem}
|
||||
section.gaps li{margin:.3rem 0}
|
||||
@media(max-width:700px){.chain{grid-template-columns:1fr 1fr}}
|
||||
"""
|
||||
|
||||
PICK_JS = """(function(){var a=[].slice.call(document.querySelectorAll('article[data-fx]'));function show(){var h=location.hash.slice(1);var any=a.some(function(x){return x.id===h});a.forEach(function(x){x.hidden=any&&x.id!==h})}
|
||||
window.addEventListener('hashchange',show);show()})()"""
|
||||
|
||||
|
||||
def g(chosen, i):
|
||||
return next((c["gloss"] for c in chosen if c["id"] == i), "?")
|
||||
|
||||
|
||||
def val(v):
|
||||
(k, x), = v.items()
|
||||
return f"{x}‰" if k == "Grade" else str(x)
|
||||
|
||||
|
||||
def slots_html(fx):
|
||||
s, ch = fx["slots"], fx["chosen"]
|
||||
rows = []
|
||||
for r in s["meaning"]["referents"]:
|
||||
ent = f" · entity <code>{e(r['entity'])}</code>" if r["entity"] else ""
|
||||
rows.append(f"<tr><td><code>{e(r['id'])}</code></td><td>referent</td><td>{e(g(ch, r['kind']))} / {e(g(ch, r['concept']))}{ent}</td></tr>")
|
||||
for p in s["meaning"]["predications"]:
|
||||
roles = ", ".join(f"{e(g(ch, x['role']))}=<code>{e(x['filler'])}</code>" for x in p["roles"])
|
||||
rows.append(f"<tr><td><code>{e(p['id'])}</code></td><td>predication</td><td>{e(g(ch, p['concept']))} in frame {e(g(ch, p['frame']))}: {roles}</td></tr>")
|
||||
for r in s["meaning"]["relations"]:
|
||||
rows.append(f"<tr><td></td><td>relation</td><td>{e(g(ch, r['relation']))}: <code>{e(r['from'])}</code> → <code>{e(r['to'])}</code></td></tr>")
|
||||
meaning = "<table><tr><th>node</th><th>kind</th><th>content</th></tr>" + "".join(rows) + "</table>"
|
||||
d = s["discourse"]
|
||||
drows = []
|
||||
if d["speech_act"]:
|
||||
drows.append(f"<tr><td>speech act</td><td>{e(g(ch, d['speech_act']))}</td></tr>")
|
||||
if d["addressee"]:
|
||||
drows.append(f"<tr><td>addressee</td><td><code>{e(d['addressee'])}</code></td></tr>")
|
||||
if d["addressee_consent"]:
|
||||
drows.append(f"<tr><td>addressee consent</td><td>predication <code>{e(d['addressee_consent'])}</code> (what the addressee asked for)</td></tr>")
|
||||
drows.append(f"<tr><td>source</td><td>{e(s['provenance']['source'])}, kind {e(s['kind'])}</td></tr>")
|
||||
return (f"<h3>Slot: meaning</h3>{meaning}<h3>Slot: discourse</h3><table>{''.join(drows)}</table>"
|
||||
+ stance_register(fx))
|
||||
|
||||
|
||||
def stance_register(fx):
|
||||
s = fx["slots"]
|
||||
eff = fx["outcome"].get("feature_effects") or []
|
||||
rows = []
|
||||
for i, f in enumerate(s["features"]):
|
||||
note = eff[i] if i < len(eff) else ""
|
||||
cls = "read" if "read it" in note else "unread" if "did not read" in note else "refuse" if "refus" in note else ""
|
||||
rows.append(f"<tr><td>{e(f['gloss'])} <span class='meta'>{e(f['id'])}</span></td><td>{e(f['slot'])} @ {e(f['at'])}</td><td>{e(val(f['value']))}</td><td><span class='tag {cls}'>{e(note)}</span></td></tr>")
|
||||
feats = ("<table><tr><th>feature</th><th>slot</th><th>value</th><th>measured effect on the lowerer</th></tr>" + "".join(rows) + "</table>") if rows else "<p class='meta'>No features: no hedge, certainty, polarity, irony or stance set.</p>"
|
||||
return ("<h3>Slot: stance, hedge, certainty, restraint (features)</h3>" + feats
|
||||
+ "<p class='meta'>Register: the packet has no register slot. Register and address belong to a target-language profile, not to the packet; this lowerer reads none.</p>")
|
||||
|
||||
|
||||
def outcome_html(fx):
|
||||
o = fx["outcome"]
|
||||
if o["kind"] == "lowered":
|
||||
ipa = o["ipa"]
|
||||
ipa_s = ("IPA of cited entities: <span class='mono'>" + e(" ".join(ipa["ok"])) + "</span>") if "ok" in ipa else f"IPA refused: {e(ipa['error']['reason'])} <code>{e(ipa['error']['variant'])}</code>"
|
||||
sc = []
|
||||
for s in o["scores"]:
|
||||
ref = corpus[s["item"]]["text"]
|
||||
sc.append(f"<div class='ref'>{'Target' if s['target'] else 'Other reference'} <code>{e(s['item'])}</code>: <q>{e(ref)}</q></div>"
|
||||
f"<table><tr><td width=110>chrF {s['chrf']:.1f}</td><td><div class='bar'><i style='width:{min(s['chrf'],100):.0f}%'></i></div></td>"
|
||||
f"<td width=150>meaning cosine {s['meaning']:.3f}</td><td><div class='bar'><i style='width:{max(s['meaning'],0)*100:.0f}%'></i></div></td></tr></table>")
|
||||
return (f"<h3>Lowered English</h3><div class='out'>{e(o['text'])}</div><p class='meta'>{ipa_s}</p>"
|
||||
f"<h3>Closeness to the corpus ({e(data['build']['embedder'])})</h3>{''.join(sc)}")
|
||||
if o["kind"] == "lower_error":
|
||||
er = o["error"]
|
||||
return f"<h3>Refused by the lowerer</h3><div class='refusal'><b>LowerError::{e(er['variant'])}</b>: {e(er['reason'])}</div><p class='meta'>No text, so no score: nothing was invented to score.</p>"
|
||||
items = "".join(f"<li><b>{e(r['rule'])}</b> <span class='mono'>{e(r['detail'])}</span></li>" for r in o["refusals"])
|
||||
return f"<h3>Refused at admission</h3><div class='refusal'><ul>{items}</ul></div><p class='meta'>The packet never became a lowering input, so the lowerer never saw it.</p>"
|
||||
|
||||
|
||||
def card(fx):
|
||||
o = fx["outcome"]
|
||||
ch = "".join(f"<tr><td><code>{e(c['id'])}</code></td><td>{e(c['gloss'] or '(unresolved)')}</td></tr>" for c in fx["chosen"])
|
||||
steps = [("1 Typed slots", "meaning, stance, discourse"), ("2 Inventory entries", f"{len(fx['chosen'])} ids, rev {fx['slots']['inventory_rev']}"),
|
||||
("3 Lowered English", {"lowered": "text", "lower_error": "refused", "admission_refused": "never reached"}[o["kind"]]),
|
||||
("4 Closeness", "scored" if o["kind"] == "lowered" else "no text, no score")]
|
||||
chain = "".join(f"<div><b>{e(a)}</b>{e(b)}</div>" for a, b in steps)
|
||||
return (f"<article id='{e(fx['id'])}' data-fx><h2>{e(fx['title'])}</h2><p>{e(fx['note'])}</p>"
|
||||
f"<p class='meta'>Packet <code>{e(fx['packet_id'])}</code> for <code>{e(fx['corpus_item'])}</code>. Made from: {e(fx['source'])}.</p>"
|
||||
f"<div class='chain'>{chain}</div>{slots_html(fx)}"
|
||||
f"<details><summary>Inventory entries chosen ({len(fx['chosen'])})</summary><table><tr><th>id</th><th>gloss at this revision</th></tr>{ch}</table></details>"
|
||||
f"{outcome_html(fx)}</article>")
|
||||
|
||||
|
||||
nav = "".join(f"<a class='{'lowered' if f['outcome']['kind']=='lowered' else 'refused'}' href='#{e(f['id'])}'>{e(f['title'])}</a>" for f in data["fixtures"])
|
||||
b = data["build"]
|
||||
page = f"""<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>Weksa: show her working</title><style>{CSS}</style></head><body>
|
||||
<h1>Show her working</h1>
|
||||
<p class="lede">A meaning packet goes in. The English lowerer picks inventory entries, writes a sentence, and a scorer measures how close that sentence is to a human-written target. Every number and sentence below was produced by running the real Rust lowerer and scorer when this page was built. The refusals are results too: when the lowerer cannot say a meaning without changing it, it says so instead of dropping the part it cannot say.</p>
|
||||
<p class="meta">Built from <code>{e(b['commit'][:10])}</code>, embedder <code>{e(b['embedder'])}</code>. English only. Fixtures are the committed <code>nibu-proof-001</code> packet and variants of it, each variant described in its own card. Scores are against the committed corpus: chrF (0-100) and embedding cosine. Raw data: <a href="data.json">data.json</a>.</p>
|
||||
<nav class="pick">{nav}</nav>
|
||||
{''.join(card(f) for f in data['fixtures'])}
|
||||
<script>{PICK_JS}</script></body></html>"""
|
||||
(out_dir / "index.html").write_text(page, encoding="utf-8")
|
||||
print(f"wrote {out_dir}/index.html and data.json ({len(data['fixtures'])} fixtures)")
|
||||
Loading…
Reference in a new issue