From 366a2a50a4e82cd2de5584a57baf76c37c255336 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 19:49:26 +0000 Subject: [PATCH 1/7] CorpusItem.names_by: who placed an item's name spans (key 11, default None) Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/corpus.rs | 6 ++ crates/weksa/src/split.rs | 1 + crates/weksa/tests/common/mod.rs | 1 + crates/weksa/tests/entities.rs | 71 +++++++++++++++++++ crates/weksa/tests/split_schema.rs | 1 + schemas/cultnet/weksa.corpus_item.schema.json | 46 ++++++++++++ 6 files changed, 126 insertions(+) diff --git a/crates/weksa/src/corpus.rs b/crates/weksa/src/corpus.rs index 4afa662..1fb7fa1 100644 --- a/crates/weksa/src/corpus.rs +++ b/crates/weksa/src/corpus.rs @@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize}; use crate::form::{FormError, is_slug, string_form}; use crate::packet::EntityId; +use crate::speaker::Annotator; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub struct Turn { @@ -103,6 +104,11 @@ pub struct CorpusItem { /// Which entities the text names, and where. Fixed with the text at intake. #[cultcache(key = 10)] pub names: Vec, + /// Who placed `names`: the annotator of the intake that wrote them. `None` + /// when nothing records one (the cast seeds, `weksa corpus add`, the Dao De + /// Jing bed); it labels the spans, never the line or its speaker. + #[cultcache(key = 11, default)] + pub names_by: Option, } impl CorpusItem { diff --git a/crates/weksa/src/split.rs b/crates/weksa/src/split.rs index 898439d..d22d3db 100644 --- a/crates/weksa/src/split.rs +++ b/crates/weksa/src/split.rs @@ -208,6 +208,7 @@ pub(crate) mod fixture { language: "en-US".into(), speaker: EntityId::parse("entity:nibu").unwrap(), names: vec![], + names_by: None, group: group.into(), prior_turns: vec![], source, diff --git a/crates/weksa/tests/common/mod.rs b/crates/weksa/tests/common/mod.rs index fa2deb9..8ee0596 100644 --- a/crates/weksa/tests/common/mod.rs +++ b/crates/weksa/tests/common/mod.rs @@ -205,6 +205,7 @@ pub fn item_in(key: &str, text: &str, group: &str, source: Source) -> CorpusItem language: "en-US".into(), speaker: eid("nibu"), names: vec![], + names_by: None, group: group.into(), prior_turns: vec![], source, diff --git a/crates/weksa/tests/entities.rs b/crates/weksa/tests/entities.rs index 5f1f7cf..03d139d 100644 --- a/crates/weksa/tests/entities.rs +++ b/crates/weksa/tests/entities.rs @@ -502,3 +502,74 @@ fn a_malformed_pronunciation_is_refused() { } assert_eq!(store.get(&eid("kay")).unwrap(), None); } + +const LANTERN: &str = r#"{"slug":"lantern-wager","language":"en-US","admitted_on":"2026-11-13", + "source":{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}}, + "cast":[{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:tamsin","forms":["Tamsin","Tam"]}], + "lines":[{"group":"s1","speaker":"entity:orsa","text":"Tamsin, bring the lantern."}, + {"group":"s1","speaker":"entity:tamsin","text":"Tamsin's lantern, said Orsa."}, + {"group":"s2","speaker":"entity:orsa","text":"Quiet, both of you."}]}"#; + +fn marked(line: u32, text: &str) -> weksa::encode::MarkedLine { + weksa::encode::MarkedLine { line, marked: text.into() } +} + +#[test] +fn an_imported_work_passes_the_store_doors() { + use weksa::Annotator; + use weksa::corpus::ModelFamily; + let d = tempfile::tempdir().unwrap(); + let mut entities = EntityStore::open(&d.path().join("e.cc")).unwrap(); + let mut corpus = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + let model = Annotator::Model { family: ModelFamily::Anthropic, generator: "g".into() }; + let work = || weksa::encode::WorkDraft::parse(LANTERN).unwrap(); + let (cast, items) = work() + .into_import(&[marked(1, "[[entity:tamsin|Tamsin]], bring the lantern.")], &model) + .unwrap(); + for e in cast { + assert_eq!(entities.put(e).unwrap(), AddOutcome::Added); + } + for i in items.clone() { + assert_eq!(corpus.add(i, &entities).unwrap(), AddOutcome::Added); + } + drop(corpus); + let mut corpus = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + let stored = corpus.get("item:lantern-wager:1").unwrap().unwrap(); + assert_eq!(stored.names_by, Some(model.clone())); + assert_eq!(stored.names.len(), 1); + assert_eq!(corpus.add(items[0].clone(), &entities).unwrap(), AddOutcome::Unchanged); + + // A form the entity does not list is the store's refusal, not the importer's. + let (_, bad) = work() + .into_import(&[marked(2, "[[entity:tamsin|Tamsin's]] lantern, said Orsa.")], &model) + .unwrap(); + assert!(matches!( + refusal(corpus.add(bad[0].clone(), &entities)), + StoreRefusal::NameSpanNotAForm { .. } + )); + + // An operator run over another line labels its own spans. + let (_, op) = work() + .into_import(&[marked(3, "Quiet, both of you.")], &Annotator::Operator) + .unwrap(); + assert_eq!(corpus.add(op[0].clone(), &entities).unwrap(), AddOutcome::Added); + assert_eq!( + corpus.get("item:lantern-wager:3").unwrap().unwrap().names_by, + Some(Annotator::Operator) + ); + // The first run's line keeps its label. + assert_eq!(corpus.get("item:lantern-wager:1").unwrap().unwrap().names_by, Some(model)); +} + +#[test] +fn committed_items_read_with_no_names_label() { + let data = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data"); + for sub in ["daodejing", "original"] { + let d = tempfile::tempdir().unwrap(); + let copy = d.path().join("c.cc"); + std::fs::copy(data.join(sub).join("corpus.cc"), ©).unwrap(); + let items = CorpusStore::open(©).unwrap().all().unwrap(); + assert!(!items.is_empty(), "{sub}"); + assert!(items.iter().all(|i| i.names_by.is_none()), "{sub}"); + } +} diff --git a/crates/weksa/tests/split_schema.rs b/crates/weksa/tests/split_schema.rs index bb6bc78..19749de 100644 --- a/crates/weksa/tests/split_schema.rs +++ b/crates/weksa/tests/split_schema.rs @@ -9,6 +9,7 @@ fn item(group: &str, source: Source) -> CorpusItem { language: "en-US".into(), speaker: weksa::EntityId::parse("entity:s").unwrap(), names: vec![], + names_by: None, group: group.into(), prior_turns: vec![], source, diff --git a/schemas/cultnet/weksa.corpus_item.schema.json b/schemas/cultnet/weksa.corpus_item.schema.json index 1bd6083..3eba155 100644 --- a/schemas/cultnet/weksa.corpus_item.schema.json +++ b/schemas/cultnet/weksa.corpus_item.schema.json @@ -24,6 +24,17 @@ "$ref": "#/$defs/NameSpan" } }, + "names_by": { + "description": "Who placed `names`: the annotator of the intake that wrote them. `None`\nwhen nothing records one (the cast seeds, `weksa corpus add`, the Dao De\nJing bed); it labels the spans, never the line or its speaker.", + "anyOf": [ + { + "$ref": "#/$defs/Annotator" + }, + { + "type": "null" + } + ] + }, "prior_turns": { "type": "array", "items": { @@ -63,6 +74,41 @@ "names" ], "$defs": { + "Annotator": { + "description": "Who annotated a document. A model's generator is an attestation string, held\nto the same form as every provenance string.", + "oneOf": [ + { + "type": "string", + "enum": [ + "Operator" + ] + }, + { + "type": "object", + "properties": { + "Model": { + "type": "object", + "properties": { + "family": { + "$ref": "#/$defs/ModelFamily" + }, + "generator": { + "type": "string" + } + }, + "required": [ + "family", + "generator" + ] + } + }, + "additionalProperties": false, + "required": [ + "Model" + ] + } + ] + }, "EntityId": { "type": "string", "pattern": "^entity:[a-z0-9]([a-z0-9-]{0,46}[a-z0-9])?$" -- 2.47.3 From 29a8dff6d514973d36cc652f477d48f5f4cb83f5 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 19:49:26 +0000 Subject: [PATCH 2/7] corpus import: a public-domain work's cast and marked lines, spans labelled names_by WorkDraft::into_import mints PublicDomain entities and items from a transcribed work and a [[entity|form]] name-span answer; scene put's inline annotator match becomes one fn shared with corpus import. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/encode.rs | 320 ++++++++++++++++++++++++++++++++++++- crates/weksa/src/main.rs | 81 ++++++++-- crates/weksa/tests/cli.rs | 38 +++++ 3 files changed, 426 insertions(+), 13 deletions(-) diff --git a/crates/weksa/src/encode.rs b/crates/weksa/src/encode.rs index 1ddabd3..27fafa0 100644 --- a/crates/weksa/src/encode.rs +++ b/crates/weksa/src/encode.rs @@ -7,7 +7,8 @@ //! provenance, constraints and extensions, and an empty trace, so a model that //! saw a Validation item cannot hand its words to whoever reads packets. The //! editor writes family, gloss, value kind and slots only; `into_revision` sets -//! every id and revision number. +//! every id and revision number. The public-domain intake file (`WorkDraft`) and +//! its name-span answer (`MarkedLine`) are read here too. use std::fmt; use std::num::NonZeroU32; @@ -15,7 +16,12 @@ use std::num::NonZeroU32; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; -use crate::corpus::CorpusItemId; +use unicode_normalization::UnicodeNormalization; + +use crate::corpus::{CorpusItem, CorpusItemId, NameSpan, Source, item_key}; +use crate::entity::{Entity, EntityOrigin, LanguageForms}; +use crate::form::is_slug; +use crate::packet::EntityId; use crate::inventory::{ Change, Family, InvId, Inventory, InventoryEntry, InventoryRevision, Slot, ValueKind, }; @@ -358,6 +364,196 @@ impl LoweringDraft { } } +/// One cast member of a work: the entity and the forms the work spells it with, +/// in the work's language, citation form first. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CastDraft { + pub entity: EntityId, + pub forms: Vec, +} + +/// One line of a work: its scene group, the speaker its speech prefix names, text. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LineDraft { + pub group: String, + pub speaker: EntityId, + pub text: String, +} + +/// A transcription of a public-domain work in hand: provenance, cast forms, +/// lines, scene groups and the speaker each speech prefix names. It holds no +/// distributor header, licence or trademark text, and nothing reads a URL. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WorkDraft { + pub slug: String, + pub language: String, + pub admitted_on: String, + pub source: Source, + pub cast: Vec, + pub lines: Vec, +} + +/// One line of a name-span answer: the line number and the line copied with +/// `[[entity:|
]]` around each place it names a cast member. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MarkedLine { + pub line: u32, + pub marked: String, +} + +impl MarkedLine { + pub fn parse(answer: &str) -> Result { + from_json(answer) + } +} + +/// Why an import was refused before any document was written. The text names +/// the kind and the line number only, never a value. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ImportRefusal { + NotPublicDomain, + CastRepeated, + NoSuchLine { line: u32 }, + LineMarkedTwice { line: u32 }, + Group { line: u32 }, + SpeakerNotInCast { line: u32 }, + Markup { line: u32 }, + NotInCast { line: u32 }, +} + +impl fmt::Display for ImportRefusal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NotPublicDomain => write!(f, "the work's source is not public domain"), + Self::CastRepeated => write!(f, "a cast entity is listed twice"), + Self::NoSuchLine { line } => write!(f, "the answer marks a line that is not there (line {line})"), + Self::LineMarkedTwice { line } => write!(f, "line {line} is marked twice"), + Self::Group { line } => write!(f, "the group of line {line} is not a slug"), + Self::SpeakerNotInCast { line } => write!(f, "the speaker of line {line} is not in the cast"), + Self::Markup { line } => write!(f, "the marked text of line {line} is not its text with markup"), + Self::NotInCast { line } => write!(f, "line {line} marks an entity outside the cast"), + } + } +} + +impl std::error::Error for ImportRefusal {} + +impl WorkDraft { + pub fn parse(file: &str) -> Result { + from_json(file) + } + + /// The cast entities and the corpus items for the lines `marked` covers, in + /// line order, each item's spans read from its markup and labelled + /// `annotator`. Decides everything before any write; whether the documents + /// are admissible is `EntityStore::put` and `CorpusStore::add`'s to refuse. + pub fn into_import( + self, + marked: &[MarkedLine], + annotator: &Annotator, + ) -> Result<(Vec, Vec), ImportRefusal> { + let Source::PublicDomain { work, .. } = &self.source else { + return Err(ImportRefusal::NotPublicDomain); + }; + let mut entities = Vec::new(); + for c in &self.cast { + if entities.iter().any(|e: &Entity| e.id == c.entity) { + return Err(ImportRefusal::CastRepeated); + } + entities.push(Entity { + id: c.entity.clone(), + forms: vec![LanguageForms { + language: self.language.clone(), + forms: c.forms.iter().map(|f| f.nfc().collect()).collect(), + pronunciation: None, + }], + origin: EntityOrigin::PublicDomain { work: work.clone() }, + }); + } + let in_cast = |id: &EntityId| entities.iter().any(|e| &e.id == id); + let mut answer: Vec<&MarkedLine> = marked.iter().collect(); + answer.sort_by_key(|m| m.line); + let mut items = Vec::new(); + for (i, m) in answer.iter().enumerate() { + let line = m.line; + if line == 0 || line as usize > self.lines.len() { + return Err(ImportRefusal::NoSuchLine { line }); + } + if i > 0 && answer[i - 1].line == line { + return Err(ImportRefusal::LineMarkedTwice { line }); + } + let draft = &self.lines[line as usize - 1]; + let group = format!("{}-{}", self.slug, draft.group); + if !is_slug(&group) { + return Err(ImportRefusal::Group { line }); + } + if !in_cast(&draft.speaker) { + return Err(ImportRefusal::SpeakerNotInCast { line }); + } + let text: String = draft.text.nfc().collect(); + if text.contains("[[") || text.contains("]]") { + return Err(ImportRefusal::Markup { line }); + } + let marked_nfc: String = m.marked.nfc().collect(); + let (plain, names) = match unmark(&marked_nfc) { + Some((plain, names)) if plain == text => (plain, names), + _ => return Err(ImportRefusal::Markup { line }), + }; + if names.iter().any(|n| !in_cast(&n.entity)) { + return Err(ImportRefusal::NotInCast { line }); + } + items.push(CorpusItem { + id: item_key(&self.slug, line), + text: plain, + language: self.language.clone(), + speaker: draft.speaker.clone(), + group, + prior_turns: Vec::new(), + source: self.source.clone(), + admitted_on: self.admitted_on.clone(), + supersedes: None, + withdrawn: false, + names, + names_by: Some(annotator.clone()), + }); + } + Ok((entities, items)) + } +} + +/// The text of `marked` with `[[entity|form]]` replaced by `form`, and the +/// byte span of each form in that text. `None` for any malformed markup. The +/// only search is for the markup delimiters. +fn unmark(marked: &str) -> Option<(String, Vec)> { + let mut out = String::with_capacity(marked.len()); + let mut names = Vec::new(); + let mut rest = marked; + while let Some(at) = rest.find("[[") { + out.push_str(&rest[..at]); + let inner = &rest[at + 2..]; + let (id, after) = inner.split_once('|')?; + let (form, tail) = after.split_once("]]")?; + if form.is_empty() || form.contains("[[") || id.contains("]]") { + return None; + } + let entity = EntityId::parse(id).ok()?; + let start = u32::try_from(out.len()).ok()?; + out.push_str(form); + names.push(NameSpan { + entity, + start, + end: u32::try_from(out.len()).ok()?, + }); + rest = tail; + } + out.push_str(rest); + Some((out, names)) +} + /// `encoder:`, a digest of the model id and the prompt digest. pub fn encoder_id(model: &str, prompt_digest: &str) -> EncoderId { let digest = fnv1a64(format!("weksa-encoder-v1:{model}:{prompt_digest}").as_bytes()); @@ -746,4 +942,124 @@ mod tests { assert!(entries.iter().all(|e| e.annotator == by)); assert_eq!(entries[1].examples, ["x"]); } + + const WORK: &str = r#"{"slug":"lantern-wager","language":"en-US","admitted_on":"2026-11-13", + "source":{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}}, + "cast":[{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:tamsin","forms":["Tamsin","Tam"]},{"entity":"entity:wexmoor","forms":["Wexmoor"]}], + "lines":[{"group":"s1","speaker":"entity:orsa","text":"Tamsin, bring the lantern."}, + {"group":"s1","speaker":"entity:tamsin","text":"Orsa, I will not, said Tam."}, + {"group":"s2","speaker":"entity:wexmoor","text":"Quiet, both of you."}]}"#; + + fn marked(line: u32, text: &str) -> MarkedLine { + MarkedLine { line, marked: text.into() } + } + + fn model_annotator() -> Annotator { + Annotator::Model { + family: crate::corpus::ModelFamily::Anthropic, + generator: "g".into(), + } + } + + fn import( + work: &str, + answer: &[MarkedLine], + ) -> Result<(Vec, Vec), ImportRefusal> { + WorkDraft::parse(work).unwrap().into_import(answer, &model_annotator()) + } + + #[test] + fn a_work_imports_its_cast_and_marked_lines() { + let answer = [ + marked(1, "[[entity:tamsin|Tamsin]], bring the lantern."), + marked(2, "[[entity:orsa|Orsa]], I will not, said [[entity:tamsin|Tam]]."), + ]; + let (entities, items) = import(WORK, &answer).unwrap(); + assert_eq!(entities.len(), 3); + for e in &entities { + assert!(matches!(&e.origin, EntityOrigin::PublicDomain { work } if work == "The Lantern Wager")); + assert_eq!(e.forms[0].language, "en-US"); + } + assert_eq!(items.len(), 2); + assert_eq!(items[0].id, "item:lantern-wager:1"); + assert_eq!(items[1].id, "item:lantern-wager:2"); + assert_eq!(items[0].group, "lantern-wager-s1"); + let span = |e: &str, text: &str, form: &str| { + let start = text.find(form).unwrap() as u32; + NameSpan { entity: EntityId::parse(e).unwrap(), start, end: start + form.len() as u32 } + }; + let t2 = &items[1].text; + assert_eq!(items[0].names, [span("entity:tamsin", &items[0].text, "Tamsin")]); + assert_eq!( + items[1].names, + [span("entity:orsa", t2, "Orsa"), span("entity:tamsin", t2, "Tam")] + ); + // "will" is unmarked and unspanned; the line's speaker is the file's. + assert_eq!(items[1].speaker, EntityId::parse("entity:tamsin").unwrap()); + for i in &items { + assert_eq!(i.names_by, Some(model_annotator())); + assert!(i.prior_turns.is_empty() && i.supersedes.is_none() && !i.withdrawn); + } + } + + #[test] + fn an_import_refuses_before_any_document() { + let ok = marked(1, "Tamsin, bring the lantern."); + let refused = |work: &str, answer: &[MarkedLine]| import(work, answer).unwrap_err(); + let cast_authored = WORK.replace( + r#"{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}}"#, + r#""OperatorWritten""#, + ); + assert_eq!(refused(&cast_authored, std::slice::from_ref(&ok)), ImportRefusal::NotPublicDomain); + let repeated = WORK.replace(r#"{"entity":"entity:orsa","forms":["Orsa"]}"#, r#"{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:orsa","forms":["O"]}"#); + assert_eq!(refused(&repeated, std::slice::from_ref(&ok)), ImportRefusal::CastRepeated); + let stranger = WORK.replace(r#""speaker":"entity:orsa""#, r#""speaker":"entity:stranger""#); + assert_eq!(refused(&stranger, std::slice::from_ref(&ok)), ImportRefusal::SpeakerNotInCast { line: 1 }); + assert_eq!(refused(WORK, &[marked(9, "x")]), ImportRefusal::NoSuchLine { line: 9 }); + assert_eq!(refused(WORK, &[marked(0, "x")]), ImportRefusal::NoSuchLine { line: 0 }); + assert_eq!( + refused(WORK, &[ok.clone(), ok.clone()]), + ImportRefusal::LineMarkedTwice { line: 1 } + ); + for bad in [ + "[[entity:tamsin|Tamsin, bring the lantern.", + "[[entity:tamsin Tamsin]], bring the lantern.", + "[[tamsin|Tamsin]], bring the lantern.", + "[[entity:tamsin|]]Tamsin, bring the lantern.", + "Tamsin, bring the lantern", + ] { + assert_eq!(refused(WORK, &[marked(1, bad)]), ImportRefusal::Markup { line: 1 }, "{bad}"); + } + assert_eq!( + refused(WORK, &[marked(1, "[[entity:stranger|Tamsin]], bring the lantern.")]), + ImportRefusal::NotInCast { line: 1 } + ); + let group = WORK.replace(r#""group":"s1""#, r#""group":"S 1""#); + assert_eq!(refused(&group, &[ok]), ImportRefusal::Group { line: 1 }); + let bracketed = WORK.replace("Tamsin, bring the lantern.", "Tamsin [[bring]] the lantern."); + assert_eq!( + refused(&bracketed, &[marked(1, "[[entity:tamsin|Tamsin]] [[entity:orsa|bring]] the lantern.")]), + ImportRefusal::Markup { line: 1 } + ); + // The refusal text names the kind and the line, no value. + assert_eq!( + ImportRefusal::Markup { line: 1 }.to_string(), + "the marked text of line 1 is not its text with markup" + ); + } + + #[test] + fn markup_offsets_are_bytes_of_the_nfc_text() { + let work = WORK + .replace("Tamsin, bring the lantern.", "Caf\u{e9} Zo\u{eb}") + .replace(r#""forms":["Orsa"]"#, r#""forms":["Zoe\u0308"]"#); + let decomposed = "Cafe\u{301} [[entity:orsa|Zoe\u{308}]]"; + let (entities, items) = import(&work, &[marked(1, decomposed)]).unwrap(); + assert_eq!(entities[0].forms[0].forms, ["Zo\u{eb}"]); + let item = &items[0]; + assert_eq!(item.text, "Caf\u{e9} Zo\u{eb}"); + let n = &item.names[0]; + assert_eq!(&item.text[n.start as usize..n.end as usize], "Zo\u{eb}"); + assert_eq!(n.start, 6); + } } diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index e6a6cbb..646d823 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -8,7 +8,7 @@ use clap::error::{ContextKind, ContextValue, ErrorKind}; use clap::{Args, Parser, Subcommand, ValueEnum}; use unicode_normalization::UnicodeNormalization; use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn}; -use weksa::encode::{EditDraft, OntologyDraft}; +use weksa::encode::{EditDraft, MarkedLine, OntologyDraft, WorkDraft}; use weksa::entity::{Entity, EntityOrigin, LanguageForms, Pronunciation}; use weksa::inventory::{proof_revision, seed, speaker_scene_revision}; use weksa::packet::{FeatValue, FeatureValue, Grade}; @@ -242,6 +242,22 @@ enum CorpusCommand { Withdraw { key: String, }, + /// Import a transcribed public-domain work: mint its cast as entities and + /// add each line a name-span answer covers (`docs/name-span-annotation.md`). + Import { + /// The work file: provenance, cast, lines. + work: PathBuf, + /// The annotator's answer: one `{"line": N, "marked": "..."}` per line. + names: PathBuf, + #[arg(long, requires = "generator", conflicts_with = "operator")] + model_family: Option, + /// The attestation string naming the annotating model. + #[arg(long, requires = "model_family")] + generator: Option, + /// Annotated by the operator, not a model. + #[arg(long)] + operator: bool, + }, } #[derive(Subcommand)] @@ -681,6 +697,7 @@ impl AddArgs { supersedes: self.supersedes, withdrawn: false, names, + names_by: None, }) } } @@ -698,6 +715,22 @@ fn split_text(item: &CorpusItem) -> String { .map_or_else(|| "excluded".to_string(), |s| format!("{s:?}")) } +/// The annotator the flags name: a model family with its generator, or the operator. +fn annotator( + model_family: Option, + generator: Option, + operator: bool, +) -> Result { + match (model_family, generator, operator) { + (Some(family), Some(generator), false) => Ok(Annotator::Model { + family: family.into(), + generator, + }), + (None, None, true) => Ok(Annotator::Operator), + _ => bail!("name the annotator: --model-family with --generator, or --operator"), + } +} + fn refused(e: StoreError) -> anyhow::Error { anyhow::anyhow!("{e}") } @@ -737,13 +770,46 @@ fn main() -> Result<()> { .map_err(refused)?; println!("{outcome:?} {key}"); } + Command::Corpus(CorpusCommand::Import { + work, + names, + model_family, + generator, + operator, + }) => { + let annotator = annotator(model_family, generator, operator)?; + let draft = WorkDraft::parse(&std::fs::read_to_string(&work)?)?; + let mut marked = Vec::new(); + for (i, line) in std::fs::read_to_string(&names)?.lines().enumerate() { + if line.trim().is_empty() { + continue; + } + marked.push( + MarkedLine::parse(line) + .map_err(|e| anyhow::anyhow!("names line {}: {e}", i + 1))?, + ); + } + let (new_entities, items) = draft.into_import(&marked, &annotator)?; + let mut entities = EntityStore::open(&cli.entities)?; + for e in new_entities { + let id = e.id.clone(); + let outcome = entities.put(e).map_err(refused)?; + println!("{outcome:?} {id}"); + } + let mut store = CorpusStore::open(&cli.corpus)?; + for item in items { + let (key, split) = (item.id.clone(), split_text(&item)); + let outcome = store.add(item, &entities).map_err(refused)?; + println!("{outcome:?} {key} {split}"); + } + } Command::Corpus(CorpusCommand::Dump) => { for i in CorpusStore::open(&cli.corpus)?.all()? { let split = split_text(&i); let withdrawn = if i.withdrawn { " withdrawn" } else { "" }; println!( - "{} [{} {} group={} {split}{withdrawn}]\n {:?}\n names: {:?}\n source: {:?}", - i.id, i.language, i.speaker, i.group, i.text, i.names, i.source + "{} [{} {} group={} {split}{withdrawn}]\n {:?}\n names: {:?} by {:?}\n source: {:?}", + i.id, i.language, i.speaker, i.group, i.text, i.names, i.names_by, i.source ); } } @@ -880,14 +946,7 @@ fn main() -> Result<()> { generator, operator, }) => { - let annotator = match (model_family, generator, operator) { - (Some(family), Some(generator), false) => Annotator::Model { - family: family.into(), - generator, - }, - (None, None, true) => Annotator::Operator, - _ => bail!("name the annotator: --model-family with --generator, or --operator"), - }; + let annotator = annotator(model_family, generator, operator)?; let inventory_store = InventoryStore::open(&cli.inventory)?; let inventory = inventory_store.inventory()?; let corpus = CorpusStore::open(&cli.corpus)?; diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 7f624b8..8e3d66e 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -2164,3 +2164,41 @@ fn inventory_admit_cap_is_256_and_a_refusal_writes_nothing() { assert!(admitted.status.success(), "{}", out(&admitted)); assert!(out(&admitted).contains("admitted rev:2 with 256 entries")); } + +#[test] +fn corpus_import_labels_spans_and_is_idempotent() { + let d = tempfile::tempdir().unwrap(); + let work = d.path().join("work.json"); + std::fs::write( + &work, + r#"{"slug":"lantern-wager","language":"en-US","admitted_on":"2026-11-13", + "source":{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}}, + "cast":[{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:tamsin","forms":["Tamsin","Tam"]}], + "lines":[{"group":"s1","speaker":"entity:orsa","text":"Tamsin, bring the lantern."}, + {"group":"s1","speaker":"entity:tamsin","text":"Orsa, I will not."}]}"#, + ) + .unwrap(); + let names = d.path().join("names.jsonl"); + std::fs::write( + &names, + concat!( + r#"{"line":1,"marked":"[[entity:tamsin|Tamsin]], bring the lantern."}"#, "\n\n", + r#"{"line":2,"marked":"[[entity:orsa|Orsa]], I will not."}"#, "\n", + ), + ) + .unwrap(); + let (w, n) = (work.to_str().unwrap(), names.to_str().unwrap()); + let bare = weksa(d.path(), &["corpus", "import", w, n]); + assert!(!bare.status.success()); + let flags = ["--model-family", "anthropic", "--generator", "a name annotator"]; + let run = || weksa(d.path(), &[&["corpus", "import", w, n][..], &flags].concat()); + let first = run(); + assert!(first.status.success(), "{}", out(&first)); + let text = out(&first); + assert!(text.contains("Added item:lantern-wager:1 ") && text.contains("Added item:lantern-wager:2 "), "{text}"); + assert!(text.contains("Added entity:orsa"), "{text}"); + let second = out(&run()); + assert!(second.contains("Unchanged item:lantern-wager:1 ") && !second.contains("Added"), "{second}"); + let dump = out(&weksa(d.path(), &["corpus", "dump"])); + assert!(dump.contains("by Some(Model"), "{dump}"); +} -- 2.47.3 From 0b829c7c33915f0ca3930963ec598268fdc182c3 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 19:49:26 +0000 Subject: [PATCH 3/7] Docs: name-span rubric r1 and map item 6a Co-Authored-By: Claude Sonnet 5.5 --- docs/name-span-annotation.md | 44 ++++++++++++++++++++++++ docs/targets/interlingua-lowering-map.md | 3 +- 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 docs/name-span-annotation.md diff --git a/docs/name-span-annotation.md b/docs/name-span-annotation.md new file mode 100644 index 0000000..0305158 --- /dev/null +++ b/docs/name-span-annotation.md @@ -0,0 +1,44 @@ +# Name-span annotation + +How the name spans of an imported public-domain work are placed. A span has a +committed rule that produced it; this file is that rule. + +## Why + +- `pd-annotation-model-then-attest`: a labelled model places the spans of a + public-domain work; the operator may attest them afterwards. +- `leak-owned-by-flow-and-scorer-3`: spans are fixed with the text at intake and + written by no loop writer. The offsets are derived from markup the annotator + writes, never typed by a model (inline marking, as in GPT-NER, Wang et al. + 2023); `CorpusStore::add` judges range, grapheme boundary, form equality and + overlap. + +The work file (`weksa corpus import `) is a transcription of a work in +hand: provenance, cast forms, lines, scene groups and the speaker each speech +prefix names. It holds no distributor header, licence or trademark text, and the +binary fetches nothing. + +## Rubric r1 + +You are given a work's cast: each member's entity id and the forms the work spells that member with. For each numbered line, copy the line exactly, character for character, and wrap each place where the line names a cast member by one of that member's listed forms as [[|]]. Mark a form only where it refers to that cast member; a listed form used as an ordinary word, or naming someone else, stays unmarked. Do not mark pronouns, titles or descriptions unless the exact text is a listed form. When a form is listed for two members, mark the one the line refers to; when you cannot tell, leave it unmarked. Answer one JSON line per input line, in the order given, and nothing else: {"line": N, "marked": "..."}. + +## Procedure + +- One fresh model agent per work. Its brief is rubric r1, the cast as + `: , ` lines and the work's lines as ``; + nothing else (no store, no loop, no other answer). +- Import with `--model-family` and `--generator ' (name-span + rubric r1, docs/name-span-annotation.md)'`. An operator-written or + operator-reviewed answer imports with `--operator`. +- A parse failure or a refused line is re-drawn once and logged. +- A line the answer does not cover is not imported by that run; a later run over + other lines of the work does not collide (keys are `item::`). +- A line whose own text holds `[[` or `]]` cannot be marked and is refused; it + can still enter by hand through `weksa corpus add --name`. + +## Label scope + +`names_by` labels the spans only, not the line or its speaker. One label per +item: a later attestation is a superseding item with the same text and group. +`None` means unrecorded (the cast seeds, `weksa corpus add`, the Dao De Jing +bed), not attested. diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 82d53f2..1600f6b 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -38,7 +38,7 @@ means no store holds it and nothing writes it. | Community (cut `language-ontology`, follow-up `language-ontology-communities`) | Key `community:`; type `weksa.community`, built fresh when the cut is mapped: `data/cultural-ontology/` (draft YAML profiles, read by no code) is deleted from main (cut 5e `delete-cultural-ontology`) and stays in git history before that cut's commit | Revised by commit; a speaker names the communities it belongs to by id | The cultural ontology's author (owner-attested evidence only; Discord-only content stays deleted, `discord-only-content-deleted`) | | Speaker (cut `speaker-scene`) | Key `speaker:@`; type `weksa.speaker`: stable axes as feature values of slot `Speaker` (origin, socioeconomic standing, age cohort, education and whatever the loop adds), at an inventory revision, and an `Annotator` | A new revision appends, never edits; a corpus item and every prior turn name a speaker slug, and a round records the speaker revisions it read | The annotator it names: `PersonaSnapshot` (a projection of a `gamecult.persona_state.v0` document, naming its source document and `updatedAt`), `Operator`, `WorkMetadata` (a public-domain work's own cast list and narration), or `Model { family, generator }` (the 2b `ModelFamily`) | | Scene (cut `speaker-scene`) | Key `scene:@`, the item's split group; type `weksa.scene`: participants (speaker slugs) and, per ordered pair, relation features of slot `Scene` (relative status, intimacy, in-group, audience and whatever the loop adds), at an inventory revision (provenance; a run reads the features at its own head), and an `Annotator` | As Speaker. Keyed by the group, so a scene never straddles the split. `data/original/speaker-state.cc` holds Model-annotated keeper scenes, one per Train or Validation group of both languages (`weksa scene put|dump`); a Test group has none and reads `NoScene` | As Speaker; a model-annotated scene is never Test evidence | -| Corpus item | Key `item::`, `n` canonical decimal >= 1 (no sign, no leading zero, from cut 2b); type `weksa.corpus_item` with text, language, speaker, split group, prior turns, source. Source `ModelWritten` carries a `ModelFamily` enum, the generator string as the fixture wrote it, and a prompt digest (cut 2b). The item carries no speaker state of its own: its speaker slug and group name the Speaker and Scene documents | Admitted once; the whole item is immutable (re-adding an identical item is a no-op, any difference is refused; a correction is a new item that `supersedes` the old); text already held under another key is refused unless the new item supersedes that key (cut 2b); withdrawn, never deleted, so old scores stay explainable | Importers and `weksa corpus add`, under `corpus-cast-plus-public-domain`; the source enum has no variant for Discord or any training-forbidden source (`corpus-clean`) | +| Corpus item | Key `item::`, `n` canonical decimal >= 1 (no sign, no leading zero, from cut 2b); type `weksa.corpus_item` with text, language, speaker, split group, prior turns, source and who placed its name spans (`names_by`, an `Annotator`, absent when unrecorded; cut `pd-name-spans`). Source `ModelWritten` carries a `ModelFamily` enum, the generator string as the fixture wrote it, and a prompt digest (cut 2b). The item carries no speaker state of its own: its speaker slug and group name the Speaker and Scene documents | Admitted once; the whole item is immutable (re-adding an identical item is a no-op, any difference is refused; a correction is a new item that `supersedes` the old); text already held under another key is refused unless the new item supersedes that key (cut 2b); withdrawn, never deleted, so old scores stay explainable | Importers and `weksa corpus add`, under `corpus-cast-plus-public-domain`; the source enum has no variant for Discord or any training-forbidden source (`corpus-clean`) | | Split assignment | Derived: `split(scheme, item.group)` | Fixed for a scheme; a new scheme is a new constant, never a reassignment. A model-written item whose group derives test is in no split; a superseding item keeps its predecessor's group | The hash function. Nobody assigns by hand, so there is no writer to forbid | | Loop round | Key `run: