From 29a8dff6d514973d36cc652f477d48f5f4cb83f5 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 19:49:26 +0000 Subject: [PATCH] corpus import: a public-domain work's cast and marked lines, spans labelled names_by WorkDraft::into_import mints PublicDomain entities and items from a transcribed work and a [[entity|form]] name-span answer; scene put's inline annotator match becomes one fn shared with corpus import. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/encode.rs | 320 ++++++++++++++++++++++++++++++++++++- crates/weksa/src/main.rs | 81 ++++++++-- crates/weksa/tests/cli.rs | 38 +++++ 3 files changed, 426 insertions(+), 13 deletions(-) diff --git a/crates/weksa/src/encode.rs b/crates/weksa/src/encode.rs index 1ddabd3..27fafa0 100644 --- a/crates/weksa/src/encode.rs +++ b/crates/weksa/src/encode.rs @@ -7,7 +7,8 @@ //! provenance, constraints and extensions, and an empty trace, so a model that //! saw a Validation item cannot hand its words to whoever reads packets. The //! editor writes family, gloss, value kind and slots only; `into_revision` sets -//! every id and revision number. +//! every id and revision number. The public-domain intake file (`WorkDraft`) and +//! its name-span answer (`MarkedLine`) are read here too. use std::fmt; use std::num::NonZeroU32; @@ -15,7 +16,12 @@ use std::num::NonZeroU32; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; -use crate::corpus::CorpusItemId; +use unicode_normalization::UnicodeNormalization; + +use crate::corpus::{CorpusItem, CorpusItemId, NameSpan, Source, item_key}; +use crate::entity::{Entity, EntityOrigin, LanguageForms}; +use crate::form::is_slug; +use crate::packet::EntityId; use crate::inventory::{ Change, Family, InvId, Inventory, InventoryEntry, InventoryRevision, Slot, ValueKind, }; @@ -358,6 +364,196 @@ impl LoweringDraft { } } +/// One cast member of a work: the entity and the forms the work spells it with, +/// in the work's language, citation form first. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CastDraft { + pub entity: EntityId, + pub forms: Vec, +} + +/// One line of a work: its scene group, the speaker its speech prefix names, text. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LineDraft { + pub group: String, + pub speaker: EntityId, + pub text: String, +} + +/// A transcription of a public-domain work in hand: provenance, cast forms, +/// lines, scene groups and the speaker each speech prefix names. It holds no +/// distributor header, licence or trademark text, and nothing reads a URL. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WorkDraft { + pub slug: String, + pub language: String, + pub admitted_on: String, + pub source: Source, + pub cast: Vec, + pub lines: Vec, +} + +/// One line of a name-span answer: the line number and the line copied with +/// `[[entity:|
]]` around each place it names a cast member. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MarkedLine { + pub line: u32, + pub marked: String, +} + +impl MarkedLine { + pub fn parse(answer: &str) -> Result { + from_json(answer) + } +} + +/// Why an import was refused before any document was written. The text names +/// the kind and the line number only, never a value. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ImportRefusal { + NotPublicDomain, + CastRepeated, + NoSuchLine { line: u32 }, + LineMarkedTwice { line: u32 }, + Group { line: u32 }, + SpeakerNotInCast { line: u32 }, + Markup { line: u32 }, + NotInCast { line: u32 }, +} + +impl fmt::Display for ImportRefusal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NotPublicDomain => write!(f, "the work's source is not public domain"), + Self::CastRepeated => write!(f, "a cast entity is listed twice"), + Self::NoSuchLine { line } => write!(f, "the answer marks a line that is not there (line {line})"), + Self::LineMarkedTwice { line } => write!(f, "line {line} is marked twice"), + Self::Group { line } => write!(f, "the group of line {line} is not a slug"), + Self::SpeakerNotInCast { line } => write!(f, "the speaker of line {line} is not in the cast"), + Self::Markup { line } => write!(f, "the marked text of line {line} is not its text with markup"), + Self::NotInCast { line } => write!(f, "line {line} marks an entity outside the cast"), + } + } +} + +impl std::error::Error for ImportRefusal {} + +impl WorkDraft { + pub fn parse(file: &str) -> Result { + from_json(file) + } + + /// The cast entities and the corpus items for the lines `marked` covers, in + /// line order, each item's spans read from its markup and labelled + /// `annotator`. Decides everything before any write; whether the documents + /// are admissible is `EntityStore::put` and `CorpusStore::add`'s to refuse. + pub fn into_import( + self, + marked: &[MarkedLine], + annotator: &Annotator, + ) -> Result<(Vec, Vec), ImportRefusal> { + let Source::PublicDomain { work, .. } = &self.source else { + return Err(ImportRefusal::NotPublicDomain); + }; + let mut entities = Vec::new(); + for c in &self.cast { + if entities.iter().any(|e: &Entity| e.id == c.entity) { + return Err(ImportRefusal::CastRepeated); + } + entities.push(Entity { + id: c.entity.clone(), + forms: vec![LanguageForms { + language: self.language.clone(), + forms: c.forms.iter().map(|f| f.nfc().collect()).collect(), + pronunciation: None, + }], + origin: EntityOrigin::PublicDomain { work: work.clone() }, + }); + } + let in_cast = |id: &EntityId| entities.iter().any(|e| &e.id == id); + let mut answer: Vec<&MarkedLine> = marked.iter().collect(); + answer.sort_by_key(|m| m.line); + let mut items = Vec::new(); + for (i, m) in answer.iter().enumerate() { + let line = m.line; + if line == 0 || line as usize > self.lines.len() { + return Err(ImportRefusal::NoSuchLine { line }); + } + if i > 0 && answer[i - 1].line == line { + return Err(ImportRefusal::LineMarkedTwice { line }); + } + let draft = &self.lines[line as usize - 1]; + let group = format!("{}-{}", self.slug, draft.group); + if !is_slug(&group) { + return Err(ImportRefusal::Group { line }); + } + if !in_cast(&draft.speaker) { + return Err(ImportRefusal::SpeakerNotInCast { line }); + } + let text: String = draft.text.nfc().collect(); + if text.contains("[[") || text.contains("]]") { + return Err(ImportRefusal::Markup { line }); + } + let marked_nfc: String = m.marked.nfc().collect(); + let (plain, names) = match unmark(&marked_nfc) { + Some((plain, names)) if plain == text => (plain, names), + _ => return Err(ImportRefusal::Markup { line }), + }; + if names.iter().any(|n| !in_cast(&n.entity)) { + return Err(ImportRefusal::NotInCast { line }); + } + items.push(CorpusItem { + id: item_key(&self.slug, line), + text: plain, + language: self.language.clone(), + speaker: draft.speaker.clone(), + group, + prior_turns: Vec::new(), + source: self.source.clone(), + admitted_on: self.admitted_on.clone(), + supersedes: None, + withdrawn: false, + names, + names_by: Some(annotator.clone()), + }); + } + Ok((entities, items)) + } +} + +/// The text of `marked` with `[[entity|form]]` replaced by `form`, and the +/// byte span of each form in that text. `None` for any malformed markup. The +/// only search is for the markup delimiters. +fn unmark(marked: &str) -> Option<(String, Vec)> { + let mut out = String::with_capacity(marked.len()); + let mut names = Vec::new(); + let mut rest = marked; + while let Some(at) = rest.find("[[") { + out.push_str(&rest[..at]); + let inner = &rest[at + 2..]; + let (id, after) = inner.split_once('|')?; + let (form, tail) = after.split_once("]]")?; + if form.is_empty() || form.contains("[[") || id.contains("]]") { + return None; + } + let entity = EntityId::parse(id).ok()?; + let start = u32::try_from(out.len()).ok()?; + out.push_str(form); + names.push(NameSpan { + entity, + start, + end: u32::try_from(out.len()).ok()?, + }); + rest = tail; + } + out.push_str(rest); + Some((out, names)) +} + /// `encoder:`, a digest of the model id and the prompt digest. pub fn encoder_id(model: &str, prompt_digest: &str) -> EncoderId { let digest = fnv1a64(format!("weksa-encoder-v1:{model}:{prompt_digest}").as_bytes()); @@ -746,4 +942,124 @@ mod tests { assert!(entries.iter().all(|e| e.annotator == by)); assert_eq!(entries[1].examples, ["x"]); } + + const WORK: &str = r#"{"slug":"lantern-wager","language":"en-US","admitted_on":"2026-11-13", + "source":{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}}, + "cast":[{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:tamsin","forms":["Tamsin","Tam"]},{"entity":"entity:wexmoor","forms":["Wexmoor"]}], + "lines":[{"group":"s1","speaker":"entity:orsa","text":"Tamsin, bring the lantern."}, + {"group":"s1","speaker":"entity:tamsin","text":"Orsa, I will not, said Tam."}, + {"group":"s2","speaker":"entity:wexmoor","text":"Quiet, both of you."}]}"#; + + fn marked(line: u32, text: &str) -> MarkedLine { + MarkedLine { line, marked: text.into() } + } + + fn model_annotator() -> Annotator { + Annotator::Model { + family: crate::corpus::ModelFamily::Anthropic, + generator: "g".into(), + } + } + + fn import( + work: &str, + answer: &[MarkedLine], + ) -> Result<(Vec, Vec), ImportRefusal> { + WorkDraft::parse(work).unwrap().into_import(answer, &model_annotator()) + } + + #[test] + fn a_work_imports_its_cast_and_marked_lines() { + let answer = [ + marked(1, "[[entity:tamsin|Tamsin]], bring the lantern."), + marked(2, "[[entity:orsa|Orsa]], I will not, said [[entity:tamsin|Tam]]."), + ]; + let (entities, items) = import(WORK, &answer).unwrap(); + assert_eq!(entities.len(), 3); + for e in &entities { + assert!(matches!(&e.origin, EntityOrigin::PublicDomain { work } if work == "The Lantern Wager")); + assert_eq!(e.forms[0].language, "en-US"); + } + assert_eq!(items.len(), 2); + assert_eq!(items[0].id, "item:lantern-wager:1"); + assert_eq!(items[1].id, "item:lantern-wager:2"); + assert_eq!(items[0].group, "lantern-wager-s1"); + let span = |e: &str, text: &str, form: &str| { + let start = text.find(form).unwrap() as u32; + NameSpan { entity: EntityId::parse(e).unwrap(), start, end: start + form.len() as u32 } + }; + let t2 = &items[1].text; + assert_eq!(items[0].names, [span("entity:tamsin", &items[0].text, "Tamsin")]); + assert_eq!( + items[1].names, + [span("entity:orsa", t2, "Orsa"), span("entity:tamsin", t2, "Tam")] + ); + // "will" is unmarked and unspanned; the line's speaker is the file's. + assert_eq!(items[1].speaker, EntityId::parse("entity:tamsin").unwrap()); + for i in &items { + assert_eq!(i.names_by, Some(model_annotator())); + assert!(i.prior_turns.is_empty() && i.supersedes.is_none() && !i.withdrawn); + } + } + + #[test] + fn an_import_refuses_before_any_document() { + let ok = marked(1, "Tamsin, bring the lantern."); + let refused = |work: &str, answer: &[MarkedLine]| import(work, answer).unwrap_err(); + let cast_authored = WORK.replace( + r#"{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}}"#, + r#""OperatorWritten""#, + ); + assert_eq!(refused(&cast_authored, std::slice::from_ref(&ok)), ImportRefusal::NotPublicDomain); + let repeated = WORK.replace(r#"{"entity":"entity:orsa","forms":["Orsa"]}"#, r#"{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:orsa","forms":["O"]}"#); + assert_eq!(refused(&repeated, std::slice::from_ref(&ok)), ImportRefusal::CastRepeated); + let stranger = WORK.replace(r#""speaker":"entity:orsa""#, r#""speaker":"entity:stranger""#); + assert_eq!(refused(&stranger, std::slice::from_ref(&ok)), ImportRefusal::SpeakerNotInCast { line: 1 }); + assert_eq!(refused(WORK, &[marked(9, "x")]), ImportRefusal::NoSuchLine { line: 9 }); + assert_eq!(refused(WORK, &[marked(0, "x")]), ImportRefusal::NoSuchLine { line: 0 }); + assert_eq!( + refused(WORK, &[ok.clone(), ok.clone()]), + ImportRefusal::LineMarkedTwice { line: 1 } + ); + for bad in [ + "[[entity:tamsin|Tamsin, bring the lantern.", + "[[entity:tamsin Tamsin]], bring the lantern.", + "[[tamsin|Tamsin]], bring the lantern.", + "[[entity:tamsin|]]Tamsin, bring the lantern.", + "Tamsin, bring the lantern", + ] { + assert_eq!(refused(WORK, &[marked(1, bad)]), ImportRefusal::Markup { line: 1 }, "{bad}"); + } + assert_eq!( + refused(WORK, &[marked(1, "[[entity:stranger|Tamsin]], bring the lantern.")]), + ImportRefusal::NotInCast { line: 1 } + ); + let group = WORK.replace(r#""group":"s1""#, r#""group":"S 1""#); + assert_eq!(refused(&group, &[ok]), ImportRefusal::Group { line: 1 }); + let bracketed = WORK.replace("Tamsin, bring the lantern.", "Tamsin [[bring]] the lantern."); + assert_eq!( + refused(&bracketed, &[marked(1, "[[entity:tamsin|Tamsin]] [[entity:orsa|bring]] the lantern.")]), + ImportRefusal::Markup { line: 1 } + ); + // The refusal text names the kind and the line, no value. + assert_eq!( + ImportRefusal::Markup { line: 1 }.to_string(), + "the marked text of line 1 is not its text with markup" + ); + } + + #[test] + fn markup_offsets_are_bytes_of_the_nfc_text() { + let work = WORK + .replace("Tamsin, bring the lantern.", "Caf\u{e9} Zo\u{eb}") + .replace(r#""forms":["Orsa"]"#, r#""forms":["Zoe\u0308"]"#); + let decomposed = "Cafe\u{301} [[entity:orsa|Zoe\u{308}]]"; + let (entities, items) = import(&work, &[marked(1, decomposed)]).unwrap(); + assert_eq!(entities[0].forms[0].forms, ["Zo\u{eb}"]); + let item = &items[0]; + assert_eq!(item.text, "Caf\u{e9} Zo\u{eb}"); + let n = &item.names[0]; + assert_eq!(&item.text[n.start as usize..n.end as usize], "Zo\u{eb}"); + assert_eq!(n.start, 6); + } } diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index e6a6cbb..646d823 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -8,7 +8,7 @@ use clap::error::{ContextKind, ContextValue, ErrorKind}; use clap::{Args, Parser, Subcommand, ValueEnum}; use unicode_normalization::UnicodeNormalization; use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn}; -use weksa::encode::{EditDraft, OntologyDraft}; +use weksa::encode::{EditDraft, MarkedLine, OntologyDraft, WorkDraft}; use weksa::entity::{Entity, EntityOrigin, LanguageForms, Pronunciation}; use weksa::inventory::{proof_revision, seed, speaker_scene_revision}; use weksa::packet::{FeatValue, FeatureValue, Grade}; @@ -242,6 +242,22 @@ enum CorpusCommand { Withdraw { key: String, }, + /// Import a transcribed public-domain work: mint its cast as entities and + /// add each line a name-span answer covers (`docs/name-span-annotation.md`). + Import { + /// The work file: provenance, cast, lines. + work: PathBuf, + /// The annotator's answer: one `{"line": N, "marked": "..."}` per line. + names: PathBuf, + #[arg(long, requires = "generator", conflicts_with = "operator")] + model_family: Option, + /// The attestation string naming the annotating model. + #[arg(long, requires = "model_family")] + generator: Option, + /// Annotated by the operator, not a model. + #[arg(long)] + operator: bool, + }, } #[derive(Subcommand)] @@ -681,6 +697,7 @@ impl AddArgs { supersedes: self.supersedes, withdrawn: false, names, + names_by: None, }) } } @@ -698,6 +715,22 @@ fn split_text(item: &CorpusItem) -> String { .map_or_else(|| "excluded".to_string(), |s| format!("{s:?}")) } +/// The annotator the flags name: a model family with its generator, or the operator. +fn annotator( + model_family: Option, + generator: Option, + operator: bool, +) -> Result { + match (model_family, generator, operator) { + (Some(family), Some(generator), false) => Ok(Annotator::Model { + family: family.into(), + generator, + }), + (None, None, true) => Ok(Annotator::Operator), + _ => bail!("name the annotator: --model-family with --generator, or --operator"), + } +} + fn refused(e: StoreError) -> anyhow::Error { anyhow::anyhow!("{e}") } @@ -737,13 +770,46 @@ fn main() -> Result<()> { .map_err(refused)?; println!("{outcome:?} {key}"); } + Command::Corpus(CorpusCommand::Import { + work, + names, + model_family, + generator, + operator, + }) => { + let annotator = annotator(model_family, generator, operator)?; + let draft = WorkDraft::parse(&std::fs::read_to_string(&work)?)?; + let mut marked = Vec::new(); + for (i, line) in std::fs::read_to_string(&names)?.lines().enumerate() { + if line.trim().is_empty() { + continue; + } + marked.push( + MarkedLine::parse(line) + .map_err(|e| anyhow::anyhow!("names line {}: {e}", i + 1))?, + ); + } + let (new_entities, items) = draft.into_import(&marked, &annotator)?; + let mut entities = EntityStore::open(&cli.entities)?; + for e in new_entities { + let id = e.id.clone(); + let outcome = entities.put(e).map_err(refused)?; + println!("{outcome:?} {id}"); + } + let mut store = CorpusStore::open(&cli.corpus)?; + for item in items { + let (key, split) = (item.id.clone(), split_text(&item)); + let outcome = store.add(item, &entities).map_err(refused)?; + println!("{outcome:?} {key} {split}"); + } + } Command::Corpus(CorpusCommand::Dump) => { for i in CorpusStore::open(&cli.corpus)?.all()? { let split = split_text(&i); let withdrawn = if i.withdrawn { " withdrawn" } else { "" }; println!( - "{} [{} {} group={} {split}{withdrawn}]\n {:?}\n names: {:?}\n source: {:?}", - i.id, i.language, i.speaker, i.group, i.text, i.names, i.source + "{} [{} {} group={} {split}{withdrawn}]\n {:?}\n names: {:?} by {:?}\n source: {:?}", + i.id, i.language, i.speaker, i.group, i.text, i.names, i.names_by, i.source ); } } @@ -880,14 +946,7 @@ fn main() -> Result<()> { generator, operator, }) => { - let annotator = match (model_family, generator, operator) { - (Some(family), Some(generator), false) => Annotator::Model { - family: family.into(), - generator, - }, - (None, None, true) => Annotator::Operator, - _ => bail!("name the annotator: --model-family with --generator, or --operator"), - }; + let annotator = annotator(model_family, generator, operator)?; let inventory_store = InventoryStore::open(&cli.inventory)?; let inventory = inventory_store.inventory()?; let corpus = CorpusStore::open(&cli.corpus)?; diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 7f624b8..8e3d66e 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -2164,3 +2164,41 @@ fn inventory_admit_cap_is_256_and_a_refusal_writes_nothing() { assert!(admitted.status.success(), "{}", out(&admitted)); assert!(out(&admitted).contains("admitted rev:2 with 256 entries")); } + +#[test] +fn corpus_import_labels_spans_and_is_idempotent() { + let d = tempfile::tempdir().unwrap(); + let work = d.path().join("work.json"); + std::fs::write( + &work, + r#"{"slug":"lantern-wager","language":"en-US","admitted_on":"2026-11-13", + "source":{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}}, + "cast":[{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:tamsin","forms":["Tamsin","Tam"]}], + "lines":[{"group":"s1","speaker":"entity:orsa","text":"Tamsin, bring the lantern."}, + {"group":"s1","speaker":"entity:tamsin","text":"Orsa, I will not."}]}"#, + ) + .unwrap(); + let names = d.path().join("names.jsonl"); + std::fs::write( + &names, + concat!( + r#"{"line":1,"marked":"[[entity:tamsin|Tamsin]], bring the lantern."}"#, "\n\n", + r#"{"line":2,"marked":"[[entity:orsa|Orsa]], I will not."}"#, "\n", + ), + ) + .unwrap(); + let (w, n) = (work.to_str().unwrap(), names.to_str().unwrap()); + let bare = weksa(d.path(), &["corpus", "import", w, n]); + assert!(!bare.status.success()); + let flags = ["--model-family", "anthropic", "--generator", "a name annotator"]; + let run = || weksa(d.path(), &[&["corpus", "import", w, n][..], &flags].concat()); + let first = run(); + assert!(first.status.success(), "{}", out(&first)); + let text = out(&first); + assert!(text.contains("Added item:lantern-wager:1 ") && text.contains("Added item:lantern-wager:2 "), "{text}"); + assert!(text.contains("Added entity:orsa"), "{text}"); + let second = out(&run()); + assert!(second.contains("Unchanged item:lantern-wager:1 ") && !second.contains("Added"), "{second}"); + let dump = out(&weksa(d.path(), &["corpus", "dump"])); + assert!(dump.contains("by Some(Model"), "{dump}"); +}