corpus import: a public-domain work's cast and marked lines, spans labelled names_by

WorkDraft::into_import mints PublicDomain entities and items from a transcribed
work and a [[entity|form]] name-span answer; scene put's inline annotator match
becomes one fn shared with corpus import.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Weksa 2026-11-13 19:49:26 +00:00
commit 29a8dff6d5
3 changed files with 426 additions and 13 deletions

View file

@ -7,7 +7,8 @@
//! provenance, constraints and extensions, and an empty trace, so a model that
//! saw a Validation item cannot hand its words to whoever reads packets. The
//! editor writes family, gloss, value kind and slots only; `into_revision` sets
//! every id and revision number.
//! every id and revision number. The public-domain intake file (`WorkDraft`) and
//! its name-span answer (`MarkedLine`) are read here too.
use std::fmt;
use std::num::NonZeroU32;
@ -15,7 +16,12 @@ use std::num::NonZeroU32;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use crate::corpus::CorpusItemId;
use unicode_normalization::UnicodeNormalization;
use crate::corpus::{CorpusItem, CorpusItemId, NameSpan, Source, item_key};
use crate::entity::{Entity, EntityOrigin, LanguageForms};
use crate::form::is_slug;
use crate::packet::EntityId;
use crate::inventory::{
Change, Family, InvId, Inventory, InventoryEntry, InventoryRevision, Slot, ValueKind,
};
@ -358,6 +364,196 @@ impl LoweringDraft {
}
}
/// One cast member of a work: the entity and the forms the work spells it with,
/// in the work's language, citation form first.
#[derive(Clone, Debug, PartialEq, Eq, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct CastDraft {
pub entity: EntityId,
pub forms: Vec<String>,
}
/// One line of a work: its scene group, the speaker its speech prefix names, text.
#[derive(Clone, Debug, PartialEq, Eq, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct LineDraft {
pub group: String,
pub speaker: EntityId,
pub text: String,
}
/// A transcription of a public-domain work in hand: provenance, cast forms,
/// lines, scene groups and the speaker each speech prefix names. It holds no
/// distributor header, licence or trademark text, and nothing reads a URL.
#[derive(Clone, Debug, PartialEq, Eq, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct WorkDraft {
pub slug: String,
pub language: String,
pub admitted_on: String,
pub source: Source,
pub cast: Vec<CastDraft>,
pub lines: Vec<LineDraft>,
}
/// One line of a name-span answer: the line number and the line copied with
/// `[[entity:<slug>|<form>]]` around each place it names a cast member.
#[derive(Clone, Debug, PartialEq, Eq, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct MarkedLine {
pub line: u32,
pub marked: String,
}
impl MarkedLine {
pub fn parse(answer: &str) -> Result<Self, DraftError> {
from_json(answer)
}
}
/// Why an import was refused before any document was written. The text names
/// the kind and the line number only, never a value.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum ImportRefusal {
NotPublicDomain,
CastRepeated,
NoSuchLine { line: u32 },
LineMarkedTwice { line: u32 },
Group { line: u32 },
SpeakerNotInCast { line: u32 },
Markup { line: u32 },
NotInCast { line: u32 },
}
impl fmt::Display for ImportRefusal {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::NotPublicDomain => write!(f, "the work's source is not public domain"),
Self::CastRepeated => write!(f, "a cast entity is listed twice"),
Self::NoSuchLine { line } => write!(f, "the answer marks a line that is not there (line {line})"),
Self::LineMarkedTwice { line } => write!(f, "line {line} is marked twice"),
Self::Group { line } => write!(f, "the group of line {line} is not a slug"),
Self::SpeakerNotInCast { line } => write!(f, "the speaker of line {line} is not in the cast"),
Self::Markup { line } => write!(f, "the marked text of line {line} is not its text with markup"),
Self::NotInCast { line } => write!(f, "line {line} marks an entity outside the cast"),
}
}
}
impl std::error::Error for ImportRefusal {}
impl WorkDraft {
pub fn parse(file: &str) -> Result<Self, DraftError> {
from_json(file)
}
/// The cast entities and the corpus items for the lines `marked` covers, in
/// line order, each item's spans read from its markup and labelled
/// `annotator`. Decides everything before any write; whether the documents
/// are admissible is `EntityStore::put` and `CorpusStore::add`'s to refuse.
pub fn into_import(
self,
marked: &[MarkedLine],
annotator: &Annotator,
) -> Result<(Vec<Entity>, Vec<CorpusItem>), ImportRefusal> {
let Source::PublicDomain { work, .. } = &self.source else {
return Err(ImportRefusal::NotPublicDomain);
};
let mut entities = Vec::new();
for c in &self.cast {
if entities.iter().any(|e: &Entity| e.id == c.entity) {
return Err(ImportRefusal::CastRepeated);
}
entities.push(Entity {
id: c.entity.clone(),
forms: vec![LanguageForms {
language: self.language.clone(),
forms: c.forms.iter().map(|f| f.nfc().collect()).collect(),
pronunciation: None,
}],
origin: EntityOrigin::PublicDomain { work: work.clone() },
});
}
let in_cast = |id: &EntityId| entities.iter().any(|e| &e.id == id);
let mut answer: Vec<&MarkedLine> = marked.iter().collect();
answer.sort_by_key(|m| m.line);
let mut items = Vec::new();
for (i, m) in answer.iter().enumerate() {
let line = m.line;
if line == 0 || line as usize > self.lines.len() {
return Err(ImportRefusal::NoSuchLine { line });
}
if i > 0 && answer[i - 1].line == line {
return Err(ImportRefusal::LineMarkedTwice { line });
}
let draft = &self.lines[line as usize - 1];
let group = format!("{}-{}", self.slug, draft.group);
if !is_slug(&group) {
return Err(ImportRefusal::Group { line });
}
if !in_cast(&draft.speaker) {
return Err(ImportRefusal::SpeakerNotInCast { line });
}
let text: String = draft.text.nfc().collect();
if text.contains("[[") || text.contains("]]") {
return Err(ImportRefusal::Markup { line });
}
let marked_nfc: String = m.marked.nfc().collect();
let (plain, names) = match unmark(&marked_nfc) {
Some((plain, names)) if plain == text => (plain, names),
_ => return Err(ImportRefusal::Markup { line }),
};
if names.iter().any(|n| !in_cast(&n.entity)) {
return Err(ImportRefusal::NotInCast { line });
}
items.push(CorpusItem {
id: item_key(&self.slug, line),
text: plain,
language: self.language.clone(),
speaker: draft.speaker.clone(),
group,
prior_turns: Vec::new(),
source: self.source.clone(),
admitted_on: self.admitted_on.clone(),
supersedes: None,
withdrawn: false,
names,
names_by: Some(annotator.clone()),
});
}
Ok((entities, items))
}
}
/// The text of `marked` with `[[entity|form]]` replaced by `form`, and the
/// byte span of each form in that text. `None` for any malformed markup. The
/// only search is for the markup delimiters.
fn unmark(marked: &str) -> Option<(String, Vec<NameSpan>)> {
let mut out = String::with_capacity(marked.len());
let mut names = Vec::new();
let mut rest = marked;
while let Some(at) = rest.find("[[") {
out.push_str(&rest[..at]);
let inner = &rest[at + 2..];
let (id, after) = inner.split_once('|')?;
let (form, tail) = after.split_once("]]")?;
if form.is_empty() || form.contains("[[") || id.contains("]]") {
return None;
}
let entity = EntityId::parse(id).ok()?;
let start = u32::try_from(out.len()).ok()?;
out.push_str(form);
names.push(NameSpan {
entity,
start,
end: u32::try_from(out.len()).ok()?,
});
rest = tail;
}
out.push_str(rest);
Some((out, names))
}
/// `encoder:<hex16>`, a digest of the model id and the prompt digest.
pub fn encoder_id(model: &str, prompt_digest: &str) -> EncoderId {
let digest = fnv1a64(format!("weksa-encoder-v1:{model}:{prompt_digest}").as_bytes());
@ -746,4 +942,124 @@ mod tests {
assert!(entries.iter().all(|e| e.annotator == by));
assert_eq!(entries[1].examples, ["x"]);
}
const WORK: &str = r#"{"slug":"lantern-wager","language":"en-US","admitted_on":"2026-11-13",
"source":{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}},
"cast":[{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:tamsin","forms":["Tamsin","Tam"]},{"entity":"entity:wexmoor","forms":["Wexmoor"]}],
"lines":[{"group":"s1","speaker":"entity:orsa","text":"Tamsin, bring the lantern."},
{"group":"s1","speaker":"entity:tamsin","text":"Orsa, I will not, said Tam."},
{"group":"s2","speaker":"entity:wexmoor","text":"Quiet, both of you."}]}"#;
fn marked(line: u32, text: &str) -> MarkedLine {
MarkedLine { line, marked: text.into() }
}
fn model_annotator() -> Annotator {
Annotator::Model {
family: crate::corpus::ModelFamily::Anthropic,
generator: "g".into(),
}
}
fn import(
work: &str,
answer: &[MarkedLine],
) -> Result<(Vec<Entity>, Vec<CorpusItem>), ImportRefusal> {
WorkDraft::parse(work).unwrap().into_import(answer, &model_annotator())
}
#[test]
fn a_work_imports_its_cast_and_marked_lines() {
let answer = [
marked(1, "[[entity:tamsin|Tamsin]], bring the lantern."),
marked(2, "[[entity:orsa|Orsa]], I will not, said [[entity:tamsin|Tam]]."),
];
let (entities, items) = import(WORK, &answer).unwrap();
assert_eq!(entities.len(), 3);
for e in &entities {
assert!(matches!(&e.origin, EntityOrigin::PublicDomain { work } if work == "The Lantern Wager"));
assert_eq!(e.forms[0].language, "en-US");
}
assert_eq!(items.len(), 2);
assert_eq!(items[0].id, "item:lantern-wager:1");
assert_eq!(items[1].id, "item:lantern-wager:2");
assert_eq!(items[0].group, "lantern-wager-s1");
let span = |e: &str, text: &str, form: &str| {
let start = text.find(form).unwrap() as u32;
NameSpan { entity: EntityId::parse(e).unwrap(), start, end: start + form.len() as u32 }
};
let t2 = &items[1].text;
assert_eq!(items[0].names, [span("entity:tamsin", &items[0].text, "Tamsin")]);
assert_eq!(
items[1].names,
[span("entity:orsa", t2, "Orsa"), span("entity:tamsin", t2, "Tam")]
);
// "will" is unmarked and unspanned; the line's speaker is the file's.
assert_eq!(items[1].speaker, EntityId::parse("entity:tamsin").unwrap());
for i in &items {
assert_eq!(i.names_by, Some(model_annotator()));
assert!(i.prior_turns.is_empty() && i.supersedes.is_none() && !i.withdrawn);
}
}
#[test]
fn an_import_refuses_before_any_document() {
let ok = marked(1, "Tamsin, bring the lantern.");
let refused = |work: &str, answer: &[MarkedLine]| import(work, answer).unwrap_err();
let cast_authored = WORK.replace(
r#"{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}}"#,
r#""OperatorWritten""#,
);
assert_eq!(refused(&cast_authored, std::slice::from_ref(&ok)), ImportRefusal::NotPublicDomain);
let repeated = WORK.replace(r#"{"entity":"entity:orsa","forms":["Orsa"]}"#, r#"{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:orsa","forms":["O"]}"#);
assert_eq!(refused(&repeated, std::slice::from_ref(&ok)), ImportRefusal::CastRepeated);
let stranger = WORK.replace(r#""speaker":"entity:orsa""#, r#""speaker":"entity:stranger""#);
assert_eq!(refused(&stranger, std::slice::from_ref(&ok)), ImportRefusal::SpeakerNotInCast { line: 1 });
assert_eq!(refused(WORK, &[marked(9, "x")]), ImportRefusal::NoSuchLine { line: 9 });
assert_eq!(refused(WORK, &[marked(0, "x")]), ImportRefusal::NoSuchLine { line: 0 });
assert_eq!(
refused(WORK, &[ok.clone(), ok.clone()]),
ImportRefusal::LineMarkedTwice { line: 1 }
);
for bad in [
"[[entity:tamsin|Tamsin, bring the lantern.",
"[[entity:tamsin Tamsin]], bring the lantern.",
"[[tamsin|Tamsin]], bring the lantern.",
"[[entity:tamsin|]]Tamsin, bring the lantern.",
"Tamsin, bring the lantern",
] {
assert_eq!(refused(WORK, &[marked(1, bad)]), ImportRefusal::Markup { line: 1 }, "{bad}");
}
assert_eq!(
refused(WORK, &[marked(1, "[[entity:stranger|Tamsin]], bring the lantern.")]),
ImportRefusal::NotInCast { line: 1 }
);
let group = WORK.replace(r#""group":"s1""#, r#""group":"S 1""#);
assert_eq!(refused(&group, &[ok]), ImportRefusal::Group { line: 1 });
let bracketed = WORK.replace("Tamsin, bring the lantern.", "Tamsin [[bring]] the lantern.");
assert_eq!(
refused(&bracketed, &[marked(1, "[[entity:tamsin|Tamsin]] [[entity:orsa|bring]] the lantern.")]),
ImportRefusal::Markup { line: 1 }
);
// The refusal text names the kind and the line, no value.
assert_eq!(
ImportRefusal::Markup { line: 1 }.to_string(),
"the marked text of line 1 is not its text with markup"
);
}
#[test]
fn markup_offsets_are_bytes_of_the_nfc_text() {
let work = WORK
.replace("Tamsin, bring the lantern.", "Caf\u{e9} Zo\u{eb}")
.replace(r#""forms":["Orsa"]"#, r#""forms":["Zoe\u0308"]"#);
let decomposed = "Cafe\u{301} [[entity:orsa|Zoe\u{308}]]";
let (entities, items) = import(&work, &[marked(1, decomposed)]).unwrap();
assert_eq!(entities[0].forms[0].forms, ["Zo\u{eb}"]);
let item = &items[0];
assert_eq!(item.text, "Caf\u{e9} Zo\u{eb}");
let n = &item.names[0];
assert_eq!(&item.text[n.start as usize..n.end as usize], "Zo\u{eb}");
assert_eq!(n.start, 6);
}
}

View file

@ -8,7 +8,7 @@ use clap::error::{ContextKind, ContextValue, ErrorKind};
use clap::{Args, Parser, Subcommand, ValueEnum};
use unicode_normalization::UnicodeNormalization;
use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn};
use weksa::encode::{EditDraft, OntologyDraft};
use weksa::encode::{EditDraft, MarkedLine, OntologyDraft, WorkDraft};
use weksa::entity::{Entity, EntityOrigin, LanguageForms, Pronunciation};
use weksa::inventory::{proof_revision, seed, speaker_scene_revision};
use weksa::packet::{FeatValue, FeatureValue, Grade};
@ -242,6 +242,22 @@ enum CorpusCommand {
Withdraw {
key: String,
},
/// Import a transcribed public-domain work: mint its cast as entities and
/// add each line a name-span answer covers (`docs/name-span-annotation.md`).
Import {
/// The work file: provenance, cast, lines.
work: PathBuf,
/// The annotator's answer: one `{"line": N, "marked": "..."}` per line.
names: PathBuf,
#[arg(long, requires = "generator", conflicts_with = "operator")]
model_family: Option<ModelFamilyArg>,
/// The attestation string naming the annotating model.
#[arg(long, requires = "model_family")]
generator: Option<String>,
/// Annotated by the operator, not a model.
#[arg(long)]
operator: bool,
},
}
#[derive(Subcommand)]
@ -681,6 +697,7 @@ impl AddArgs {
supersedes: self.supersedes,
withdrawn: false,
names,
names_by: None,
})
}
}
@ -698,6 +715,22 @@ fn split_text(item: &CorpusItem) -> String {
.map_or_else(|| "excluded".to_string(), |s| format!("{s:?}"))
}
/// The annotator the flags name: a model family with its generator, or the operator.
fn annotator(
model_family: Option<ModelFamilyArg>,
generator: Option<String>,
operator: bool,
) -> Result<Annotator> {
match (model_family, generator, operator) {
(Some(family), Some(generator), false) => Ok(Annotator::Model {
family: family.into(),
generator,
}),
(None, None, true) => Ok(Annotator::Operator),
_ => bail!("name the annotator: --model-family with --generator, or --operator"),
}
}
fn refused(e: StoreError) -> anyhow::Error {
anyhow::anyhow!("{e}")
}
@ -737,13 +770,46 @@ fn main() -> Result<()> {
.map_err(refused)?;
println!("{outcome:?} {key}");
}
Command::Corpus(CorpusCommand::Import {
work,
names,
model_family,
generator,
operator,
}) => {
let annotator = annotator(model_family, generator, operator)?;
let draft = WorkDraft::parse(&std::fs::read_to_string(&work)?)?;
let mut marked = Vec::new();
for (i, line) in std::fs::read_to_string(&names)?.lines().enumerate() {
if line.trim().is_empty() {
continue;
}
marked.push(
MarkedLine::parse(line)
.map_err(|e| anyhow::anyhow!("names line {}: {e}", i + 1))?,
);
}
let (new_entities, items) = draft.into_import(&marked, &annotator)?;
let mut entities = EntityStore::open(&cli.entities)?;
for e in new_entities {
let id = e.id.clone();
let outcome = entities.put(e).map_err(refused)?;
println!("{outcome:?} {id}");
}
let mut store = CorpusStore::open(&cli.corpus)?;
for item in items {
let (key, split) = (item.id.clone(), split_text(&item));
let outcome = store.add(item, &entities).map_err(refused)?;
println!("{outcome:?} {key} {split}");
}
}
Command::Corpus(CorpusCommand::Dump) => {
for i in CorpusStore::open(&cli.corpus)?.all()? {
let split = split_text(&i);
let withdrawn = if i.withdrawn { " withdrawn" } else { "" };
println!(
"{} [{} {} group={} {split}{withdrawn}]\n {:?}\n names: {:?}\n source: {:?}",
i.id, i.language, i.speaker, i.group, i.text, i.names, i.source
"{} [{} {} group={} {split}{withdrawn}]\n {:?}\n names: {:?} by {:?}\n source: {:?}",
i.id, i.language, i.speaker, i.group, i.text, i.names, i.names_by, i.source
);
}
}
@ -880,14 +946,7 @@ fn main() -> Result<()> {
generator,
operator,
}) => {
let annotator = match (model_family, generator, operator) {
(Some(family), Some(generator), false) => Annotator::Model {
family: family.into(),
generator,
},
(None, None, true) => Annotator::Operator,
_ => bail!("name the annotator: --model-family with --generator, or --operator"),
};
let annotator = annotator(model_family, generator, operator)?;
let inventory_store = InventoryStore::open(&cli.inventory)?;
let inventory = inventory_store.inventory()?;
let corpus = CorpusStore::open(&cli.corpus)?;

View file

@ -2164,3 +2164,41 @@ fn inventory_admit_cap_is_256_and_a_refusal_writes_nothing() {
assert!(admitted.status.success(), "{}", out(&admitted));
assert!(out(&admitted).contains("admitted rev:2 with 256 entries"));
}
#[test]
fn corpus_import_labels_spans_and_is_idempotent() {
let d = tempfile::tempdir().unwrap();
let work = d.path().join("work.json");
std::fs::write(
&work,
r#"{"slug":"lantern-wager","language":"en-US","admitted_on":"2026-11-13",
"source":{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}},
"cast":[{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:tamsin","forms":["Tamsin","Tam"]}],
"lines":[{"group":"s1","speaker":"entity:orsa","text":"Tamsin, bring the lantern."},
{"group":"s1","speaker":"entity:tamsin","text":"Orsa, I will not."}]}"#,
)
.unwrap();
let names = d.path().join("names.jsonl");
std::fs::write(
&names,
concat!(
r#"{"line":1,"marked":"[[entity:tamsin|Tamsin]], bring the lantern."}"#, "\n\n",
r#"{"line":2,"marked":"[[entity:orsa|Orsa]], I will not."}"#, "\n",
),
)
.unwrap();
let (w, n) = (work.to_str().unwrap(), names.to_str().unwrap());
let bare = weksa(d.path(), &["corpus", "import", w, n]);
assert!(!bare.status.success());
let flags = ["--model-family", "anthropic", "--generator", "a name annotator"];
let run = || weksa(d.path(), &[&["corpus", "import", w, n][..], &flags].concat());
let first = run();
assert!(first.status.success(), "{}", out(&first));
let text = out(&first);
assert!(text.contains("Added item:lantern-wager:1 ") && text.contains("Added item:lantern-wager:2 "), "{text}");
assert!(text.contains("Added entity:orsa"), "{text}");
let second = out(&run());
assert!(second.contains("Unchanged item:lantern-wager:1 ") && !second.contains("Added"), "{second}");
let dump = out(&weksa(d.path(), &["corpus", "dump"]));
assert!(dump.contains("by Some(Model"), "{dump}");
}