diff --git a/crates/weksa/src/corpus.rs b/crates/weksa/src/corpus.rs index 4afa662..1fb7fa1 100644 --- a/crates/weksa/src/corpus.rs +++ b/crates/weksa/src/corpus.rs @@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize}; use crate::form::{FormError, is_slug, string_form}; use crate::packet::EntityId; +use crate::speaker::Annotator; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub struct Turn { @@ -103,6 +104,11 @@ pub struct CorpusItem { /// Which entities the text names, and where. Fixed with the text at intake. #[cultcache(key = 10)] pub names: Vec, + /// Who placed `names`: the annotator of the intake that wrote them. `None` + /// when nothing records one (the cast seeds, `weksa corpus add`, the Dao De + /// Jing bed); it labels the spans, never the line or its speaker. + #[cultcache(key = 11, default)] + pub names_by: Option, } impl CorpusItem { diff --git a/crates/weksa/src/split.rs b/crates/weksa/src/split.rs index 898439d..d22d3db 100644 --- a/crates/weksa/src/split.rs +++ b/crates/weksa/src/split.rs @@ -208,6 +208,7 @@ pub(crate) mod fixture { language: "en-US".into(), speaker: EntityId::parse("entity:nibu").unwrap(), names: vec![], + names_by: None, group: group.into(), prior_turns: vec![], source, diff --git a/crates/weksa/tests/common/mod.rs b/crates/weksa/tests/common/mod.rs index fa2deb9..8ee0596 100644 --- a/crates/weksa/tests/common/mod.rs +++ b/crates/weksa/tests/common/mod.rs @@ -205,6 +205,7 @@ pub fn item_in(key: &str, text: &str, group: &str, source: Source) -> CorpusItem language: "en-US".into(), speaker: eid("nibu"), names: vec![], + names_by: None, group: group.into(), prior_turns: vec![], source, diff --git a/crates/weksa/tests/entities.rs b/crates/weksa/tests/entities.rs index 5f1f7cf..03d139d 100644 --- a/crates/weksa/tests/entities.rs +++ b/crates/weksa/tests/entities.rs @@ -502,3 +502,74 @@ fn a_malformed_pronunciation_is_refused() { } assert_eq!(store.get(&eid("kay")).unwrap(), None); } + +const LANTERN: &str = r#"{"slug":"lantern-wager","language":"en-US","admitted_on":"2026-11-13", + "source":{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}}, + "cast":[{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:tamsin","forms":["Tamsin","Tam"]}], + "lines":[{"group":"s1","speaker":"entity:orsa","text":"Tamsin, bring the lantern."}, + {"group":"s1","speaker":"entity:tamsin","text":"Tamsin's lantern, said Orsa."}, + {"group":"s2","speaker":"entity:orsa","text":"Quiet, both of you."}]}"#; + +fn marked(line: u32, text: &str) -> weksa::encode::MarkedLine { + weksa::encode::MarkedLine { line, marked: text.into() } +} + +#[test] +fn an_imported_work_passes_the_store_doors() { + use weksa::Annotator; + use weksa::corpus::ModelFamily; + let d = tempfile::tempdir().unwrap(); + let mut entities = EntityStore::open(&d.path().join("e.cc")).unwrap(); + let mut corpus = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + let model = Annotator::Model { family: ModelFamily::Anthropic, generator: "g".into() }; + let work = || weksa::encode::WorkDraft::parse(LANTERN).unwrap(); + let (cast, items) = work() + .into_import(&[marked(1, "[[entity:tamsin|Tamsin]], bring the lantern.")], &model) + .unwrap(); + for e in cast { + assert_eq!(entities.put(e).unwrap(), AddOutcome::Added); + } + for i in items.clone() { + assert_eq!(corpus.add(i, &entities).unwrap(), AddOutcome::Added); + } + drop(corpus); + let mut corpus = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + let stored = corpus.get("item:lantern-wager:1").unwrap().unwrap(); + assert_eq!(stored.names_by, Some(model.clone())); + assert_eq!(stored.names.len(), 1); + assert_eq!(corpus.add(items[0].clone(), &entities).unwrap(), AddOutcome::Unchanged); + + // A form the entity does not list is the store's refusal, not the importer's. + let (_, bad) = work() + .into_import(&[marked(2, "[[entity:tamsin|Tamsin's]] lantern, said Orsa.")], &model) + .unwrap(); + assert!(matches!( + refusal(corpus.add(bad[0].clone(), &entities)), + StoreRefusal::NameSpanNotAForm { .. } + )); + + // An operator run over another line labels its own spans. + let (_, op) = work() + .into_import(&[marked(3, "Quiet, both of you.")], &Annotator::Operator) + .unwrap(); + assert_eq!(corpus.add(op[0].clone(), &entities).unwrap(), AddOutcome::Added); + assert_eq!( + corpus.get("item:lantern-wager:3").unwrap().unwrap().names_by, + Some(Annotator::Operator) + ); + // The first run's line keeps its label. + assert_eq!(corpus.get("item:lantern-wager:1").unwrap().unwrap().names_by, Some(model)); +} + +#[test] +fn committed_items_read_with_no_names_label() { + let data = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data"); + for sub in ["daodejing", "original"] { + let d = tempfile::tempdir().unwrap(); + let copy = d.path().join("c.cc"); + std::fs::copy(data.join(sub).join("corpus.cc"), ©).unwrap(); + let items = CorpusStore::open(©).unwrap().all().unwrap(); + assert!(!items.is_empty(), "{sub}"); + assert!(items.iter().all(|i| i.names_by.is_none()), "{sub}"); + } +} diff --git a/crates/weksa/tests/split_schema.rs b/crates/weksa/tests/split_schema.rs index bb6bc78..19749de 100644 --- a/crates/weksa/tests/split_schema.rs +++ b/crates/weksa/tests/split_schema.rs @@ -9,6 +9,7 @@ fn item(group: &str, source: Source) -> CorpusItem { language: "en-US".into(), speaker: weksa::EntityId::parse("entity:s").unwrap(), names: vec![], + names_by: None, group: group.into(), prior_turns: vec![], source, diff --git a/schemas/cultnet/weksa.corpus_item.schema.json b/schemas/cultnet/weksa.corpus_item.schema.json index 1bd6083..3eba155 100644 --- a/schemas/cultnet/weksa.corpus_item.schema.json +++ b/schemas/cultnet/weksa.corpus_item.schema.json @@ -24,6 +24,17 @@ "$ref": "#/$defs/NameSpan" } }, + "names_by": { + "description": "Who placed `names`: the annotator of the intake that wrote them. `None`\nwhen nothing records one (the cast seeds, `weksa corpus add`, the Dao De\nJing bed); it labels the spans, never the line or its speaker.", + "anyOf": [ + { + "$ref": "#/$defs/Annotator" + }, + { + "type": "null" + } + ] + }, "prior_turns": { "type": "array", "items": { @@ -63,6 +74,41 @@ "names" ], "$defs": { + "Annotator": { + "description": "Who annotated a document. A model's generator is an attestation string, held\nto the same form as every provenance string.", + "oneOf": [ + { + "type": "string", + "enum": [ + "Operator" + ] + }, + { + "type": "object", + "properties": { + "Model": { + "type": "object", + "properties": { + "family": { + "$ref": "#/$defs/ModelFamily" + }, + "generator": { + "type": "string" + } + }, + "required": [ + "family", + "generator" + ] + } + }, + "additionalProperties": false, + "required": [ + "Model" + ] + } + ] + }, "EntityId": { "type": "string", "pattern": "^entity:[a-z0-9]([a-z0-9-]{0,46}[a-z0-9])?$"