From 74ab5082c838a38fcc421878f62df11ed656dbed Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 16:52:48 +0000 Subject: [PATCH 01/14] Red: the item's speaker prints as a fixed token with its forms (two speakers) Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/context.rs | 66 +++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs index 9d68517..25cd4fe 100644 --- a/crates/weksa/tests/context.rs +++ b/crates/weksa/tests/context.rs @@ -885,6 +885,72 @@ mod lowering { assert!(text.contains("NIBUCANARY"), "{text}"); } + /// The proof packet admitted for an item whose speaker is `speaker` and + /// whose prior turn is `other`'s, with `speaker` on referent n3 and `other` + /// on referent n1, then lowered in en-US over an entity port holding `held`. + fn lowered_for(speaker: &str, other: &str, held: &[Entity]) -> String { + let cast = InventoryStore::open(&data("interlingua/inventory.cc")).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let mut entities = EntityStore::open(&dir.path().join("e.cc")).unwrap(); + for slug in [speaker, other] { + entities.put(super::common::ent(slug, "en-US", &["Stored"])).unwrap(); + } + let mut corpus = CorpusStore::open(&dir.path().join("c.cc")).unwrap(); + let mut item = super::common::item_in("item:nibu:1", "A line.", &super::common::group_in_bucket(10), super::common::cast()); + item.speaker = super::common::eid(speaker); + item.prior_turns = vec![weksa::Turn { speaker: super::common::eid(other), text: "Well?".into() }]; + corpus.add(item, &entities).unwrap(); + let mut packet = proof_packet(); + packet.referents[0].entity = Some(super::common::eid(other)); + packet.referents[2].entity = Some(super::common::eid(speaker)); + let view = PacketStore::open(&dir.path().join("p.cc")) + .unwrap() + .admit(packet, &cast, &corpus, &entities) + .unwrap(); + let state = weksa::SpeakerStateStore::open(&dir.path().join("s.cc")).unwrap(); + let view = view.with_scene(&cast.inventory().unwrap(), &state, &corpus).unwrap(); + let item = CorpusItemId::parse("item:nibu:1").unwrap(); + Prompt::lower( + &view, + &item, + &super::common::eid(speaker), + "en-US", + &cast.inventory().unwrap(), + &Held(held.to_vec()), + &weksa::Ontology::empty(), + ) + .unwrap() + .text() + .to_string() + } + + /// Persona's ruling `speaker-alias-token-in-lowering-prompt`: the item's + /// speaker prints as one fixed role token, the same for every speaker and + /// every item, with the speaker's forms under it; its entity id never + /// prints. Another cited entity still prints under its own id. + #[test] + fn the_item_speaker_is_a_fixed_token_with_its_forms() { + const TOKEN: &str = "entity:speaker"; + let forms = |slug: &str, form: &str| super::common::ent(slug, "en-US", &[form]); + let other = forms("othercite", "OTTERBLOOM"); + let a = lowered_for("keeperzed", "othercite", &[forms("keeperzed", "ZEBRAQUILL"), other.clone()]); + let b = lowered_for("secondvoice", "othercite", &[forms("secondvoice", "MARMOTLUTE"), other.clone()]); + for (text, id, form, other_form) in [(&a, "keeperzed", "ZEBRAQUILL", "MARMOTLUTE"), (&b, "secondvoice", "MARMOTLUTE", "ZEBRAQUILL")] { + assert!(!text.contains(id), "{id} in {text}"); + assert!(!text.contains(other_form), "{other_form} in {text}"); + assert!(text.contains(r#""concept":"concept:5","entity":"entity:speaker""#), "{text}"); + assert!(text.contains(&format!(r#""{TOKEN}":["{form}"]"#)), "{text}"); + assert!(text.contains(r#""entity":"entity:othercite""#), "{text}"); + assert!(text.contains(r#""entity:othercite":["OTTERBLOOM"]"#), "{text}"); + } + // The token is one constant: the entities lines differ only in the speaker's form. + assert_eq!(a.replace("ZEBRAQUILL", "F"), b.replace("MARMOTLUTE", "F")); + // A speaker with no form in the language does not refuse and gets no token entry. + let bare = lowered_for("keeperzed", "othercite", &[other]); + assert!(!bare.contains("keeperzed") && !bare.contains(TOKEN), "{bare}"); + assert!(bare.contains(r#"{"entity:othercite":["OTTERBLOOM"]}"#), "{bare}"); + } + fn feature(inv: &Inventory, gloss: &str) -> weksa::InvId { inv.entries().find(|e| e.gloss == gloss).unwrap().id } From b97ec1d048470b853959f85f59f0a5b86fbe5bad Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 17:05:56 +0000 Subject: [PATCH 02/14] Prompt::lower prints the item's speaker as a fixed token with its forms The speaker's id never prints; its forms, shared by every item, print under the constant entity:speaker. A speaker with no form in the language names no entity and does not refuse. A referent citing the real id entity:speaker is refused (SpeakerTokenTaken). Docs state what the promise guards. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/examples/show_working.rs | 1 + crates/weksa/src/context.rs | 49 ++++++++++++++---- crates/weksa/src/lower.rs | 4 ++ crates/weksa/tests/context.rs | 66 ++++++++++++++++++------ crates/weksa/tests/golf_loop.rs | 19 +++---- docs/targets/interlingua-lowering-map.md | 2 +- 6 files changed, 103 insertions(+), 38 deletions(-) diff --git a/crates/weksa/examples/show_working.rs b/crates/weksa/examples/show_working.rs index 6c39bdb..6a31b26 100644 --- a/crates/weksa/examples/show_working.rs +++ b/crates/weksa/examples/show_working.rs @@ -260,6 +260,7 @@ fn lower_error(e: &LowerError) -> Value { LowerError::AskerNotAddressee { predication } => ("AskerNotAddressee", format!("the agent of {predication} is not the addressee")), LowerError::EntityRead { entity, .. } => ("EntityRead", format!("the entity store failed reading {entity}")), LowerError::SceneUnread => ("SceneUnread", "the view's scene was never read".to_string()), + LowerError::SpeakerTokenTaken => ("SpeakerTokenTaken", "a referent cites the entity id the speaker token uses".to_string()), }; json!({ "variant": variant, "reason": reason }) } diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index cac786f..cad1ff9 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -539,6 +539,13 @@ struct Meaning<'a> { } /// Printed in place of a prior turn whose text no live Train item holds. +/// What the lowering prompt prints in place of the item's speaker's entity id: +/// one constant, the same for every speaker and every item, so no id and +/// nothing derived from one reaches the lowerer. It parses as an entity id so +/// the packet keeps its shape. A real entity of this id is refused at lowering +/// ([`LowerError::SpeakerTokenTaken`]); the entity store does not reserve it. +pub const SPEAKER_TOKEN: &str = "entity:speaker"; + const ELIDED: &str = "[elided]"; impl Prompt { @@ -637,12 +644,18 @@ impl Prompt { /// discourse and the scene relation `with_scene` read, and `language`, /// never the caller; the scene's values are never printed, only the entries /// they select. `item` goes only to [`Prompt::item_keys`], never into the - /// text. `speaker` is the item's speaker: it is never printed. A referent - /// citing it is printed with no entity, and it has no line in `entities:` - /// and no form to find, so a speaker the language has no form for does not - /// refuse. Any other cited entity is printed with its forms. A cited - /// entity the port lacks, or one with no form in `language`, is an error - /// and no prompt. The caller chooses the language; the view has none. + /// text. `speaker` is the item's speaker: its id is never printed. The + /// promise guards the id that names an entity of this item, not the + /// speaker's forms, which every item of the language shares. A referent + /// citing the speaker is printed with [`SPEAKER_TOKEN`] as its entity, and + /// the entities line holds the speaker's forms under that token. A speaker + /// the language has no form for, or the port lacks, does not refuse: its + /// referents print no entity and the line has no entry. Any other cited + /// entity is printed under its own id with its forms; one the port lacks, + /// or one with no form in `language`, is an error and no prompt. A + /// referent citing the token's own id as an entity is + /// [`LowerError::SpeakerTokenTaken`]. The caller chooses the language; the + /// view has none. pub fn lower( input: &LoweringInput, item: &CorpusItemId, @@ -655,14 +668,28 @@ impl Prompt { if input.scene_read() == &SceneRead::Unread { return Err(LowerError::SceneUnread); } - let mut forms: BTreeMap<&str, Vec> = BTreeMap::new(); + let token = EntityId::parse(SPEAKER_TOKEN).expect("the token is an entity id"); let mut referents = input.referents().to_vec(); - for referent in &mut referents { - if referent.entity.as_ref() == Some(speaker) { - referent.entity = None; + if referents.iter().any(|r| r.entity.as_ref() == Some(&token)) { + return Err(LowerError::SpeakerTokenTaken); + } + let mut forms: BTreeMap<&str, Vec> = BTreeMap::new(); + if referents.iter().any(|r| r.entity.as_ref() == Some(speaker)) { + let spoken = match entity(entities, speaker) { + Ok(found) => found.forms_in(language).filter(|f| !f.is_empty()).map(<[String]>::to_vec), + Err(LowerError::UnknownEntity { .. }) => None, + Err(e) => return Err(e), + }; + if let Some(spoken) = &spoken { + forms.insert(token.as_str(), spoken.clone()); + } + for referent in &mut referents { + if referent.entity.as_ref() == Some(speaker) { + referent.entity = spoken.as_ref().map(|_| token.clone()); + } } } - for id in referents.iter().filter_map(|r| r.entity.as_ref()) { + for id in referents.iter().filter_map(|r| r.entity.as_ref()).filter(|id| *id != &token) { let found = entity(entities, id)?; let named = found .forms_in(language) diff --git a/crates/weksa/src/lower.rs b/crates/weksa/src/lower.rs index 28f4ca2..12fbb61 100644 --- a/crates/weksa/src/lower.rs +++ b/crates/weksa/src/lower.rs @@ -184,6 +184,9 @@ pub enum LowerError { EntityRead { entity: EntityId, why: String }, /// A lowering prompt was asked of a view whose scene was never read. SceneUnread, + /// A referent cites the entity whose id is the lowering prompt's speaker + /// token ([`crate::context::SPEAKER_TOKEN`]), which would read as the speaker. + SpeakerTokenTaken, } impl fmt::Display for LowerError { @@ -206,6 +209,7 @@ impl fmt::Display for LowerError { LowerError::AskerNotAddressee { .. } => "request agent is not the addressee", LowerError::EntityRead { .. } => "entity port failed to read", LowerError::SceneUnread => "the view's scene was never read", + LowerError::SpeakerTokenTaken => "a referent cites the entity id the speaker token uses", }) } } diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs index 25cd4fe..e3d8c1b 100644 --- a/crates/weksa/tests/context.rs +++ b/crates/weksa/tests/context.rs @@ -808,11 +808,11 @@ mod lowering { } } - /// The item's speaker is never printed, though a referent cites it: its - /// referent loses the entity, the forms map loses its line, and the view - /// itself is not changed. + /// The item's speaker id is never printed, though a referent cites it: the + /// referent holds the token, the forms line holds the speaker's forms under + /// it, and the view itself is not changed. #[test] - fn a_cited_speaker_is_stripped_from_the_lowering_prompt() { + fn a_cited_speaker_is_replaced_by_the_token_in_the_lowering_prompt() { let cast = InventoryStore::open(&data("interlingua/inventory.cc")).unwrap(); let cast_entities = EntityStore::open(&data("entity/entity.cc")).unwrap(); let view = admitted(proof_packet(), &cast, "corpus/corpus.cc", &cast_entities); @@ -825,18 +825,49 @@ mod lowering { }; let prompt = lower(&held).unwrap(); let text = prompt.text(); - for banned in ["entity:nibu", "NIBUCANARY"] { - assert!(!text.contains(banned), "{banned} in {text}"); - } - assert!(text.contains("\n\nentities: {}\n\n"), "{text}"); - assert!(text.contains(r#""id":"n3","kind":"referent_kind:1","concept":"concept:5","entity":null"#), "{text}"); + assert!(!text.contains("entity:nibu"), "{text}"); + assert!(text.contains("\n\nentities: {\"entity:speaker\":[\"NIBUCANARY\"]}\n\n"), "{text}"); + assert!(text.contains(r#""id":"n3","kind":"referent_kind:1","concept":"concept:5","entity":"entity:speaker""#), "{text}"); assert!(view.referents().iter().any(|r| r.entity.as_ref() == Some(&speaker))); - // A speaker the language has no form for, or the port lacks, does not refuse. - assert!(lower(&Held(vec![])).is_ok()); - assert!(lower(&Held(vec![nibu(&[("sa-Latn", &["Nibuwa"])])])).is_ok()); + // A speaker the language has no form for, or the port lacks, does not + // refuse: its referent names no entity and the line has no entry. + for held in [Held(vec![]), Held(vec![nibu(&[("sa-Latn", &["Nibuwa"])])])] { + let prompt = lower(&held).unwrap(); + let text = prompt.text(); + assert!(text.contains("\n\nentities: {}\n\n") && !text.contains("entity:speaker"), "{text}"); + assert!(text.contains(r#""concept":"concept:5","entity":null"#), "{text}"); + } } - /// Only the speaker is stripped: when the item's speaker is someone else, + /// A real entity whose id is the token would read as the speaker: refused. + #[test] + fn a_cited_entity_with_the_tokens_id_is_refused() { + let cast = InventoryStore::open(&data("interlingua/inventory.cc")).unwrap(); + let mut packet = proof_packet(); + packet.referents[2].entity = Some(EntityId::parse(weksa::context::SPEAKER_TOKEN).unwrap()); + let dir = tempfile::tempdir().unwrap(); + let mut entities = EntityStore::open(&dir.path().join("e.cc")).unwrap(); + entities.put(super::common::ent("speaker", "en-US", &["Spk"])).unwrap(); + entities.put(super::common::ent("nibu", "en-US", &["Nibu"])).unwrap(); + let mut corpus = CorpusStore::open(&dir.path().join("c.cc")).unwrap(); + let mut item = super::common::item_in("item:nibu:1", "A line.", &super::common::group_in_bucket(10), super::common::cast()); + item.prior_turns = vec![weksa::Turn { speaker: super::common::eid("speaker"), text: "Well?".into() }]; + corpus.add(item, &entities).unwrap(); + let view = PacketStore::open(&dir.path().join("p.cc")).unwrap().admit(packet, &cast, &corpus, &entities).unwrap(); + let state = weksa::SpeakerStateStore::open(&dir.path().join("s.cc")).unwrap(); + let view = view.with_scene(&cast.inventory().unwrap(), &state, &corpus).unwrap(); + let item = CorpusItemId::parse("item:nibu:1").unwrap(); + let held = Held(vec![super::common::ent("speaker", "en-US", &["Spk"]), nibu(&[("en-US", &["Nibu"])])]); + for speaker in ["entity:nibu", "entity:speaker"] { + let speaker = EntityId::parse(speaker).unwrap(); + assert_eq!( + Prompt::lower(&view, &item, &speaker, "en-US", &cast.inventory().unwrap(), &held, &weksa::Ontology::empty()).err(), + Some(LowerError::SpeakerTokenTaken) + ); + } + } + + /// Only the speaker is replaced: when the item's speaker is someone else, /// the cited entity keeps its id in the packet and its forms line. #[test] fn a_cited_entity_that_is_not_the_speaker_is_kept() { @@ -851,9 +882,9 @@ mod lowering { assert!(text.contains("\n\nentities: {\"entity:nibu\":[\"NIBUCANARY\"]}\n\n"), "{text}"); } - /// Every referent citing the speaker is stripped, not the first. + /// Every referent citing the speaker holds the token, not the first. #[test] - fn a_speaker_cited_on_two_referents_prints_on_neither() { + fn a_speaker_cited_on_two_referents_prints_the_token_on_both() { let cast = InventoryStore::open(&data("interlingua/inventory.cc")).unwrap(); let cast_entities = EntityStore::open(&data("entity/entity.cc")).unwrap(); let mut packet = proof_packet(); @@ -865,8 +896,9 @@ mod lowering { let held = Held(vec![nibu(&[("en-US", &["NIBUCANARY"])])]); let prompt = Prompt::lower(&view, &item, &speaker, "en-US", &cast.inventory().unwrap(), &held, &weksa::Ontology::empty()).unwrap(); let text = prompt.text(); - assert!(!text.contains("entity:nibu") && !text.contains("NIBUCANARY"), "{text}"); - assert!(text.contains("\n\nentities: {}\n\n"), "{text}"); + assert!(!text.contains("entity:nibu"), "{text}"); + assert_eq!(text.matches(r#""entity":"entity:speaker""#).count(), 2, "{text}"); + assert!(text.contains("\n\nentities: {\"entity:speaker\":[\"NIBUCANARY\"]}\n\n"), "{text}"); } /// The speaker is matched by whole id: a cited entity whose id the diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index a49df9e..0d75b3b 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -2889,8 +2889,9 @@ fn the_lowerer_sees_no_item_text() { } /// The promise `lower-prompt-no-item-text`: no lowering prompt holds the -/// item's speaker id. A referent citing the speaker must not put that entity -/// id (nor its forms) into the prompt; another cited entity still prints. +/// item's speaker id. A referent citing the speaker holds the fixed token +/// instead and its forms print under the token (forms are shared by every +/// item); another cited entity still prints under its own id. #[test] fn the_lowerer_never_sees_the_items_speaker() { use weksa::entity::{Entity, EntityOrigin, LanguageForms}; @@ -2947,11 +2948,10 @@ fn the_lowerer_never_sees_the_items_speaker() { assert_eq!(with_other.len(), 1, "one lowering prompt cites the other entity"); let text = with_other[0]; assert!(text.contains("entity:othercite"), "{text}"); - for banned in ["entity:spkcanary", "ZEBRAQUILL"] { - assert!(!text.contains(banned), "{banned} in {text}"); - } + assert!(!text.contains("entity:spkcanary"), "{text}"); + assert!(text.contains(r#""entity:speaker":["ZEBRAQUILL"]"#), "{text}"); for (_, t) in model.seen.iter().filter(|(_, t)| t.starts_with(LOWER_INSTRUCTIONS)) { - assert!(!t.contains("entity:spkcanary") && !t.contains("ZEBRAQUILL")); + assert!(!t.contains("entity:spkcanary")); } } @@ -4752,9 +4752,10 @@ fn each_item_lowers_in_its_own_language() { } else { en += 1; assert!(text.contains("\n\nlanguage: en-US\n\n"), "{text}"); - // Laozi speaks the en-US items: the lowerer is not told his name. - assert!(text.contains("\n\nentities: {}\n\n"), "{text}"); - assert!(!text.contains("entity:laozi") && !text.contains("Laozi") && !text.contains("老子"), "{text}"); + // Laozi speaks the en-US items: the lowerer gets his forms under the + // speaker token, never his id, and no ja-JP form. + assert!(text.contains("\n\nentities: {\"entity:speaker\":[\"Laozi\"]}\n\n"), "{text}"); + assert!(!text.contains("entity:laozi") && !text.contains("老子"), "{text}"); } } assert_eq!((ja, en), (3, 3)); diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 1e3fcd8..f9cf105 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -676,7 +676,7 @@ One line per cut; specs exist for the cuts about to run (2d and 4, siblings on ` 7a. `loop-context-builder`: `context.rs` (`TrainItems`, `EncoderOffer`, `Prompt`: corpus text reaches a model call only through `Prompt::mint` over live Train items or `Prompt::encode` over one offered item; instructions are the consts `MINT_INSTRUCTIONS` and `ENCODE_INSTRUCTIONS` in `context.rs`; `Checkpoint` has no public constructor, offers each item once, and its offers borrow it (the gate is typed `TestReadings` and Test offers on the round path; operator tools such as `weksa corpus dump` and `weksa score` still read corpus text of any split); prior turns print the held Train text), `SplitScheme::live`, `CorpusItem::cited_entities` (one owner of the id set `PacketStore::admit` and the offer share), and checkpoint-only Test readings (`Checkpoint`, `TestReadings`; `Aggregate` and `leak_diagnostic` carry no Test). Library only; since 7b `TrainItems::from_corpus` and `EncoderOffer::for_round` are crate-private and `RunStores` is their one public producer. Merges follow-ups `loop-context-builder-2` and `scorer-test-readings-type`. (spec r1) 7b. `golf-loop-round` (spec r1; branch `weksa-lowering/golf-loop-round`): the smallest loop that scores one round. `run.rs`: `RunStores` pins the corpus and entity stores of a run by canonical path and FNV-1a 64 digest at `create`, reopens only from `/run.cc` and refuses drift (corpus, entities, `PROMPT_SOURCE_DIGEST` of `context.rs`, a second pin on `lower.rs` retired in 7d, model, embedder); `round` encodes every live Train and Validation item through a `ModelPort` (`model.rs`; the one adapter `AnswerFiles` writes `.prompt` and reads `.answer`, so the model is the Claude session), reads each answer into a `PacketDraft` (`encode.rs`, the one place a model's JSON is read), admits through `PacketStore::admit` into `/round-/packets.cc`, lowers, scores once, and keeps a `RoundRecord` (`weksa.loop_round`, no text, no Test; Train rows, Validation only as means and refusal counts). `weksa loop open|round`. Not here: the spec-edit step, accept rule and composite, noise floor, Score and lowering documents, a subprocess adapter, the checkpoint door, a model lowerer (follow-ups `golf-loop-spec-edit`, `golf-loop-accept-rule`, `golf-loop-score-documents`, `golf-loop-subprocess-adapter`, `golf-loop-checkpoint-door-bound`, `golf-loop-model-lowerer`). `packet-store-inventory-binding` is met for loop packets only (`RunStores` admits with the run's inventory); `validate`, `packet proof|dao` and `lower` still take caller stores. Waits on 7a, `checkpoint-gate`, 4b. 7c. `golf-loop-edit-accept` (spec r1; branch `weksa-lowering/golf-loop-edit-accept`): one edit-accept cycle. The run works on its own copy of the inventory (`create` copies the caller's file to `/inventory.cc`, refusing a root that already holds a file of that name, and pins its digest as `RunRecord` key 11; `RunRecord.inventory` is provenance; a `run.cc` from before has no key 11 and does not open); the head is derived from the latest Accepted `Decision`. `weksa loop cycle` asks the editor for one inventory edit from `Prompt::mint(train, inventory, rev, feedback)` (Train blocks, the inventory listing and an `EditFeedback` line: whether the last edit was kept and the head's Validation composite rounded to `LADDER_STEP`, the Ladder release of Blum and Hardt 2015), reads it as an `EditDraft` (`encode.rs`; ids and revisions are the store's, `into_revision`), admits it through `InventoryStore::admit_revision` into `/cycle-/inventory.cc`, runs a candidate round under it, and `decide`s on Validation against the head's establishing round past a floor (never under one `LADDER_STEP`; from cut `cycle-floor-paired` the paired floor over rounds 1 to `FLOOR_ARMS_MIN` + 1, so a cycle needs four recorded rounds, where 7c measured the spread between rounds 1 and 2). An edit that fails is a rejected `Decision`, not an error. `RoundRecord` gains keys 16 (inventory digest) and 17 (cycle); an encode answer that is not UTF-8 is that item's parse refusal. Not here: Score and lowering documents (`golf-loop-score-docs`), Train rows in the edit prompt (`golf-loop-edit-train-feedback`), a paired floor (`golf-loop-paired-floor`), the checkpoint door and merging to `data/` (`golf-loop-checkpoint-door-bound`, `golf-loop-freeze-merge`), a model lowerer (`golf-loop-model-lowerer-next`; until it lands an entry an edit adds has no lowering row, so a cycle gains only by retiring entries or adding ones the encoder does not cite). Waits on 7b. -7d. `golf-loop-model-lowerer` (spec r1; branch `weksa-lowering/golf-loop-model-lowerer`): the round lowers through the `ModelPort`. `Prompt::lower` (`context.rs`) takes a `LoweringInput` (no text, key, group or speaker getter), the item's speaker id, the inventory at the view's revision and the item language's entity forms, and prints `LOWER_INSTRUCTIONS`, the meaning of the packet, the entities line and the inventory listing; the item key goes only to `item_keys`, and the speaker id is stripped: a referent citing the item's speaker is printed with no entity, and the speaker has no line in the entities map and is never looked up (a speaker the language has no form for does not refuse; a speaker named in the text cannot be named by the lowerer). The answer is `{"text"}`, read as `LoweringDraft` in `encode.rs`, held in `score_round`'s locals and scored once; it is never put in `run.cc`, a `RoundRecord`, a `Decision` or `EditFeedback`. A round has two answer phases (every encode prompt, then every lowering prompt), all or nothing: while any prompt is unanswered `packets.cc` is discarded and nothing reaches `run.cc`. `lower::lower` and the template table leave the loop (they remain `weksa lower`'s; follow-up `table-retirement`), so an entry an edit adds can be said and a cycle can buy a gain. One source pin remains: `PROMPT_SOURCE_DIGEST` covers `context.rs`, `split.rs` and `lower.rs` (`RunRecord` key 7 and `RoundRecord` key 3 are retired). The run's record pins `LowererReach` (`weksa loop open --lowerer-reach briefed-subagent|tool-less-relay`, `RunRecord` key 12) and every printed round and cycle line says it; a briefed-subagent run says its numbers are smoke-round only (rulings `lowerer-call-reach-toolless-relay`, `dao-round-seed-inventory`). A lowering is refused at `Lower` when blank, over `MAX_LOWERING_BYTES` (4096) or holding a control character other than a line break or tab; `PROMPT_SOURCE_DIGEST` also covers `packet.rs`. Not here: per-item lowering text documents (`golf-loop-score-docs`), the subprocess adapter. Waits on 7c. +7d. `golf-loop-model-lowerer` (spec r1; branch `weksa-lowering/golf-loop-model-lowerer`): the round lowers through the `ModelPort`. `Prompt::lower` (`context.rs`) takes a `LoweringInput` (no text, key, group or speaker getter), the item's speaker id, the inventory at the view's revision and the item language's entity forms, and prints `LOWER_INSTRUCTIONS`, the meaning of the packet, the entities line and the inventory listing; the item key goes only to `item_keys`, and the speaker id never prints: a referent citing the item's speaker holds the fixed token `entity:speaker` (`SPEAKER_TOKEN`, a constant the same for every speaker and item, never derived from an id) and the entities map holds the speaker's forms under that token. The promise guards the id that names an entity of this item; the speaker's forms and the role token do print, because forms are shared by every item of the language (follow-up `speaker-alias-token-in-lowering-prompt`, which restored the lowerer's ability to name its speaker after #84 removed it). A speaker the language has no form for, or the port lacks, does not refuse: its referents print no entity and the map has no entry. A referent citing the real id `entity:speaker` is refused (`LowerError::SpeakerTokenTaken`); the entity store does not reserve it. The answer is `{"text"}`, read as `LoweringDraft` in `encode.rs`, held in `score_round`'s locals and scored once; it is never put in `run.cc`, a `RoundRecord`, a `Decision` or `EditFeedback`. A round has two answer phases (every encode prompt, then every lowering prompt), all or nothing: while any prompt is unanswered `packets.cc` is discarded and nothing reaches `run.cc`. `lower::lower` and the template table leave the loop (they remain `weksa lower`'s; follow-up `table-retirement`), so an entry an edit adds can be said and a cycle can buy a gain. One source pin remains: `PROMPT_SOURCE_DIGEST` covers `context.rs`, `split.rs` and `lower.rs` (`RunRecord` key 7 and `RoundRecord` key 3 are retired). The run's record pins `LowererReach` (`weksa loop open --lowerer-reach briefed-subagent|tool-less-relay`, `RunRecord` key 12) and every printed round and cycle line says it; a briefed-subagent run says its numbers are smoke-round only (rulings `lowerer-call-reach-toolless-relay`, `dao-round-seed-inventory`). A lowering is refused at `Lower` when blank, over `MAX_LOWERING_BYTES` (4096) or holding a control character other than a line break or tab; `PROMPT_SOURCE_DIGEST` also covers `packet.rs`. Not here: per-item lowering text documents (`golf-loop-score-docs`), the subprocess adapter. Waits on 7c. 7. `golf-loop-en` (7b and 7c landed the round, `ModelPort`, encoder prompt, template digest, round records and the edit-accept cycle; the rest remains): the packet store bound to one inventory store everywhere (`packet-store-inventory-binding`), a subprocess adapter, Score documents, `weksa score` in the loop, the detached unit on Yggdrasil. Waits on 3, 5, 6 and follow-up `claude-on-yggdrasil`. 8. `style-axis`: a style-embedding score behind the scorer port. Waits on 3. 9. `ja-panel`: ja-JP into the loop's panel, in two cuts on `e9cbd9b`. 9a `ja-panel-loop` (spec r1): language is the item's and the panel the run's: `Prompt::lower` takes the item's language and prints it on a language line (no prompt names English), entity forms are read in that language, `RunRecord` pins the panel (`weksa loop open --panel`), offers, mint and recall read only panel items, round means, floors and readings are per panel language, the accept rule runs `decide` per language, the recall split works on unspaced scripts, and `weksa corpus add` has no default language. 9b `ja-panel-corpus` (spec r1, after 9a): the ja-JP slice of `data/original`, model-written Japanese Train and Validation passages and eight Aozora Bunko public-domain Test sentences with full provenance, and their entities. Cut 5 is not a prerequisite: neither language lowers with ontology entries until cut 5 gives both theirs. From 8794debdf7777349e27cf11127c959d940fc5451 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 17:23:58 +0000 Subject: [PATCH 03/14] Pin the speaker's forms by language and empty list; give ELIDED its doc back Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/context.rs | 2 +- crates/weksa/tests/context.rs | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index cad1ff9..0ecd067 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -538,7 +538,6 @@ struct Meaning<'a> { stance: &'a Stance, } -/// Printed in place of a prior turn whose text no live Train item holds. /// What the lowering prompt prints in place of the item's speaker's entity id: /// one constant, the same for every speaker and every item, so no id and /// nothing derived from one reaches the lowerer. It parses as an entity id so @@ -546,6 +545,7 @@ struct Meaning<'a> { /// ([`LowerError::SpeakerTokenTaken`]); the entity store does not reserve it. pub const SPEAKER_TOKEN: &str = "entity:speaker"; +/// Printed in place of a prior turn whose text no live Train item holds. const ELIDED: &str = "[elided]"; impl Prompt { diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs index e3d8c1b..d2550de 100644 --- a/crates/weksa/tests/context.rs +++ b/crates/weksa/tests/context.rs @@ -839,6 +839,33 @@ mod lowering { } } + /// The speaker's forms are those of the item's language, and a language whose form list is + /// an empty list is no form: the line has no token entry. + #[test] + fn the_speakers_forms_are_those_of_the_items_language() { + let cast = InventoryStore::open(&data("interlingua/inventory.cc")).unwrap(); + let cast_entities = EntityStore::open(&data("entity/entity.cc")).unwrap(); + let view = admitted(proof_packet(), &cast, "corpus/corpus.cc", &cast_entities); + let inv = cast.inventory().unwrap(); + let item = CorpusItemId::parse("item:nibu:1").unwrap(); + let speaker = EntityId::parse("entity:nibu").unwrap(); + let lower = |tag: &str, held: &Held| { + Prompt::lower(&view, &item, &speaker, tag, &inv, held, &weksa::Ontology::empty()) + .unwrap() + .text() + .to_string() + }; + let held = Held(vec![nibu(&[("en-US", &["ENGLISHFORM"]), ("ja-JP", &["NIHONGOFORM"])])]); + let ja = lower("ja-JP", &held); + assert!(ja.contains("\n\nentities: {\"entity:speaker\":[\"NIHONGOFORM\"]}\n\n"), "{ja}"); + assert!(!ja.contains("ENGLISHFORM"), "{ja}"); + let en = lower("en-US", &held); + assert!(en.contains("{\"entity:speaker\":[\"ENGLISHFORM\"]}") && !en.contains("NIHONGOFORM"), "{en}"); + let empty = lower("en-US", &Held(vec![nibu(&[("en-US", &[])])])); + assert!(empty.contains("\n\nentities: {}\n\n") && !empty.contains("entity:speaker"), "{empty}"); + assert!(empty.contains(r#""concept":"concept:5","entity":null"#), "{empty}"); + } + /// A real entity whose id is the token would read as the speaker: refused. #[test] fn a_cited_entity_with_the_tokens_id_is_refused() { From 2bb9870338f625eec9c47354c40ad6f0d7e064e4 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 17:28:13 +0000 Subject: [PATCH 04/14] Prompt::lower: the item's speaker becomes a fixed token with its forms (#85) --- crates/weksa/examples/show_working.rs | 1 + crates/weksa/src/context.rs | 49 +++++-- crates/weksa/src/lower.rs | 4 + crates/weksa/tests/context.rs | 159 ++++++++++++++++++++--- crates/weksa/tests/golf_loop.rs | 19 +-- docs/targets/interlingua-lowering-map.md | 2 +- 6 files changed, 196 insertions(+), 38 deletions(-) diff --git a/crates/weksa/examples/show_working.rs b/crates/weksa/examples/show_working.rs index 6c39bdb..6a31b26 100644 --- a/crates/weksa/examples/show_working.rs +++ b/crates/weksa/examples/show_working.rs @@ -260,6 +260,7 @@ fn lower_error(e: &LowerError) -> Value { LowerError::AskerNotAddressee { predication } => ("AskerNotAddressee", format!("the agent of {predication} is not the addressee")), LowerError::EntityRead { entity, .. } => ("EntityRead", format!("the entity store failed reading {entity}")), LowerError::SceneUnread => ("SceneUnread", "the view's scene was never read".to_string()), + LowerError::SpeakerTokenTaken => ("SpeakerTokenTaken", "a referent cites the entity id the speaker token uses".to_string()), }; json!({ "variant": variant, "reason": reason }) } diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index cac786f..0ecd067 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -538,6 +538,13 @@ struct Meaning<'a> { stance: &'a Stance, } +/// What the lowering prompt prints in place of the item's speaker's entity id: +/// one constant, the same for every speaker and every item, so no id and +/// nothing derived from one reaches the lowerer. It parses as an entity id so +/// the packet keeps its shape. A real entity of this id is refused at lowering +/// ([`LowerError::SpeakerTokenTaken`]); the entity store does not reserve it. +pub const SPEAKER_TOKEN: &str = "entity:speaker"; + /// Printed in place of a prior turn whose text no live Train item holds. const ELIDED: &str = "[elided]"; @@ -637,12 +644,18 @@ impl Prompt { /// discourse and the scene relation `with_scene` read, and `language`, /// never the caller; the scene's values are never printed, only the entries /// they select. `item` goes only to [`Prompt::item_keys`], never into the - /// text. `speaker` is the item's speaker: it is never printed. A referent - /// citing it is printed with no entity, and it has no line in `entities:` - /// and no form to find, so a speaker the language has no form for does not - /// refuse. Any other cited entity is printed with its forms. A cited - /// entity the port lacks, or one with no form in `language`, is an error - /// and no prompt. The caller chooses the language; the view has none. + /// text. `speaker` is the item's speaker: its id is never printed. The + /// promise guards the id that names an entity of this item, not the + /// speaker's forms, which every item of the language shares. A referent + /// citing the speaker is printed with [`SPEAKER_TOKEN`] as its entity, and + /// the entities line holds the speaker's forms under that token. A speaker + /// the language has no form for, or the port lacks, does not refuse: its + /// referents print no entity and the line has no entry. Any other cited + /// entity is printed under its own id with its forms; one the port lacks, + /// or one with no form in `language`, is an error and no prompt. A + /// referent citing the token's own id as an entity is + /// [`LowerError::SpeakerTokenTaken`]. The caller chooses the language; the + /// view has none. pub fn lower( input: &LoweringInput, item: &CorpusItemId, @@ -655,14 +668,28 @@ impl Prompt { if input.scene_read() == &SceneRead::Unread { return Err(LowerError::SceneUnread); } - let mut forms: BTreeMap<&str, Vec> = BTreeMap::new(); + let token = EntityId::parse(SPEAKER_TOKEN).expect("the token is an entity id"); let mut referents = input.referents().to_vec(); - for referent in &mut referents { - if referent.entity.as_ref() == Some(speaker) { - referent.entity = None; + if referents.iter().any(|r| r.entity.as_ref() == Some(&token)) { + return Err(LowerError::SpeakerTokenTaken); + } + let mut forms: BTreeMap<&str, Vec> = BTreeMap::new(); + if referents.iter().any(|r| r.entity.as_ref() == Some(speaker)) { + let spoken = match entity(entities, speaker) { + Ok(found) => found.forms_in(language).filter(|f| !f.is_empty()).map(<[String]>::to_vec), + Err(LowerError::UnknownEntity { .. }) => None, + Err(e) => return Err(e), + }; + if let Some(spoken) = &spoken { + forms.insert(token.as_str(), spoken.clone()); + } + for referent in &mut referents { + if referent.entity.as_ref() == Some(speaker) { + referent.entity = spoken.as_ref().map(|_| token.clone()); + } } } - for id in referents.iter().filter_map(|r| r.entity.as_ref()) { + for id in referents.iter().filter_map(|r| r.entity.as_ref()).filter(|id| *id != &token) { let found = entity(entities, id)?; let named = found .forms_in(language) diff --git a/crates/weksa/src/lower.rs b/crates/weksa/src/lower.rs index 28f4ca2..12fbb61 100644 --- a/crates/weksa/src/lower.rs +++ b/crates/weksa/src/lower.rs @@ -184,6 +184,9 @@ pub enum LowerError { EntityRead { entity: EntityId, why: String }, /// A lowering prompt was asked of a view whose scene was never read. SceneUnread, + /// A referent cites the entity whose id is the lowering prompt's speaker + /// token ([`crate::context::SPEAKER_TOKEN`]), which would read as the speaker. + SpeakerTokenTaken, } impl fmt::Display for LowerError { @@ -206,6 +209,7 @@ impl fmt::Display for LowerError { LowerError::AskerNotAddressee { .. } => "request agent is not the addressee", LowerError::EntityRead { .. } => "entity port failed to read", LowerError::SceneUnread => "the view's scene was never read", + LowerError::SpeakerTokenTaken => "a referent cites the entity id the speaker token uses", }) } } diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs index 9d68517..d2550de 100644 --- a/crates/weksa/tests/context.rs +++ b/crates/weksa/tests/context.rs @@ -808,11 +808,11 @@ mod lowering { } } - /// The item's speaker is never printed, though a referent cites it: its - /// referent loses the entity, the forms map loses its line, and the view - /// itself is not changed. + /// The item's speaker id is never printed, though a referent cites it: the + /// referent holds the token, the forms line holds the speaker's forms under + /// it, and the view itself is not changed. #[test] - fn a_cited_speaker_is_stripped_from_the_lowering_prompt() { + fn a_cited_speaker_is_replaced_by_the_token_in_the_lowering_prompt() { let cast = InventoryStore::open(&data("interlingua/inventory.cc")).unwrap(); let cast_entities = EntityStore::open(&data("entity/entity.cc")).unwrap(); let view = admitted(proof_packet(), &cast, "corpus/corpus.cc", &cast_entities); @@ -825,18 +825,76 @@ mod lowering { }; let prompt = lower(&held).unwrap(); let text = prompt.text(); - for banned in ["entity:nibu", "NIBUCANARY"] { - assert!(!text.contains(banned), "{banned} in {text}"); - } - assert!(text.contains("\n\nentities: {}\n\n"), "{text}"); - assert!(text.contains(r#""id":"n3","kind":"referent_kind:1","concept":"concept:5","entity":null"#), "{text}"); + assert!(!text.contains("entity:nibu"), "{text}"); + assert!(text.contains("\n\nentities: {\"entity:speaker\":[\"NIBUCANARY\"]}\n\n"), "{text}"); + assert!(text.contains(r#""id":"n3","kind":"referent_kind:1","concept":"concept:5","entity":"entity:speaker""#), "{text}"); assert!(view.referents().iter().any(|r| r.entity.as_ref() == Some(&speaker))); - // A speaker the language has no form for, or the port lacks, does not refuse. - assert!(lower(&Held(vec![])).is_ok()); - assert!(lower(&Held(vec![nibu(&[("sa-Latn", &["Nibuwa"])])])).is_ok()); + // A speaker the language has no form for, or the port lacks, does not + // refuse: its referent names no entity and the line has no entry. + for held in [Held(vec![]), Held(vec![nibu(&[("sa-Latn", &["Nibuwa"])])])] { + let prompt = lower(&held).unwrap(); + let text = prompt.text(); + assert!(text.contains("\n\nentities: {}\n\n") && !text.contains("entity:speaker"), "{text}"); + assert!(text.contains(r#""concept":"concept:5","entity":null"#), "{text}"); + } } - /// Only the speaker is stripped: when the item's speaker is someone else, + /// The speaker's forms are those of the item's language, and a language whose form list is + /// an empty list is no form: the line has no token entry. + #[test] + fn the_speakers_forms_are_those_of_the_items_language() { + let cast = InventoryStore::open(&data("interlingua/inventory.cc")).unwrap(); + let cast_entities = EntityStore::open(&data("entity/entity.cc")).unwrap(); + let view = admitted(proof_packet(), &cast, "corpus/corpus.cc", &cast_entities); + let inv = cast.inventory().unwrap(); + let item = CorpusItemId::parse("item:nibu:1").unwrap(); + let speaker = EntityId::parse("entity:nibu").unwrap(); + let lower = |tag: &str, held: &Held| { + Prompt::lower(&view, &item, &speaker, tag, &inv, held, &weksa::Ontology::empty()) + .unwrap() + .text() + .to_string() + }; + let held = Held(vec![nibu(&[("en-US", &["ENGLISHFORM"]), ("ja-JP", &["NIHONGOFORM"])])]); + let ja = lower("ja-JP", &held); + assert!(ja.contains("\n\nentities: {\"entity:speaker\":[\"NIHONGOFORM\"]}\n\n"), "{ja}"); + assert!(!ja.contains("ENGLISHFORM"), "{ja}"); + let en = lower("en-US", &held); + assert!(en.contains("{\"entity:speaker\":[\"ENGLISHFORM\"]}") && !en.contains("NIHONGOFORM"), "{en}"); + let empty = lower("en-US", &Held(vec![nibu(&[("en-US", &[])])])); + assert!(empty.contains("\n\nentities: {}\n\n") && !empty.contains("entity:speaker"), "{empty}"); + assert!(empty.contains(r#""concept":"concept:5","entity":null"#), "{empty}"); + } + + /// A real entity whose id is the token would read as the speaker: refused. + #[test] + fn a_cited_entity_with_the_tokens_id_is_refused() { + let cast = InventoryStore::open(&data("interlingua/inventory.cc")).unwrap(); + let mut packet = proof_packet(); + packet.referents[2].entity = Some(EntityId::parse(weksa::context::SPEAKER_TOKEN).unwrap()); + let dir = tempfile::tempdir().unwrap(); + let mut entities = EntityStore::open(&dir.path().join("e.cc")).unwrap(); + entities.put(super::common::ent("speaker", "en-US", &["Spk"])).unwrap(); + entities.put(super::common::ent("nibu", "en-US", &["Nibu"])).unwrap(); + let mut corpus = CorpusStore::open(&dir.path().join("c.cc")).unwrap(); + let mut item = super::common::item_in("item:nibu:1", "A line.", &super::common::group_in_bucket(10), super::common::cast()); + item.prior_turns = vec![weksa::Turn { speaker: super::common::eid("speaker"), text: "Well?".into() }]; + corpus.add(item, &entities).unwrap(); + let view = PacketStore::open(&dir.path().join("p.cc")).unwrap().admit(packet, &cast, &corpus, &entities).unwrap(); + let state = weksa::SpeakerStateStore::open(&dir.path().join("s.cc")).unwrap(); + let view = view.with_scene(&cast.inventory().unwrap(), &state, &corpus).unwrap(); + let item = CorpusItemId::parse("item:nibu:1").unwrap(); + let held = Held(vec![super::common::ent("speaker", "en-US", &["Spk"]), nibu(&[("en-US", &["Nibu"])])]); + for speaker in ["entity:nibu", "entity:speaker"] { + let speaker = EntityId::parse(speaker).unwrap(); + assert_eq!( + Prompt::lower(&view, &item, &speaker, "en-US", &cast.inventory().unwrap(), &held, &weksa::Ontology::empty()).err(), + Some(LowerError::SpeakerTokenTaken) + ); + } + } + + /// Only the speaker is replaced: when the item's speaker is someone else, /// the cited entity keeps its id in the packet and its forms line. #[test] fn a_cited_entity_that_is_not_the_speaker_is_kept() { @@ -851,9 +909,9 @@ mod lowering { assert!(text.contains("\n\nentities: {\"entity:nibu\":[\"NIBUCANARY\"]}\n\n"), "{text}"); } - /// Every referent citing the speaker is stripped, not the first. + /// Every referent citing the speaker holds the token, not the first. #[test] - fn a_speaker_cited_on_two_referents_prints_on_neither() { + fn a_speaker_cited_on_two_referents_prints_the_token_on_both() { let cast = InventoryStore::open(&data("interlingua/inventory.cc")).unwrap(); let cast_entities = EntityStore::open(&data("entity/entity.cc")).unwrap(); let mut packet = proof_packet(); @@ -865,8 +923,9 @@ mod lowering { let held = Held(vec![nibu(&[("en-US", &["NIBUCANARY"])])]); let prompt = Prompt::lower(&view, &item, &speaker, "en-US", &cast.inventory().unwrap(), &held, &weksa::Ontology::empty()).unwrap(); let text = prompt.text(); - assert!(!text.contains("entity:nibu") && !text.contains("NIBUCANARY"), "{text}"); - assert!(text.contains("\n\nentities: {}\n\n"), "{text}"); + assert!(!text.contains("entity:nibu"), "{text}"); + assert_eq!(text.matches(r#""entity":"entity:speaker""#).count(), 2, "{text}"); + assert!(text.contains("\n\nentities: {\"entity:speaker\":[\"NIBUCANARY\"]}\n\n"), "{text}"); } /// The speaker is matched by whole id: a cited entity whose id the @@ -885,6 +944,72 @@ mod lowering { assert!(text.contains("NIBUCANARY"), "{text}"); } + /// The proof packet admitted for an item whose speaker is `speaker` and + /// whose prior turn is `other`'s, with `speaker` on referent n3 and `other` + /// on referent n1, then lowered in en-US over an entity port holding `held`. + fn lowered_for(speaker: &str, other: &str, held: &[Entity]) -> String { + let cast = InventoryStore::open(&data("interlingua/inventory.cc")).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let mut entities = EntityStore::open(&dir.path().join("e.cc")).unwrap(); + for slug in [speaker, other] { + entities.put(super::common::ent(slug, "en-US", &["Stored"])).unwrap(); + } + let mut corpus = CorpusStore::open(&dir.path().join("c.cc")).unwrap(); + let mut item = super::common::item_in("item:nibu:1", "A line.", &super::common::group_in_bucket(10), super::common::cast()); + item.speaker = super::common::eid(speaker); + item.prior_turns = vec![weksa::Turn { speaker: super::common::eid(other), text: "Well?".into() }]; + corpus.add(item, &entities).unwrap(); + let mut packet = proof_packet(); + packet.referents[0].entity = Some(super::common::eid(other)); + packet.referents[2].entity = Some(super::common::eid(speaker)); + let view = PacketStore::open(&dir.path().join("p.cc")) + .unwrap() + .admit(packet, &cast, &corpus, &entities) + .unwrap(); + let state = weksa::SpeakerStateStore::open(&dir.path().join("s.cc")).unwrap(); + let view = view.with_scene(&cast.inventory().unwrap(), &state, &corpus).unwrap(); + let item = CorpusItemId::parse("item:nibu:1").unwrap(); + Prompt::lower( + &view, + &item, + &super::common::eid(speaker), + "en-US", + &cast.inventory().unwrap(), + &Held(held.to_vec()), + &weksa::Ontology::empty(), + ) + .unwrap() + .text() + .to_string() + } + + /// Persona's ruling `speaker-alias-token-in-lowering-prompt`: the item's + /// speaker prints as one fixed role token, the same for every speaker and + /// every item, with the speaker's forms under it; its entity id never + /// prints. Another cited entity still prints under its own id. + #[test] + fn the_item_speaker_is_a_fixed_token_with_its_forms() { + const TOKEN: &str = "entity:speaker"; + let forms = |slug: &str, form: &str| super::common::ent(slug, "en-US", &[form]); + let other = forms("othercite", "OTTERBLOOM"); + let a = lowered_for("keeperzed", "othercite", &[forms("keeperzed", "ZEBRAQUILL"), other.clone()]); + let b = lowered_for("secondvoice", "othercite", &[forms("secondvoice", "MARMOTLUTE"), other.clone()]); + for (text, id, form, other_form) in [(&a, "keeperzed", "ZEBRAQUILL", "MARMOTLUTE"), (&b, "secondvoice", "MARMOTLUTE", "ZEBRAQUILL")] { + assert!(!text.contains(id), "{id} in {text}"); + assert!(!text.contains(other_form), "{other_form} in {text}"); + assert!(text.contains(r#""concept":"concept:5","entity":"entity:speaker""#), "{text}"); + assert!(text.contains(&format!(r#""{TOKEN}":["{form}"]"#)), "{text}"); + assert!(text.contains(r#""entity":"entity:othercite""#), "{text}"); + assert!(text.contains(r#""entity:othercite":["OTTERBLOOM"]"#), "{text}"); + } + // The token is one constant: the entities lines differ only in the speaker's form. + assert_eq!(a.replace("ZEBRAQUILL", "F"), b.replace("MARMOTLUTE", "F")); + // A speaker with no form in the language does not refuse and gets no token entry. + let bare = lowered_for("keeperzed", "othercite", &[other]); + assert!(!bare.contains("keeperzed") && !bare.contains(TOKEN), "{bare}"); + assert!(bare.contains(r#"{"entity:othercite":["OTTERBLOOM"]}"#), "{bare}"); + } + fn feature(inv: &Inventory, gloss: &str) -> weksa::InvId { inv.entries().find(|e| e.gloss == gloss).unwrap().id } diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index a49df9e..0d75b3b 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -2889,8 +2889,9 @@ fn the_lowerer_sees_no_item_text() { } /// The promise `lower-prompt-no-item-text`: no lowering prompt holds the -/// item's speaker id. A referent citing the speaker must not put that entity -/// id (nor its forms) into the prompt; another cited entity still prints. +/// item's speaker id. A referent citing the speaker holds the fixed token +/// instead and its forms print under the token (forms are shared by every +/// item); another cited entity still prints under its own id. #[test] fn the_lowerer_never_sees_the_items_speaker() { use weksa::entity::{Entity, EntityOrigin, LanguageForms}; @@ -2947,11 +2948,10 @@ fn the_lowerer_never_sees_the_items_speaker() { assert_eq!(with_other.len(), 1, "one lowering prompt cites the other entity"); let text = with_other[0]; assert!(text.contains("entity:othercite"), "{text}"); - for banned in ["entity:spkcanary", "ZEBRAQUILL"] { - assert!(!text.contains(banned), "{banned} in {text}"); - } + assert!(!text.contains("entity:spkcanary"), "{text}"); + assert!(text.contains(r#""entity:speaker":["ZEBRAQUILL"]"#), "{text}"); for (_, t) in model.seen.iter().filter(|(_, t)| t.starts_with(LOWER_INSTRUCTIONS)) { - assert!(!t.contains("entity:spkcanary") && !t.contains("ZEBRAQUILL")); + assert!(!t.contains("entity:spkcanary")); } } @@ -4752,9 +4752,10 @@ fn each_item_lowers_in_its_own_language() { } else { en += 1; assert!(text.contains("\n\nlanguage: en-US\n\n"), "{text}"); - // Laozi speaks the en-US items: the lowerer is not told his name. - assert!(text.contains("\n\nentities: {}\n\n"), "{text}"); - assert!(!text.contains("entity:laozi") && !text.contains("Laozi") && !text.contains("老子"), "{text}"); + // Laozi speaks the en-US items: the lowerer gets his forms under the + // speaker token, never his id, and no ja-JP form. + assert!(text.contains("\n\nentities: {\"entity:speaker\":[\"Laozi\"]}\n\n"), "{text}"); + assert!(!text.contains("entity:laozi") && !text.contains("老子"), "{text}"); } } assert_eq!((ja, en), (3, 3)); diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 1e3fcd8..f9cf105 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -676,7 +676,7 @@ One line per cut; specs exist for the cuts about to run (2d and 4, siblings on ` 7a. `loop-context-builder`: `context.rs` (`TrainItems`, `EncoderOffer`, `Prompt`: corpus text reaches a model call only through `Prompt::mint` over live Train items or `Prompt::encode` over one offered item; instructions are the consts `MINT_INSTRUCTIONS` and `ENCODE_INSTRUCTIONS` in `context.rs`; `Checkpoint` has no public constructor, offers each item once, and its offers borrow it (the gate is typed `TestReadings` and Test offers on the round path; operator tools such as `weksa corpus dump` and `weksa score` still read corpus text of any split); prior turns print the held Train text), `SplitScheme::live`, `CorpusItem::cited_entities` (one owner of the id set `PacketStore::admit` and the offer share), and checkpoint-only Test readings (`Checkpoint`, `TestReadings`; `Aggregate` and `leak_diagnostic` carry no Test). Library only; since 7b `TrainItems::from_corpus` and `EncoderOffer::for_round` are crate-private and `RunStores` is their one public producer. Merges follow-ups `loop-context-builder-2` and `scorer-test-readings-type`. (spec r1) 7b. `golf-loop-round` (spec r1; branch `weksa-lowering/golf-loop-round`): the smallest loop that scores one round. `run.rs`: `RunStores` pins the corpus and entity stores of a run by canonical path and FNV-1a 64 digest at `create`, reopens only from `/run.cc` and refuses drift (corpus, entities, `PROMPT_SOURCE_DIGEST` of `context.rs`, a second pin on `lower.rs` retired in 7d, model, embedder); `round` encodes every live Train and Validation item through a `ModelPort` (`model.rs`; the one adapter `AnswerFiles` writes `.prompt` and reads `.answer`, so the model is the Claude session), reads each answer into a `PacketDraft` (`encode.rs`, the one place a model's JSON is read), admits through `PacketStore::admit` into `/round-/packets.cc`, lowers, scores once, and keeps a `RoundRecord` (`weksa.loop_round`, no text, no Test; Train rows, Validation only as means and refusal counts). `weksa loop open|round`. Not here: the spec-edit step, accept rule and composite, noise floor, Score and lowering documents, a subprocess adapter, the checkpoint door, a model lowerer (follow-ups `golf-loop-spec-edit`, `golf-loop-accept-rule`, `golf-loop-score-documents`, `golf-loop-subprocess-adapter`, `golf-loop-checkpoint-door-bound`, `golf-loop-model-lowerer`). `packet-store-inventory-binding` is met for loop packets only (`RunStores` admits with the run's inventory); `validate`, `packet proof|dao` and `lower` still take caller stores. Waits on 7a, `checkpoint-gate`, 4b. 7c. `golf-loop-edit-accept` (spec r1; branch `weksa-lowering/golf-loop-edit-accept`): one edit-accept cycle. The run works on its own copy of the inventory (`create` copies the caller's file to `/inventory.cc`, refusing a root that already holds a file of that name, and pins its digest as `RunRecord` key 11; `RunRecord.inventory` is provenance; a `run.cc` from before has no key 11 and does not open); the head is derived from the latest Accepted `Decision`. `weksa loop cycle` asks the editor for one inventory edit from `Prompt::mint(train, inventory, rev, feedback)` (Train blocks, the inventory listing and an `EditFeedback` line: whether the last edit was kept and the head's Validation composite rounded to `LADDER_STEP`, the Ladder release of Blum and Hardt 2015), reads it as an `EditDraft` (`encode.rs`; ids and revisions are the store's, `into_revision`), admits it through `InventoryStore::admit_revision` into `/cycle-/inventory.cc`, runs a candidate round under it, and `decide`s on Validation against the head's establishing round past a floor (never under one `LADDER_STEP`; from cut `cycle-floor-paired` the paired floor over rounds 1 to `FLOOR_ARMS_MIN` + 1, so a cycle needs four recorded rounds, where 7c measured the spread between rounds 1 and 2). An edit that fails is a rejected `Decision`, not an error. `RoundRecord` gains keys 16 (inventory digest) and 17 (cycle); an encode answer that is not UTF-8 is that item's parse refusal. Not here: Score and lowering documents (`golf-loop-score-docs`), Train rows in the edit prompt (`golf-loop-edit-train-feedback`), a paired floor (`golf-loop-paired-floor`), the checkpoint door and merging to `data/` (`golf-loop-checkpoint-door-bound`, `golf-loop-freeze-merge`), a model lowerer (`golf-loop-model-lowerer-next`; until it lands an entry an edit adds has no lowering row, so a cycle gains only by retiring entries or adding ones the encoder does not cite). Waits on 7b. -7d. `golf-loop-model-lowerer` (spec r1; branch `weksa-lowering/golf-loop-model-lowerer`): the round lowers through the `ModelPort`. `Prompt::lower` (`context.rs`) takes a `LoweringInput` (no text, key, group or speaker getter), the item's speaker id, the inventory at the view's revision and the item language's entity forms, and prints `LOWER_INSTRUCTIONS`, the meaning of the packet, the entities line and the inventory listing; the item key goes only to `item_keys`, and the speaker id is stripped: a referent citing the item's speaker is printed with no entity, and the speaker has no line in the entities map and is never looked up (a speaker the language has no form for does not refuse; a speaker named in the text cannot be named by the lowerer). The answer is `{"text"}`, read as `LoweringDraft` in `encode.rs`, held in `score_round`'s locals and scored once; it is never put in `run.cc`, a `RoundRecord`, a `Decision` or `EditFeedback`. A round has two answer phases (every encode prompt, then every lowering prompt), all or nothing: while any prompt is unanswered `packets.cc` is discarded and nothing reaches `run.cc`. `lower::lower` and the template table leave the loop (they remain `weksa lower`'s; follow-up `table-retirement`), so an entry an edit adds can be said and a cycle can buy a gain. One source pin remains: `PROMPT_SOURCE_DIGEST` covers `context.rs`, `split.rs` and `lower.rs` (`RunRecord` key 7 and `RoundRecord` key 3 are retired). The run's record pins `LowererReach` (`weksa loop open --lowerer-reach briefed-subagent|tool-less-relay`, `RunRecord` key 12) and every printed round and cycle line says it; a briefed-subagent run says its numbers are smoke-round only (rulings `lowerer-call-reach-toolless-relay`, `dao-round-seed-inventory`). A lowering is refused at `Lower` when blank, over `MAX_LOWERING_BYTES` (4096) or holding a control character other than a line break or tab; `PROMPT_SOURCE_DIGEST` also covers `packet.rs`. Not here: per-item lowering text documents (`golf-loop-score-docs`), the subprocess adapter. Waits on 7c. +7d. `golf-loop-model-lowerer` (spec r1; branch `weksa-lowering/golf-loop-model-lowerer`): the round lowers through the `ModelPort`. `Prompt::lower` (`context.rs`) takes a `LoweringInput` (no text, key, group or speaker getter), the item's speaker id, the inventory at the view's revision and the item language's entity forms, and prints `LOWER_INSTRUCTIONS`, the meaning of the packet, the entities line and the inventory listing; the item key goes only to `item_keys`, and the speaker id never prints: a referent citing the item's speaker holds the fixed token `entity:speaker` (`SPEAKER_TOKEN`, a constant the same for every speaker and item, never derived from an id) and the entities map holds the speaker's forms under that token. The promise guards the id that names an entity of this item; the speaker's forms and the role token do print, because forms are shared by every item of the language (follow-up `speaker-alias-token-in-lowering-prompt`, which restored the lowerer's ability to name its speaker after #84 removed it). A speaker the language has no form for, or the port lacks, does not refuse: its referents print no entity and the map has no entry. A referent citing the real id `entity:speaker` is refused (`LowerError::SpeakerTokenTaken`); the entity store does not reserve it. The answer is `{"text"}`, read as `LoweringDraft` in `encode.rs`, held in `score_round`'s locals and scored once; it is never put in `run.cc`, a `RoundRecord`, a `Decision` or `EditFeedback`. A round has two answer phases (every encode prompt, then every lowering prompt), all or nothing: while any prompt is unanswered `packets.cc` is discarded and nothing reaches `run.cc`. `lower::lower` and the template table leave the loop (they remain `weksa lower`'s; follow-up `table-retirement`), so an entry an edit adds can be said and a cycle can buy a gain. One source pin remains: `PROMPT_SOURCE_DIGEST` covers `context.rs`, `split.rs` and `lower.rs` (`RunRecord` key 7 and `RoundRecord` key 3 are retired). The run's record pins `LowererReach` (`weksa loop open --lowerer-reach briefed-subagent|tool-less-relay`, `RunRecord` key 12) and every printed round and cycle line says it; a briefed-subagent run says its numbers are smoke-round only (rulings `lowerer-call-reach-toolless-relay`, `dao-round-seed-inventory`). A lowering is refused at `Lower` when blank, over `MAX_LOWERING_BYTES` (4096) or holding a control character other than a line break or tab; `PROMPT_SOURCE_DIGEST` also covers `packet.rs`. Not here: per-item lowering text documents (`golf-loop-score-docs`), the subprocess adapter. Waits on 7c. 7. `golf-loop-en` (7b and 7c landed the round, `ModelPort`, encoder prompt, template digest, round records and the edit-accept cycle; the rest remains): the packet store bound to one inventory store everywhere (`packet-store-inventory-binding`), a subprocess adapter, Score documents, `weksa score` in the loop, the detached unit on Yggdrasil. Waits on 3, 5, 6 and follow-up `claude-on-yggdrasil`. 8. `style-axis`: a style-embedding score behind the scorer port. Waits on 3. 9. `ja-panel`: ja-JP into the loop's panel, in two cuts on `e9cbd9b`. 9a `ja-panel-loop` (spec r1): language is the item's and the panel the run's: `Prompt::lower` takes the item's language and prints it on a language line (no prompt names English), entity forms are read in that language, `RunRecord` pins the panel (`weksa loop open --panel`), offers, mint and recall read only panel items, round means, floors and readings are per panel language, the accept rule runs `decide` per language, the recall split works on unspaced scripts, and `weksa corpus add` has no default language. 9b `ja-panel-corpus` (spec r1, after 9a): the ja-JP slice of `data/original`, model-written Japanese Train and Validation passages and eight Aozora Bunko public-domain Test sentences with full provenance, and their entities. Cut 5 is not a prerequisite: neither language lowers with ontology entries until cut 5 gives both theirs. From b7501a758d760d214aa3c44e730fc05a121dae5e Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 19:32:04 +0000 Subject: [PATCH 05/14] Map: golf loop runs in a Claude session; Gutenberg terms note (#86) --- .../2026-10-29-golf-loop-prior-art.md | 4 +- docs/research/2026-11-13-gutenberg-terms.md | 59 +++++++++++++++++++ docs/targets/dao-de-jing-test-bed.md | 7 ++- docs/targets/interlingua-lowering-map.md | 15 +++-- 4 files changed, 75 insertions(+), 10 deletions(-) create mode 100644 docs/research/2026-11-13-gutenberg-terms.md diff --git a/docs/research/2026-10-29-golf-loop-prior-art.md b/docs/research/2026-10-29-golf-loop-prior-art.md index e2a24ae..7fb5510 100644 --- a/docs/research/2026-10-29-golf-loop-prior-art.md +++ b/docs/research/2026-10-29-golf-loop-prior-art.md @@ -65,7 +65,7 @@ Verified licence facts are few. Discord data excluded by policy; not read. - PersonaChat (Zhang et al. 2018): ParlAI project page, archived 2020-08-01, states no licence, only "You may continue to use the data without issue". [V] https://parl.ai/projects/personachat/ ; paper https://arxiv.org/abs/1801.07243 says data released open source via ParlAI; licence text [GAP]. Crowdworker-written, persona-statement-conditioned chit-chat [B]. - LIGHT: ParlAI page states no licence; contents: 663 locations, 3,462 objects, 1,755 character types, 11,000 episodes; WILD 41,131 training episodes from game deployment (human players, so a different provenance than crowdwork). [V] https://parl.ai/projects/light/ . Licence [GAP: check ParlAI repo]. - Cornell Movie-Dialogs Corpus: licence/terms page fetch failed (certificate rejected) https://www.cs.cornell.edu/~cristian/Cornell_Movie-Dialogs_Corpus.html . [GAP] Underlying scripts are copyrighted film text [B]; dataset licence unverified. -- Project Gutenberg: "vast majority" of eBooks public domain in the US, so no one can grant or withhold permission for uses; PG cannot sublicense copyrighted items; subject to its Terms of Use, which the permission page links and does not summarise; no statement on ML/bulk use on that page. [V] https://www.gutenberg.org/policy/permission.html . Terms-of-Use text on bulk/automated access and the trademark licence: [GAP, not read]. Items public domain in the US may not be elsewhere (same page). +- Project Gutenberg: "vast majority" of eBooks public domain in the US, so no one can grant or withhold permission for uses; PG cannot sublicense copyrighted items; subject to its Terms of Use, which the permission page links and does not summarise; no statement on ML/bulk use on that page. [V] https://www.gutenberg.org/policy/permission.html . Terms-of-Use text on bulk/automated access and the trademark licence: read 2026-11-13, see `2026-11-13-gutenberg-terms.md`. Items public domain in the US may not be elsewhere (same page). - Google Synthetic-Persona-Chat (model-generated, persona-chat-style): CC BY-NC? listed as cc-by-4.0 on HF. [V search-only] https://huggingface.co/datasets/google/Synthetic-Persona-Chat - Operator-authored or model-generated corpus: no licence question beyond provider terms [B, unread this pass]. Problems with evaluating on model-generated text: see 2 (self-bias, preference leakage, stylistic homogeneity: Xu et al. 2025; Li et al. 2025). The same Xu et al. finding says it extends to open-ended generation. - Expressive-register-annotated English corpora (hedging, humour, affect, politeness labelled), with permissive terms: [GAP] not searched beyond the list above (e.g. Stanford Politeness Corpus, GYAFC formality, Enron etc. are [B] candidates, unverified, GYAFC is known to carry Yahoo Answers terms [B]). @@ -83,6 +83,6 @@ Verified licence facts are few. Discord data excluded by policy; not read. - No per-metric human-correlation numbers for hedging, humour, affect, stance, persona voice; nearest is formality (Briakou 2021). COMET, BLEURT, MAUVE correlations not obtained. - No source on surface-form smuggling through an LLM bottleneck or its defences (vocab restriction, paraphrase invariance). - No source on schema/inventory growth from generation failures, or its overfitting controls. -- Cornell, LIGHT, PersonaChat licence texts not obtained; Gutenberg ToU on bulk use not read. +- Cornell, LIGHT, PersonaChat licence texts not obtained. - No realiser or evaluation protocol for constructed languages from a meaning representation. - Several arXiv PDFs (Briakou, Zheng, STEL, LUAR/StyleDistance) fetched as binary; numbers above come from the ar5iv/arXiv-HTML versions where marked, otherwise search summaries. diff --git a/docs/research/2026-11-13-gutenberg-terms.md b/docs/research/2026-11-13-gutenberg-terms.md new file mode 100644 index 0000000..1d2f596 --- /dev/null +++ b/docs/research/2026-11-13-gutenberg-terms.md @@ -0,0 +1,59 @@ +# Project Gutenberg terms: bulk access and header stripping + +Read 2026-11-13 for follow-up `gutenberg-terms` (campaign `weksa-lowering`), +ahead of cut `pd-importer`. Pages: gutenberg.org `policy/permission.html`, +`policy/terms_of_use.html`, `policy/robot_access.html`, `policy/license.html`. +The quotes were checked against the live HTML of each page by a second reader. +`pd-importer`'s Imagination re-reads the pages before it fixes a fetch path, +since they change. + +## Findings + +- **Automated access to the main site is refused.** Terms of Use: "This website + is intended for human users only"; "Any perceived use of automated tools to + access this website will result in a temporary or permanent block". OPDS + feeds: no more requests than a browser user would make; heavy requesters are + blocked by IP. +- **Bulk has two sanctioned paths.** Terms of Use: take bulk downloads "from + one of our mirrors, not from the main site". The robot-access page names the + `robot/harvest` wget endpoint (`-w 2`, filterable by file type or language) + and recommends a private mirror for a full local copy; it suggests checking + periodically for changes. Catalog data are offered as + machine-readable offline catalogs, "granted to the public domain". +- **Header and licence stripping.** Licence section 1.C: for a work not + protected by US copyright, with the user in the US, the Foundation does "not + claim a right to prevent you from copying, distributing, performing, + displaying or creating derivative works", "as long as all references to + Project Gutenberg are removed", the condition holding for a user in the US. + Section 1.E opens "Unless you have removed all references to Project + Gutenberg", and 1.E.1 to 1.E.7 (including 1.E.4, do not detach the licence) + attach to a work on which the phrase "Project Gutenberg" appears or with which + it is associated, so a text with every reference removed is outside 1.E. + 1.E.2 is not that grant: it says a PG eBook derived from texts not restricted + by US copyright may be copied and distributed in the US without fee. The + trademark is registered; non-commercial use of the name needs no permission, + commercial use does. +- **No statement on machine learning or AI use** on any of the four pages. +- **US public domain only.** The Foundation makes no claim about copyright + outside the US (1.D). A work free in the US may not be free elsewhere, and + Gutenberg's own catalog is not a per-work clearance. + +## What this decides for `pd-importer` + +- Fetch from a mirror or an offline dump. The Terms of Use footnote counts "many + books" as more than about 100 a day and sends those to mirrors, while + `robot/harvest` is a `www.gutenberg.org` URL: use it (with `-w 2`) only for a + handful of works. Never crawl `gutenberg.org` pages or OPDS from the loop host. +- Strip the whole header and footer, every Project Gutenberg reference and the + licence text from the stored text; keep provenance as the ebook number and + edition in the item's source field, which is Weksa's own record and not a use + of the mark. This is the path `docs/targets/dao-de-jing-test-bed.md` already + takes. The remaining open question there, what counts as a "reference", stays + open: the safe reading is that the ebook number in a provenance field is a + catalogue identifier, not a reference inside the work, but nothing read here + rules on it. +- Per-work check stays with the importer: the work's author and first + publication date against US public-domain rules, recorded on the item. The + Foundation's catalogue does not supply it. A work Gutenberg posts with the + copyright holder's permission (1.E.3) is out of scope. +- No importer code in this note. diff --git a/docs/targets/dao-de-jing-test-bed.md b/docs/targets/dao-de-jing-test-bed.md index bd2ebbf..2b71dab 100644 --- a/docs/targets/dao-de-jing-test-bed.md +++ b/docs/targets/dao-de-jing-test-bed.md @@ -23,9 +23,10 @@ plus a separately recorded chosen gloss per lowering). `corpus add ... --language en-US --source public-domain` per item, then `packet dao `. - Source: Project Gutenberg eBook 216, "The Tao Teh King, or the Tao and its Characteristics", Laozi, tr. James Legge (Sacred Books of the East 39, 1891). - Gutenberg's licence (1.E.2) lets a work derived from texts not protected by US - copyright be used with no fee provided every Project Gutenberg reference is - removed from the work. The items hold only Legge's words, one numbered section + Gutenberg's licence (1.C) does not claim a right to prevent derivative works of + a work not protected by US copyright, for a user in the US, provided every + Project Gutenberg reference is removed (see + `docs/research/2026-11-13-gutenberg-terms.md`). The items hold only Legge's words, one numbered section or sentence each; no Gutenberg header, licence text or trademark is stored, and the eBook file itself is not committed. `edition_url` names the page the text was retrieved from, as a citation of provenance and not as part of any diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index f9cf105..82d53f2 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -340,9 +340,14 @@ line two", "その　制御" against the - Stores: `data/interlingua/inventory.cc` and `data/corpus/corpus.cc` in the repo (small, reviewed with `weksa dump`); run stores (rounds, lowerings, scores) on Yggdrasil under the loop's state root, not in git. -- The loop runs on Yggdrasil as a detached unit with a log and a status record. - Starfire runs none of it. Model calls go through a `ModelPort`; its first - adapter drives `claude -p` on the operator's subscription. Never OpenAI +- The loop is a batch command driven from a Claude Code session on Yggdrasil + (`golf-loop-in-claude`), not a detached unit. Model calls go through a + `ModelPort` whose one adapter, `AnswerFiles`, writes a `.prompt` file and reads + a `.answer` file: Claude agents answer them, one fresh subagent per encode + prompt (`encoder-isolation-subagent-per-prompt`) and a tool-less relay for + lowering prompts (`lowerer-call-reach-toolless-relay`). Builds, stores, scoring + and embeddings run on Yggdrasil; Starfire runs none of it, and Weksa needs no + claude binary or token there. Never OpenAI (`eureka-body:ruling:no-return-to-openai-supplier`), never Discord data (`gamecult-chat:ruling:discord-taints-training`). @@ -675,9 +680,9 @@ One line per cut; specs exist for the cuts about to run (2d and 4, siblings on ` 6. `pd-importer`: public-domain play and fiction dialogue into the corpus with speaker, scene group, edition provenance, and Speaker and Scene annotations by the ruled annotator. Waits on 4, follow-up `gutenberg-terms` and question `pd-speaker-annotation`. 7a. `loop-context-builder`: `context.rs` (`TrainItems`, `EncoderOffer`, `Prompt`: corpus text reaches a model call only through `Prompt::mint` over live Train items or `Prompt::encode` over one offered item; instructions are the consts `MINT_INSTRUCTIONS` and `ENCODE_INSTRUCTIONS` in `context.rs`; `Checkpoint` has no public constructor, offers each item once, and its offers borrow it (the gate is typed `TestReadings` and Test offers on the round path; operator tools such as `weksa corpus dump` and `weksa score` still read corpus text of any split); prior turns print the held Train text), `SplitScheme::live`, `CorpusItem::cited_entities` (one owner of the id set `PacketStore::admit` and the offer share), and checkpoint-only Test readings (`Checkpoint`, `TestReadings`; `Aggregate` and `leak_diagnostic` carry no Test). Library only; since 7b `TrainItems::from_corpus` and `EncoderOffer::for_round` are crate-private and `RunStores` is their one public producer. Merges follow-ups `loop-context-builder-2` and `scorer-test-readings-type`. (spec r1) 7b. `golf-loop-round` (spec r1; branch `weksa-lowering/golf-loop-round`): the smallest loop that scores one round. `run.rs`: `RunStores` pins the corpus and entity stores of a run by canonical path and FNV-1a 64 digest at `create`, reopens only from `/run.cc` and refuses drift (corpus, entities, `PROMPT_SOURCE_DIGEST` of `context.rs`, a second pin on `lower.rs` retired in 7d, model, embedder); `round` encodes every live Train and Validation item through a `ModelPort` (`model.rs`; the one adapter `AnswerFiles` writes `.prompt` and reads `.answer`, so the model is the Claude session), reads each answer into a `PacketDraft` (`encode.rs`, the one place a model's JSON is read), admits through `PacketStore::admit` into `/round-/packets.cc`, lowers, scores once, and keeps a `RoundRecord` (`weksa.loop_round`, no text, no Test; Train rows, Validation only as means and refusal counts). `weksa loop open|round`. Not here: the spec-edit step, accept rule and composite, noise floor, Score and lowering documents, a subprocess adapter, the checkpoint door, a model lowerer (follow-ups `golf-loop-spec-edit`, `golf-loop-accept-rule`, `golf-loop-score-documents`, `golf-loop-subprocess-adapter`, `golf-loop-checkpoint-door-bound`, `golf-loop-model-lowerer`). `packet-store-inventory-binding` is met for loop packets only (`RunStores` admits with the run's inventory); `validate`, `packet proof|dao` and `lower` still take caller stores. Waits on 7a, `checkpoint-gate`, 4b. -7c. `golf-loop-edit-accept` (spec r1; branch `weksa-lowering/golf-loop-edit-accept`): one edit-accept cycle. The run works on its own copy of the inventory (`create` copies the caller's file to `/inventory.cc`, refusing a root that already holds a file of that name, and pins its digest as `RunRecord` key 11; `RunRecord.inventory` is provenance; a `run.cc` from before has no key 11 and does not open); the head is derived from the latest Accepted `Decision`. `weksa loop cycle` asks the editor for one inventory edit from `Prompt::mint(train, inventory, rev, feedback)` (Train blocks, the inventory listing and an `EditFeedback` line: whether the last edit was kept and the head's Validation composite rounded to `LADDER_STEP`, the Ladder release of Blum and Hardt 2015), reads it as an `EditDraft` (`encode.rs`; ids and revisions are the store's, `into_revision`), admits it through `InventoryStore::admit_revision` into `/cycle-/inventory.cc`, runs a candidate round under it, and `decide`s on Validation against the head's establishing round past a floor (never under one `LADDER_STEP`; from cut `cycle-floor-paired` the paired floor over rounds 1 to `FLOOR_ARMS_MIN` + 1, so a cycle needs four recorded rounds, where 7c measured the spread between rounds 1 and 2). An edit that fails is a rejected `Decision`, not an error. `RoundRecord` gains keys 16 (inventory digest) and 17 (cycle); an encode answer that is not UTF-8 is that item's parse refusal. Not here: Score and lowering documents (`golf-loop-score-docs`), Train rows in the edit prompt (`golf-loop-edit-train-feedback`), a paired floor (`golf-loop-paired-floor`), the checkpoint door and merging to `data/` (`golf-loop-checkpoint-door-bound`, `golf-loop-freeze-merge`), a model lowerer (`golf-loop-model-lowerer-next`; until it lands an entry an edit adds has no lowering row, so a cycle gains only by retiring entries or adding ones the encoder does not cite). Waits on 7b. +7c. `golf-loop-edit-accept` (spec r1; branch `weksa-lowering/golf-loop-edit-accept`): one edit-accept cycle. The run works on its own copy of the inventory (`create` copies the caller's file to `/inventory.cc`, refusing a root that already holds a file of that name, and pins its digest as `RunRecord` key 11; `RunRecord.inventory` is provenance; a `run.cc` from before has no key 11 and does not open); the head is derived from the latest Accepted `Decision`. `weksa loop cycle` asks the editor for one inventory edit from `Prompt::mint(train, inventory, rev, feedback)` (Train blocks, the inventory listing and an `EditFeedback` line: whether the last edit was kept and the head's Validation composite rounded to `LADDER_STEP`, the Ladder release of Blum and Hardt 2015), reads it as an `EditDraft` (`encode.rs`; ids and revisions are the store's, `into_revision`), admits it through `InventoryStore::admit_revision` into `/cycle-/inventory.cc`, runs a candidate round under it, and `decide`s on Validation against the head's establishing round past a floor (never under one `LADDER_STEP`; from cut `cycle-floor-paired` the paired floor over rounds 1 to `FLOOR_ARMS_MIN` + 1, so a cycle needs four recorded rounds, where 7c measured the spread between rounds 1 and 2). An edit that fails is a rejected `Decision`, not an error. `RoundRecord` gains keys 16 (inventory digest) and 17 (cycle); an encode answer that is not UTF-8 is that item's parse refusal. Not here: Score and lowering documents (`golf-loop-score-docs`), Train rows in the edit prompt (`golf-loop-edit-train-feedback`), a paired floor (`golf-loop-paired-floor`), the checkpoint door and merging to `data/` (`golf-loop-checkpoint-door-bound`, `golf-loop-freeze-merge`), a model lowerer (landed as 7d: until then an entry an edit added had no lowering row). Waits on 7b. 7d. `golf-loop-model-lowerer` (spec r1; branch `weksa-lowering/golf-loop-model-lowerer`): the round lowers through the `ModelPort`. `Prompt::lower` (`context.rs`) takes a `LoweringInput` (no text, key, group or speaker getter), the item's speaker id, the inventory at the view's revision and the item language's entity forms, and prints `LOWER_INSTRUCTIONS`, the meaning of the packet, the entities line and the inventory listing; the item key goes only to `item_keys`, and the speaker id never prints: a referent citing the item's speaker holds the fixed token `entity:speaker` (`SPEAKER_TOKEN`, a constant the same for every speaker and item, never derived from an id) and the entities map holds the speaker's forms under that token. The promise guards the id that names an entity of this item; the speaker's forms and the role token do print, because forms are shared by every item of the language (follow-up `speaker-alias-token-in-lowering-prompt`, which restored the lowerer's ability to name its speaker after #84 removed it). A speaker the language has no form for, or the port lacks, does not refuse: its referents print no entity and the map has no entry. A referent citing the real id `entity:speaker` is refused (`LowerError::SpeakerTokenTaken`); the entity store does not reserve it. The answer is `{"text"}`, read as `LoweringDraft` in `encode.rs`, held in `score_round`'s locals and scored once; it is never put in `run.cc`, a `RoundRecord`, a `Decision` or `EditFeedback`. A round has two answer phases (every encode prompt, then every lowering prompt), all or nothing: while any prompt is unanswered `packets.cc` is discarded and nothing reaches `run.cc`. `lower::lower` and the template table leave the loop (they remain `weksa lower`'s; follow-up `table-retirement`), so an entry an edit adds can be said and a cycle can buy a gain. One source pin remains: `PROMPT_SOURCE_DIGEST` covers `context.rs`, `split.rs` and `lower.rs` (`RunRecord` key 7 and `RoundRecord` key 3 are retired). The run's record pins `LowererReach` (`weksa loop open --lowerer-reach briefed-subagent|tool-less-relay`, `RunRecord` key 12) and every printed round and cycle line says it; a briefed-subagent run says its numbers are smoke-round only (rulings `lowerer-call-reach-toolless-relay`, `dao-round-seed-inventory`). A lowering is refused at `Lower` when blank, over `MAX_LOWERING_BYTES` (4096) or holding a control character other than a line break or tab; `PROMPT_SOURCE_DIGEST` also covers `packet.rs`. Not here: per-item lowering text documents (`golf-loop-score-docs`), the subprocess adapter. Waits on 7c. -7. `golf-loop-en` (7b and 7c landed the round, `ModelPort`, encoder prompt, template digest, round records and the edit-accept cycle; the rest remains): the packet store bound to one inventory store everywhere (`packet-store-inventory-binding`), a subprocess adapter, Score documents, `weksa score` in the loop, the detached unit on Yggdrasil. Waits on 3, 5, 6 and follow-up `claude-on-yggdrasil`. +7. `golf-loop-en` (7b, 7c and 7d landed the round, `ModelPort` with `AnswerFiles`, encoder and lowering prompts, template digest, round records and the edit-accept cycle; the rest remains): the packet store bound to one inventory store everywhere (`packet-store-inventory-binding`), Score documents (`golf-loop-score-docs`, `golf-loop-score-documents`), the accept rule (`golf-loop-accept-rule`), the checkpoint door and freeze merge (`golf-loop-checkpoint-door-bound`, `golf-loop-freeze-merge`). No subprocess adapter and no detached unit: `golf-loop-in-claude` and `encoder-isolation-subagent-per-prompt` rule them out. Waits on 3, 5 and 6. 8. `style-axis`: a style-embedding score behind the scorer port. Waits on 3. 9. `ja-panel`: ja-JP into the loop's panel, in two cuts on `e9cbd9b`. 9a `ja-panel-loop` (spec r1): language is the item's and the panel the run's: `Prompt::lower` takes the item's language and prints it on a language line (no prompt names English), entity forms are read in that language, `RunRecord` pins the panel (`weksa loop open --panel`), offers, mint and recall read only panel items, round means, floors and readings are per panel language, the accept rule runs `decide` per language, the recall split works on unspaced scripts, and `weksa corpus add` has no default language. 9b `ja-panel-corpus` (spec r1, after 9a): the ja-JP slice of `data/original`, model-written Japanese Train and Validation passages and eight Aozora Bunko public-domain Test sentences with full provenance, and their entities. Cut 5 is not a prerequisite: neither language lowers with ontology entries until cut 5 gives both theirs. 10. `conlang-lexicon`: the first conlang's lexicon placed at concept embeddings, its morphology data, its ontology entries and the gloss reader. Waits on 3, 5 and question `first-conlang`. From d9187aeec1799000106a98560d0d5532a783d472 Mon Sep 17 00:00:00 2001 From: Weksa Date: Fri, 13 Nov 2026 20:17:45 +0000 Subject: [PATCH 06/14] pd-name-spans: corpus import mints a public-domain work's cast and labelled name spans (#87) --- crates/weksa/src/corpus.rs | 6 + crates/weksa/src/encode.rs | 341 +++++++++++++++++- crates/weksa/src/main.rs | 117 +++++- crates/weksa/src/split.rs | 1 + crates/weksa/src/store.rs | 3 + crates/weksa/tests/cli.rs | 127 +++++++ crates/weksa/tests/common/mod.rs | 1 + crates/weksa/tests/entities.rs | 93 +++++ crates/weksa/tests/split_schema.rs | 1 + docs/name-span-annotation.md | 52 +++ docs/targets/interlingua-lowering-map.md | 3 +- schemas/cultnet/weksa.corpus_item.schema.json | 46 +++ 12 files changed, 775 insertions(+), 16 deletions(-) create mode 100644 docs/name-span-annotation.md diff --git a/crates/weksa/src/corpus.rs b/crates/weksa/src/corpus.rs index 4afa662..1fb7fa1 100644 --- a/crates/weksa/src/corpus.rs +++ b/crates/weksa/src/corpus.rs @@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize}; use crate::form::{FormError, is_slug, string_form}; use crate::packet::EntityId; +use crate::speaker::Annotator; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub struct Turn { @@ -103,6 +104,11 @@ pub struct CorpusItem { /// Which entities the text names, and where. Fixed with the text at intake. #[cultcache(key = 10)] pub names: Vec, + /// Who placed `names`: the annotator of the intake that wrote them. `None` + /// when nothing records one (the cast seeds, `weksa corpus add`, the Dao De + /// Jing bed); it labels the spans, never the line or its speaker. + #[cultcache(key = 11, default)] + pub names_by: Option, } impl CorpusItem { diff --git a/crates/weksa/src/encode.rs b/crates/weksa/src/encode.rs index 1ddabd3..ed3f496 100644 --- a/crates/weksa/src/encode.rs +++ b/crates/weksa/src/encode.rs @@ -7,7 +7,8 @@ //! provenance, constraints and extensions, and an empty trace, so a model that //! saw a Validation item cannot hand its words to whoever reads packets. The //! editor writes family, gloss, value kind and slots only; `into_revision` sets -//! every id and revision number. +//! every id and revision number. The public-domain intake file (`WorkDraft`) and +//! its name-span answer (`MarkedLine`) are read here too. use std::fmt; use std::num::NonZeroU32; @@ -15,7 +16,12 @@ use std::num::NonZeroU32; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; -use crate::corpus::CorpusItemId; +use unicode_normalization::UnicodeNormalization; + +use crate::corpus::{CorpusItem, CorpusItemId, NameSpan, Source, item_key}; +use crate::entity::{Entity, EntityOrigin, LanguageForms}; +use crate::form::is_slug; +use crate::packet::EntityId; use crate::inventory::{ Change, Family, InvId, Inventory, InventoryEntry, InventoryRevision, Slot, ValueKind, }; @@ -358,6 +364,196 @@ impl LoweringDraft { } } +/// One cast member of a work: the entity and the forms the work spells it with, +/// in the work's language, citation form first. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CastDraft { + pub entity: EntityId, + pub forms: Vec, +} + +/// One line of a work: its scene group, the speaker its speech prefix names, text. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LineDraft { + pub group: String, + pub speaker: EntityId, + pub text: String, +} + +/// A transcription of a public-domain work in hand: provenance, cast forms, +/// lines, scene groups and the speaker each speech prefix names. It holds no +/// distributor header, licence or trademark text, and nothing reads a URL. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WorkDraft { + pub slug: String, + pub language: String, + pub admitted_on: String, + pub source: Source, + pub cast: Vec, + pub lines: Vec, +} + +/// One line of a name-span answer: the line number and the line copied with +/// `[[entity:|
]]` around each place it names a cast member. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MarkedLine { + pub line: u32, + pub marked: String, +} + +impl MarkedLine { + pub fn parse(answer: &str) -> Result { + from_json(answer) + } +} + +/// Why an import was refused before any document was written. The text names +/// the kind and the line number only, never a value. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ImportRefusal { + NotPublicDomain, + CastRepeated, + NoSuchLine { line: u32 }, + LineMarkedTwice { line: u32 }, + Group { line: u32 }, + SpeakerNotInCast { line: u32 }, + Markup { line: u32 }, + NotInCast { line: u32 }, +} + +impl fmt::Display for ImportRefusal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NotPublicDomain => write!(f, "the work's source is not public domain"), + Self::CastRepeated => write!(f, "a cast entity is listed twice"), + Self::NoSuchLine { line } => write!(f, "the answer marks a line that is not there (line {line})"), + Self::LineMarkedTwice { line } => write!(f, "line {line} is marked twice"), + Self::Group { line } => write!(f, "the group of line {line} is not a slug"), + Self::SpeakerNotInCast { line } => write!(f, "the speaker of line {line} is not in the cast"), + Self::Markup { line } => write!(f, "the marked text of line {line} is not its text with markup"), + Self::NotInCast { line } => write!(f, "line {line} marks an entity outside the cast"), + } + } +} + +impl std::error::Error for ImportRefusal {} + +impl WorkDraft { + pub fn parse(file: &str) -> Result { + from_json(file) + } + + /// The cast entities and the corpus items for the lines `marked` covers, in + /// line order, each item's spans read from its markup and labelled + /// `annotator`. Decides everything before any write; whether the documents + /// are admissible is `EntityStore::put` and `CorpusStore::add`'s to refuse. + pub fn into_import( + self, + marked: &[MarkedLine], + annotator: &Annotator, + ) -> Result<(Vec, Vec), ImportRefusal> { + let Source::PublicDomain { work, .. } = &self.source else { + return Err(ImportRefusal::NotPublicDomain); + }; + let mut entities = Vec::new(); + for c in &self.cast { + if entities.iter().any(|e: &Entity| e.id == c.entity) { + return Err(ImportRefusal::CastRepeated); + } + entities.push(Entity { + id: c.entity.clone(), + forms: vec![LanguageForms { + language: self.language.clone(), + forms: c.forms.iter().map(|f| f.nfc().collect()).collect(), + pronunciation: None, + }], + origin: EntityOrigin::PublicDomain { work: work.clone() }, + }); + } + let in_cast = |id: &EntityId| entities.iter().any(|e| &e.id == id); + let mut answer: Vec<&MarkedLine> = marked.iter().collect(); + answer.sort_by_key(|m| m.line); + let mut items = Vec::new(); + for (i, m) in answer.iter().enumerate() { + let line = m.line; + if line == 0 || line as usize > self.lines.len() { + return Err(ImportRefusal::NoSuchLine { line }); + } + if i > 0 && answer[i - 1].line == line { + return Err(ImportRefusal::LineMarkedTwice { line }); + } + let draft = &self.lines[line as usize - 1]; + let group = format!("{}-{}", self.slug, draft.group); + if !is_slug(&group) { + return Err(ImportRefusal::Group { line }); + } + if !in_cast(&draft.speaker) { + return Err(ImportRefusal::SpeakerNotInCast { line }); + } + let text: String = draft.text.nfc().collect(); + if text.contains("[[") || text.contains("]]") { + return Err(ImportRefusal::Markup { line }); + } + let marked_nfc: String = m.marked.nfc().collect(); + let (plain, names) = match unmark(&marked_nfc) { + Some((plain, names)) if plain == text => (plain, names), + _ => return Err(ImportRefusal::Markup { line }), + }; + if names.iter().any(|n| !in_cast(&n.entity)) { + return Err(ImportRefusal::NotInCast { line }); + } + items.push(CorpusItem { + id: item_key(&self.slug, line), + text: plain, + language: self.language.clone(), + speaker: draft.speaker.clone(), + group, + prior_turns: Vec::new(), + source: self.source.clone(), + admitted_on: self.admitted_on.clone(), + supersedes: None, + withdrawn: false, + names, + names_by: Some(annotator.clone()), + }); + } + Ok((entities, items)) + } +} + +/// The text of `marked` with `[[entity|form]]` replaced by `form`, and the +/// byte span of each form in that text. `None` for any malformed markup. The +/// only search is for the markup delimiters. +fn unmark(marked: &str) -> Option<(String, Vec)> { + let mut out = String::with_capacity(marked.len()); + let mut names = Vec::new(); + let mut rest = marked; + while let Some(at) = rest.find("[[") { + out.push_str(&rest[..at]); + let inner = &rest[at + 2..]; + let (id, after) = inner.split_once('|')?; + let (form, tail) = after.split_once("]]")?; + if form.is_empty() || form.contains("[[") || id.contains("]]") { + return None; + } + let entity = EntityId::parse(id).ok()?; + let start = u32::try_from(out.len()).ok()?; + out.push_str(form); + names.push(NameSpan { + entity, + start, + end: u32::try_from(out.len()).ok()?, + }); + rest = tail; + } + out.push_str(rest); + Some((out, names)) +} + /// `encoder:`, a digest of the model id and the prompt digest. pub fn encoder_id(model: &str, prompt_digest: &str) -> EncoderId { let digest = fnv1a64(format!("weksa-encoder-v1:{model}:{prompt_digest}").as_bytes()); @@ -746,4 +942,145 @@ mod tests { assert!(entries.iter().all(|e| e.annotator == by)); assert_eq!(entries[1].examples, ["x"]); } + + const WORK: &str = r#"{"slug":"lantern-wager","language":"en-US","admitted_on":"2026-11-13", + "source":{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}}, + "cast":[{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:tamsin","forms":["Tamsin","Tam"]},{"entity":"entity:wexmoor","forms":["Wexmoor"]}], + "lines":[{"group":"s1","speaker":"entity:orsa","text":"Tamsin, bring the lantern."}, + {"group":"s1","speaker":"entity:tamsin","text":"Orsa, I will not, said Tam."}, + {"group":"s2","speaker":"entity:wexmoor","text":"Quiet, both of you."}]}"#; + + fn marked(line: u32, text: &str) -> MarkedLine { + MarkedLine { line, marked: text.into() } + } + + fn model_annotator() -> Annotator { + Annotator::Model { + family: crate::corpus::ModelFamily::Anthropic, + generator: "g".into(), + } + } + + fn import( + work: &str, + answer: &[MarkedLine], + ) -> Result<(Vec, Vec), ImportRefusal> { + WorkDraft::parse(work).unwrap().into_import(answer, &model_annotator()) + } + + #[test] + fn a_work_imports_its_cast_and_marked_lines() { + let answer = [ + marked(1, "[[entity:tamsin|Tamsin]], bring the lantern."), + marked(2, "[[entity:orsa|Orsa]], I will not, said [[entity:tamsin|Tam]]."), + ]; + let (entities, items) = import(WORK, &answer).unwrap(); + assert_eq!(entities.len(), 3); + for e in &entities { + assert!(matches!(&e.origin, EntityOrigin::PublicDomain { work } if work == "The Lantern Wager")); + assert_eq!(e.forms[0].language, "en-US"); + } + assert_eq!(items.len(), 2); + assert_eq!(items[0].id, "item:lantern-wager:1"); + assert_eq!(items[1].id, "item:lantern-wager:2"); + assert_eq!(items[0].group, "lantern-wager-s1"); + let span = |e: &str, text: &str, form: &str| { + let start = text.find(form).unwrap() as u32; + NameSpan { entity: EntityId::parse(e).unwrap(), start, end: start + form.len() as u32 } + }; + let t2 = &items[1].text; + assert_eq!(items[0].names, [span("entity:tamsin", &items[0].text, "Tamsin")]); + assert_eq!( + items[1].names, + [span("entity:orsa", t2, "Orsa"), span("entity:tamsin", t2, "Tam")] + ); + // "will" is unmarked and unspanned; the line's speaker is the file's. + assert_eq!(items[1].speaker, EntityId::parse("entity:tamsin").unwrap()); + for i in &items { + assert_eq!(i.names_by, Some(model_annotator())); + assert!(i.prior_turns.is_empty() && i.supersedes.is_none() && !i.withdrawn); + } + } + + #[test] + fn an_import_refuses_before_any_document() { + let ok = marked(1, "Tamsin, bring the lantern."); + let refused = |work: &str, answer: &[MarkedLine]| import(work, answer).unwrap_err(); + let cast_authored = WORK.replace( + r#"{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}}"#, + r#""OperatorWritten""#, + ); + assert_eq!(refused(&cast_authored, std::slice::from_ref(&ok)), ImportRefusal::NotPublicDomain); + let repeated = WORK.replace(r#"{"entity":"entity:orsa","forms":["Orsa"]}"#, r#"{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:orsa","forms":["O"]}"#); + assert_eq!(refused(&repeated, std::slice::from_ref(&ok)), ImportRefusal::CastRepeated); + let stranger = WORK.replace(r#""speaker":"entity:orsa""#, r#""speaker":"entity:stranger""#); + assert_eq!(refused(&stranger, std::slice::from_ref(&ok)), ImportRefusal::SpeakerNotInCast { line: 1 }); + assert_eq!(refused(WORK, &[marked(9, "x")]), ImportRefusal::NoSuchLine { line: 9 }); + assert_eq!(refused(WORK, &[marked(0, "x")]), ImportRefusal::NoSuchLine { line: 0 }); + assert_eq!( + refused(WORK, &[ok.clone(), ok.clone()]), + ImportRefusal::LineMarkedTwice { line: 1 } + ); + for bad in [ + "[[entity:tamsin|Tamsin, bring the lantern.", + "[[entity:tamsin Tamsin]], bring the lantern.", + "[[tamsin|Tamsin]], bring the lantern.", + "[[entity:tamsin|]]Tamsin, bring the lantern.", + "Tamsin, bring the lantern", + ] { + assert_eq!(refused(WORK, &[marked(1, bad)]), ImportRefusal::Markup { line: 1 }, "{bad}"); + } + assert_eq!( + refused(WORK, &[marked(1, "[[entity:stranger|Tamsin]], bring the lantern.")]), + ImportRefusal::NotInCast { line: 1 } + ); + let group = WORK.replace(r#""group":"s1""#, r#""group":"S 1""#); + assert_eq!(refused(&group, &[ok]), ImportRefusal::Group { line: 1 }); + let bracketed = WORK.replace("Tamsin, bring the lantern.", "Tamsin [[bring]] the lantern."); + assert_eq!( + refused(&bracketed, &[marked(1, "[[entity:tamsin|Tamsin]] [[entity:orsa|bring]] the lantern.")]), + ImportRefusal::Markup { line: 1 } + ); + // The refusal text names the kind and the line, no value. + assert_eq!( + ImportRefusal::Markup { line: 1 }.to_string(), + "the marked text of line 1 is not its text with markup" + ); + } + + #[test] + fn markup_offsets_are_bytes_of_the_nfc_text() { + let work = WORK + .replace("Tamsin, bring the lantern.", "Caf\u{e9} Zo\u{eb}") + .replace(r#""forms":["Orsa"]"#, r#""forms":["Zoe\u0308"]"#); + let decomposed = "Cafe\u{301} [[entity:orsa|Zoe\u{308}]]"; + let (entities, items) = import(&work, &[marked(1, decomposed)]).unwrap(); + assert_eq!(entities[0].forms[0].forms, ["Zo\u{eb}"]); + let item = &items[0]; + assert_eq!(item.text, "Caf\u{e9} Zo\u{eb}"); + let n = &item.names[0]; + assert_eq!(&item.text[n.start as usize..n.end as usize], "Zo\u{eb}"); + assert_eq!(n.start, 6); + } + + #[test] + fn a_stray_close_bracket_and_a_scattered_duplicate_answer_are_refused() { + // A text holding only `]]` is not a marked line. + let close = WORK.replace("Quiet, both of you.", "Quiet ]] both of you."); + assert_eq!( + import(&close, &[marked(3, "Quiet ]] both of you.")]).unwrap_err(), + ImportRefusal::Markup { line: 3 } + ); + // The answer's order does not matter, and a duplicate need not be adjacent. + let (_, items) = import(WORK, &[marked(3, "Quiet, both of you."), marked(1, "Tamsin, bring the lantern.")]).unwrap(); + assert_eq!(items.iter().map(|i| i.id.as_str()).collect::>(), ["item:lantern-wager:1", "item:lantern-wager:3"]); + assert_eq!( + import( + WORK, + &[marked(1, "Tamsin, bring the lantern."), marked(3, "Quiet, both of you."), marked(1, "Tamsin, bring the lantern.")] + ) + .unwrap_err(), + ImportRefusal::LineMarkedTwice { line: 1 } + ); + } } diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index e6a6cbb..21693d5 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -1,14 +1,14 @@ //! The `weksa` command line: the same validator and store admission the library //! exposes, plus text projections (`dump`) that are for review and are not stores. -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use anyhow::{Context, Result, bail}; use clap::error::{ContextKind, ContextValue, ErrorKind}; use clap::{Args, Parser, Subcommand, ValueEnum}; use unicode_normalization::UnicodeNormalization; use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn}; -use weksa::encode::{EditDraft, OntologyDraft}; +use weksa::encode::{EditDraft, MarkedLine, OntologyDraft, WorkDraft}; use weksa::entity::{Entity, EntityOrigin, LanguageForms, Pronunciation}; use weksa::inventory::{proof_revision, seed, speaker_scene_revision}; use weksa::packet::{FeatValue, FeatureValue, Grade}; @@ -20,7 +20,7 @@ use weksa::run::{ use weksa::score::ScoreInput; use weksa::{ Annotator, CorpusStore, EntityId, EntityStore, Family, InventoryStore, OllamaEmbedder, OntologyStore, OptionLabel, - PacketStore, PairRelation, Party, Scene, Slot, SpeakerStateStore, ValueKind, Scorer, Split, SplitScheme, StoreError, lower, pronounce, validate, + PacketStore, PairRelation, Party, Scene, Slot, SpeakerStateStore, ValueKind, Scorer, Split, SplitScheme, StoreError, StoreRefusal, lower, pronounce, validate, }; #[derive(Parser)] @@ -242,6 +242,22 @@ enum CorpusCommand { Withdraw { key: String, }, + /// Import a transcribed public-domain work: mint its cast as entities and + /// add each line a name-span answer covers (`docs/name-span-annotation.md`). + Import { + /// The work file: provenance, cast, lines. + work: PathBuf, + /// The annotator's answer: one `{"line": N, "marked": "..."}` per line. + names: PathBuf, + #[arg(long, requires = "generator", conflicts_with = "operator")] + model_family: Option, + /// The attestation string naming the annotating model. + #[arg(long, requires = "model_family")] + generator: Option, + /// Annotated by the operator, not a model. + #[arg(long)] + operator: bool, + }, } #[derive(Subcommand)] @@ -681,6 +697,7 @@ impl AddArgs { supersedes: self.supersedes, withdrawn: false, names, + names_by: None, }) } } @@ -698,6 +715,22 @@ fn split_text(item: &CorpusItem) -> String { .map_or_else(|| "excluded".to_string(), |s| format!("{s:?}")) } +/// The annotator the flags name: a model family with its generator, or the operator. +fn annotator( + model_family: Option, + generator: Option, + operator: bool, +) -> Result { + match (model_family, generator, operator) { + (Some(family), Some(generator), false) => Ok(Annotator::Model { + family: family.into(), + generator, + }), + (None, None, true) => Ok(Annotator::Operator), + _ => bail!("name the annotator: --model-family with --generator, or --operator"), + } +} + fn refused(e: StoreError) -> anyhow::Error { anyhow::anyhow!("{e}") } @@ -737,13 +770,78 @@ fn main() -> Result<()> { .map_err(refused)?; println!("{outcome:?} {key}"); } + Command::Corpus(CorpusCommand::Import { + work, + names, + model_family, + generator, + operator, + }) => { + let annotator = annotator(model_family, generator, operator)?; + let draft = WorkDraft::parse(&std::fs::read_to_string(&work)?)?; + let mut marked = Vec::new(); + for (i, line) in std::fs::read_to_string(&names)?.lines().enumerate() { + if line.trim().is_empty() { + continue; + } + marked.push( + MarkedLine::parse(line) + .map_err(|e| anyhow::anyhow!("names line {}: {e}", i + 1))?, + ); + } + let (new_entities, items) = draft.into_import(&marked, &annotator)?; + // Every document passes the real doors on scratch copies of the + // stores before the first write, so a refusal leaves them untouched. + let scratch = std::env::temp_dir().join(format!("weksa-import-{}", std::process::id())); + std::fs::create_dir_all(&scratch)?; + let dry = (|| -> Result<()> { + let copy = |from: &Path, name: &str| -> Result { + let to = scratch.join(name); + if from.exists() { + std::fs::copy(from, &to)?; + } + Ok(to) + }; + let mut entities = EntityStore::open(©(&cli.entities, "e.cc")?)?; + let mut store = CorpusStore::open(©(&cli.corpus, "c.cc")?)?; + for e in new_entities.clone() { + entities + .put(e) + .map_err(|e| anyhow::anyhow!("a cast entity: {e}"))?; + } + for item in items.clone() { + let key = item.id.clone(); + store.add(item, &entities).map_err(|e| match e { + StoreError::Refused(StoreRefusal::S6TextHeld { held_by, .. }) => { + anyhow::anyhow!("{key} repeats the text of {held_by}") + } + e => anyhow::anyhow!("{key}: {e}"), + })?; + } + Ok(()) + })(); + let _ = std::fs::remove_dir_all(&scratch); + dry?; + let mut entities = EntityStore::open(&cli.entities)?; + for e in new_entities { + let id = e.id.clone(); + let outcome = entities.put(e).map_err(refused)?; + println!("{outcome:?} {id}"); + } + let mut store = CorpusStore::open(&cli.corpus)?; + for item in items { + let (key, split) = (item.id.clone(), split_text(&item)); + let outcome = store.add(item, &entities).map_err(refused)?; + println!("{outcome:?} {key} {split}"); + } + } Command::Corpus(CorpusCommand::Dump) => { for i in CorpusStore::open(&cli.corpus)?.all()? { let split = split_text(&i); let withdrawn = if i.withdrawn { " withdrawn" } else { "" }; println!( - "{} [{} {} group={} {split}{withdrawn}]\n {:?}\n names: {:?}\n source: {:?}", - i.id, i.language, i.speaker, i.group, i.text, i.names, i.source + "{} [{} {} group={} {split}{withdrawn}]\n {:?}\n names: {:?} by {:?}\n source: {:?}", + i.id, i.language, i.speaker, i.group, i.text, i.names, i.names_by, i.source ); } } @@ -880,14 +978,7 @@ fn main() -> Result<()> { generator, operator, }) => { - let annotator = match (model_family, generator, operator) { - (Some(family), Some(generator), false) => Annotator::Model { - family: family.into(), - generator, - }, - (None, None, true) => Annotator::Operator, - _ => bail!("name the annotator: --model-family with --generator, or --operator"), - }; + let annotator = annotator(model_family, generator, operator)?; let inventory_store = InventoryStore::open(&cli.inventory)?; let inventory = inventory_store.inventory()?; let corpus = CorpusStore::open(&cli.corpus)?; diff --git a/crates/weksa/src/split.rs b/crates/weksa/src/split.rs index 898439d..d22d3db 100644 --- a/crates/weksa/src/split.rs +++ b/crates/weksa/src/split.rs @@ -208,6 +208,7 @@ pub(crate) mod fixture { language: "en-US".into(), speaker: EntityId::parse("entity:nibu").unwrap(), names: vec![], + names_by: None, group: group.into(), prior_turns: vec![], source, diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index 56fcc5b..9c95e70 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -344,6 +344,9 @@ impl CorpusStore { if let Some(why) = empty_attestation(&item.source) { return Err(malformed(&item.id, why)); } + if let Some(by) = &item.names_by { + check_annotator(&item.id, by)?; + } check_entities(&item, entities)?; for held in self.all()? { if normalized(&held.text) == text && item.supersedes.as_deref() != Some(&held.id) { diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 7f624b8..bfcd2d8 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -2164,3 +2164,130 @@ fn inventory_admit_cap_is_256_and_a_refusal_writes_nothing() { assert!(admitted.status.success(), "{}", out(&admitted)); assert!(out(&admitted).contains("admitted rev:2 with 256 entries")); } + +#[test] +fn corpus_import_labels_spans_and_is_idempotent() { + let d = tempfile::tempdir().unwrap(); + let work = d.path().join("work.json"); + std::fs::write( + &work, + r#"{"slug":"lantern-wager","language":"en-US","admitted_on":"2026-11-13", + "source":{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}}, + "cast":[{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:tamsin","forms":["Tamsin","Tam"]}], + "lines":[{"group":"s1","speaker":"entity:orsa","text":"Tamsin, bring the lantern."}, + {"group":"s1","speaker":"entity:tamsin","text":"Orsa, I will not."}]}"#, + ) + .unwrap(); + let names = d.path().join("names.jsonl"); + std::fs::write( + &names, + concat!( + r#"{"line":1,"marked":"[[entity:tamsin|Tamsin]], bring the lantern."}"#, "\n\n", + r#"{"line":2,"marked":"[[entity:orsa|Orsa]], I will not."}"#, "\n", + ), + ) + .unwrap(); + let (w, n) = (work.to_str().unwrap(), names.to_str().unwrap()); + let bare = weksa(d.path(), &["corpus", "import", w, n]); + assert!(!bare.status.success()); + let flags = ["--model-family", "anthropic", "--generator", "a name annotator"]; + let run = || weksa(d.path(), &[&["corpus", "import", w, n][..], &flags].concat()); + let first = run(); + assert!(first.status.success(), "{}", out(&first)); + let text = out(&first); + assert!(text.contains("Added item:lantern-wager:1 ") && text.contains("Added item:lantern-wager:2 "), "{text}"); + assert!(text.contains("Added entity:orsa"), "{text}"); + let second = out(&run()); + assert!(second.contains("Unchanged item:lantern-wager:1 ") && !second.contains("Added"), "{second}"); + let dump = out(&weksa(d.path(), &["corpus", "dump"])); + assert!(dump.contains("by Some(Model"), "{dump}"); +} + +const LANTERN_CAST: &str = r#""cast":[{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:tamsin","forms":["Tamsin","Tam"]}]"#; +const LANTERN_SOURCE: &str = r#""source":{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}}"#; + +/// Writes a work of `lines` (text per line, all said by Orsa) and an answer of +/// `marked` per line, returns both paths. +fn lantern(dir: &std::path::Path, lines: &[&str], marked: &[&str]) -> (String, String) { + let ls: Vec = lines + .iter() + .map(|t| format!(r#"{{"group":"s1","speaker":"entity:orsa","text":"{t}"}}"#)) + .collect(); + let work = dir.join("w.json"); + std::fs::write( + &work, + format!( + r#"{{"slug":"lantern-wager","language":"en-US","admitted_on":"2026-11-13",{LANTERN_SOURCE},{LANTERN_CAST},"lines":[{}]}}"#, + ls.join(",") + ), + ) + .unwrap(); + let names = dir.join("n.jsonl"); + let rows: Vec = marked + .iter() + .enumerate() + .map(|(i, m)| format!(r#"{{"line":{},"marked":"{m}"}}"#, i + 1)) + .collect(); + std::fs::write(&names, rows.join("\n")).unwrap(); + (work.to_str().unwrap().into(), names.to_str().unwrap().into()) +} + +const MODEL: [&str; 4] = ["--model-family", "anthropic", "--generator", "a name annotator"]; + +#[test] +fn a_refused_import_writes_nothing_and_a_redraw_imports() { + let d = tempfile::tempdir().unwrap(); + let lines = ["Tamsin, bring the lantern.", "Tamsin's lantern, said Orsa."]; + let (w, n) = lantern(d.path(), &lines, &["[[entity:tamsin|Tamsin]], bring the lantern.", "[[entity:tamsin|Tamsin's]] lantern, said Orsa."]); + let import = |w: &str, n: &str| weksa(d.path(), &[&["corpus", "import", w, n][..], &MODEL].concat()); + let bad = import(&w, &n); + assert!(!bad.status.success()); + assert_eq!(out(&weksa(d.path(), &["corpus", "dump"])).trim(), ""); + assert!(!d.path().join("e.cc").exists() || EntityStore::open(&d.path().join("e.cc")).unwrap().all().unwrap().is_empty()); + // The redrawn answer, different on line 1 too, imports. + let (w, n) = lantern(d.path(), &lines, &["Tamsin, bring the lantern.", "[[entity:tamsin|Tamsin]]'s lantern, said Orsa."]); + let good = import(&w, &n); + assert!(good.status.success(), "{}", out(&good)); + assert!(out(&good).contains("Added item:lantern-wager:2")); +} + +#[test] +fn a_work_repeating_a_line_is_refused_naming_both_lines() { + let d = tempfile::tempdir().unwrap(); + let (w, n) = lantern(d.path(), &["No, my lord.", "No, my lord!", "Quiet, both of you."], &["No, my lord.", "No, my lord!", "Quiet, both of you."]); + let o = weksa(d.path(), &[&["corpus", "import", &w, &n][..], &MODEL].concat()); + assert!(!o.status.success()); + let err = String::from_utf8_lossy(&o.stderr).to_string(); + assert!(err.contains("item:lantern-wager:2") && err.contains("item:lantern-wager:1"), "{err}"); + assert_eq!(out(&weksa(d.path(), &["corpus", "dump"])).trim(), ""); +} + +#[test] +fn the_annotator_flags_decide_the_label() { + let d = tempfile::tempdir().unwrap(); + let (w, n) = lantern(d.path(), &["Quiet, both of you."], &["Quiet, both of you."]); + let none = weksa(d.path(), &["corpus", "import", &w, &n]); + assert!(!none.status.success()); + let blank = weksa(d.path(), &["corpus", "import", &w, &n, "--model-family", "anthropic", "--generator", " "]); + assert!(!blank.status.success()); + assert_eq!(out(&weksa(d.path(), &["corpus", "dump"])).trim(), ""); + let op = weksa(d.path(), &["corpus", "import", &w, &n, "--operator"]); + assert!(op.status.success(), "{}", out(&op)); + let dump = out(&weksa(d.path(), &["corpus", "dump"])); + assert!(dump.contains("by Some(Operator)"), "{dump}"); +} + +#[test] +fn an_import_refused_on_a_cast_entity_leaves_both_stores_untouched() { + let d = tempfile::tempdir().unwrap(); + // entity:tamsin is already held by a cast origin, so the work's PublicDomain + // claim on it (cast position 2, after orsa) is refused by EntityStore::put. + cast_entities(d.path(), &["tamsin"]); + let (w, n) = lantern(d.path(), &["Quiet, both of you."], &["Quiet, both of you."]); + let before = std::fs::read(d.path().join("e.cc")).unwrap(); + let o = weksa(d.path(), &[&["corpus", "import", &w, &n][..], &MODEL].concat()); + assert!(!o.status.success()); + assert!(String::from_utf8_lossy(&o.stderr).contains("a cast entity"), "{}", String::from_utf8_lossy(&o.stderr)); + assert_eq!(std::fs::read(d.path().join("e.cc")).unwrap(), before); + assert!(!d.path().join("c.cc").exists() || CorpusStore::open(&d.path().join("c.cc")).unwrap().all().unwrap().is_empty()); +} diff --git a/crates/weksa/tests/common/mod.rs b/crates/weksa/tests/common/mod.rs index fa2deb9..8ee0596 100644 --- a/crates/weksa/tests/common/mod.rs +++ b/crates/weksa/tests/common/mod.rs @@ -205,6 +205,7 @@ pub fn item_in(key: &str, text: &str, group: &str, source: Source) -> CorpusItem language: "en-US".into(), speaker: eid("nibu"), names: vec![], + names_by: None, group: group.into(), prior_turns: vec![], source, diff --git a/crates/weksa/tests/entities.rs b/crates/weksa/tests/entities.rs index 5f1f7cf..9d316bf 100644 --- a/crates/weksa/tests/entities.rs +++ b/crates/weksa/tests/entities.rs @@ -502,3 +502,96 @@ fn a_malformed_pronunciation_is_refused() { } assert_eq!(store.get(&eid("kay")).unwrap(), None); } + +const LANTERN: &str = r#"{"slug":"lantern-wager","language":"en-US","admitted_on":"2026-11-13", + "source":{"PublicDomain":{"work":"The Lantern Wager","author":"A. Invented","edition_url":"https://example.invalid/lantern","licence":"public domain","jurisdiction":"US","translator":"none","edition_year":1890}}, + "cast":[{"entity":"entity:orsa","forms":["Orsa"]},{"entity":"entity:tamsin","forms":["Tamsin","Tam"]}], + "lines":[{"group":"s1","speaker":"entity:orsa","text":"Tamsin, bring the lantern."}, + {"group":"s1","speaker":"entity:tamsin","text":"Tamsin's lantern, said Orsa."}, + {"group":"s2","speaker":"entity:orsa","text":"Quiet, both of you."}]}"#; + +fn marked(line: u32, text: &str) -> weksa::encode::MarkedLine { + weksa::encode::MarkedLine { line, marked: text.into() } +} + +#[test] +fn an_imported_work_passes_the_store_doors() { + use weksa::Annotator; + use weksa::corpus::ModelFamily; + let d = tempfile::tempdir().unwrap(); + let mut entities = EntityStore::open(&d.path().join("e.cc")).unwrap(); + let mut corpus = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + let model = Annotator::Model { family: ModelFamily::Anthropic, generator: "g".into() }; + let work = || weksa::encode::WorkDraft::parse(LANTERN).unwrap(); + let (cast, items) = work() + .into_import(&[marked(1, "[[entity:tamsin|Tamsin]], bring the lantern.")], &model) + .unwrap(); + for e in cast { + assert_eq!(entities.put(e).unwrap(), AddOutcome::Added); + } + for i in items.clone() { + assert_eq!(corpus.add(i, &entities).unwrap(), AddOutcome::Added); + } + drop(corpus); + let mut corpus = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + let stored = corpus.get("item:lantern-wager:1").unwrap().unwrap(); + assert_eq!(stored.names_by, Some(model.clone())); + assert_eq!(stored.names.len(), 1); + assert_eq!(corpus.add(items[0].clone(), &entities).unwrap(), AddOutcome::Unchanged); + + // A form the entity does not list is the store's refusal, not the importer's. + let (_, bad) = work() + .into_import(&[marked(2, "[[entity:tamsin|Tamsin's]] lantern, said Orsa.")], &model) + .unwrap(); + assert!(matches!( + refusal(corpus.add(bad[0].clone(), &entities)), + StoreRefusal::NameSpanNotAForm { .. } + )); + + // An operator run over another line labels its own spans. + let (_, op) = work() + .into_import(&[marked(3, "Quiet, both of you.")], &Annotator::Operator) + .unwrap(); + assert_eq!(corpus.add(op[0].clone(), &entities).unwrap(), AddOutcome::Added); + assert_eq!( + corpus.get("item:lantern-wager:3").unwrap().unwrap().names_by, + Some(Annotator::Operator) + ); + // The first run's line keeps its label. + assert_eq!(corpus.get("item:lantern-wager:1").unwrap().unwrap().names_by, Some(model)); +} + +#[test] +fn committed_items_read_with_no_names_label() { + let data = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data"); + for sub in ["daodejing", "original"] { + let d = tempfile::tempdir().unwrap(); + let copy = d.path().join("c.cc"); + std::fs::copy(data.join(sub).join("corpus.cc"), ©).unwrap(); + let items = CorpusStore::open(©).unwrap().all().unwrap(); + assert!(!items.is_empty(), "{sub}"); + assert!(items.iter().all(|i| i.names_by.is_none()), "{sub}"); + } +} + +#[test] +fn a_blank_names_by_generator_is_refused_by_the_corpus_door() { + use weksa::Annotator; + use weksa::corpus::ModelFamily; + let d = tempfile::tempdir().unwrap(); + let mut entities = EntityStore::open(&d.path().join("e.cc")).unwrap(); + let mut corpus = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + let blank = Annotator::Model { family: ModelFamily::Anthropic, generator: " ".into() }; + let (cast, items) = weksa::encode::WorkDraft::parse(LANTERN) + .unwrap() + .into_import(&[marked(3, "Quiet, both of you.")], &blank) + .unwrap(); + for e in cast { + entities.put(e).unwrap(); + } + assert!(matches!( + refusal(corpus.add(items[0].clone(), &entities)), + StoreRefusal::Malformed { .. } + )); + assert!(corpus.all().unwrap().is_empty()); +} diff --git a/crates/weksa/tests/split_schema.rs b/crates/weksa/tests/split_schema.rs index bb6bc78..19749de 100644 --- a/crates/weksa/tests/split_schema.rs +++ b/crates/weksa/tests/split_schema.rs @@ -9,6 +9,7 @@ fn item(group: &str, source: Source) -> CorpusItem { language: "en-US".into(), speaker: weksa::EntityId::parse("entity:s").unwrap(), names: vec![], + names_by: None, group: group.into(), prior_turns: vec![], source, diff --git a/docs/name-span-annotation.md b/docs/name-span-annotation.md new file mode 100644 index 0000000..546d57b --- /dev/null +++ b/docs/name-span-annotation.md @@ -0,0 +1,52 @@ +# Name-span annotation + +How the name spans of an imported public-domain work are placed. A span has a +committed rule that produced it; this file is that rule. + +## Why + +- `pd-annotation-model-then-attest`: a labelled model places the spans of a + public-domain work; the operator may attest them afterwards. +- `leak-owned-by-flow-and-scorer-3`: spans are fixed with the text at intake and + written by no loop writer. The offsets are derived from markup the annotator + writes, never typed by a model (inline marking, as in GPT-NER, Wang et al. + 2023); `CorpusStore::add` judges range, grapheme boundary, form equality and + overlap. + +The work file (`weksa corpus import `) is a transcription of a work in +hand: provenance, cast forms, lines, scene groups and the speaker each speech +prefix names. It holds no distributor header, licence or trademark text, and the +binary fetches nothing. + +## Rubric r1 + +You are given a work's cast: each member's entity id and the forms the work spells that member with. For each numbered line, copy the line exactly, character for character, and wrap each place where the line names a cast member by one of that member's listed forms as [[|]]. Mark a form only where it refers to that cast member; a listed form used as an ordinary word, or naming someone else, stays unmarked. Do not mark pronouns, titles or descriptions unless the exact text is a listed form. When a form is listed for two members, mark the one the line refers to; when you cannot tell, leave it unmarked. Answer one JSON line per input line, in the order given, and nothing else: {"line": N, "marked": "..."}. + +## Procedure + +- One fresh model agent per work. Its brief is rubric r1, the cast as + `: , ` lines and the work's lines as ``; + nothing else (no store, no loop, no other answer). +- Import with `--model-family` and `--generator ' (name-span + rubric r1, docs/name-span-annotation.md)'`. An operator-written or + operator-reviewed answer imports with `--operator`. +- Import is all or nothing: every entity and item passes the real store doors on + scratch copies before the first write, so a refusal (a parse failure, a span + the store judges, a work that repeats a line under case or width folding) + leaves the stores untouched. Re-draw the whole answer once, log it, and import + again. A work file repeating a line's text is refused naming both lines; leave + that line out of the answer file (the rest imports with its numbering intact), + since a repeated line cannot be two corpus items. +- A line the answer does not cover is not imported by that run; a later run over + other lines of the work does not collide (keys are `item::`). +- A line whose own text holds `[[` or `]]` cannot be marked and is refused; it + can still enter by hand through `weksa corpus add --name`. + +## Label scope + +`names_by` labels the spans only, not the line or its speaker. One label per +item: a later attestation would be a superseding item with the same text and +group, but no command writes one yet; question `pd-name-span-attestation` owns +that door. +`None` means unrecorded (the cast seeds, `weksa corpus add`, the Dao De Jing +bed), not attested. diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 82d53f2..1600f6b 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -38,7 +38,7 @@ means no store holds it and nothing writes it. | Community (cut `language-ontology`, follow-up `language-ontology-communities`) | Key `community:`; type `weksa.community`, built fresh when the cut is mapped: `data/cultural-ontology/` (draft YAML profiles, read by no code) is deleted from main (cut 5e `delete-cultural-ontology`) and stays in git history before that cut's commit | Revised by commit; a speaker names the communities it belongs to by id | The cultural ontology's author (owner-attested evidence only; Discord-only content stays deleted, `discord-only-content-deleted`) | | Speaker (cut `speaker-scene`) | Key `speaker:@`; type `weksa.speaker`: stable axes as feature values of slot `Speaker` (origin, socioeconomic standing, age cohort, education and whatever the loop adds), at an inventory revision, and an `Annotator` | A new revision appends, never edits; a corpus item and every prior turn name a speaker slug, and a round records the speaker revisions it read | The annotator it names: `PersonaSnapshot` (a projection of a `gamecult.persona_state.v0` document, naming its source document and `updatedAt`), `Operator`, `WorkMetadata` (a public-domain work's own cast list and narration), or `Model { family, generator }` (the 2b `ModelFamily`) | | Scene (cut `speaker-scene`) | Key `scene:@`, the item's split group; type `weksa.scene`: participants (speaker slugs) and, per ordered pair, relation features of slot `Scene` (relative status, intimacy, in-group, audience and whatever the loop adds), at an inventory revision (provenance; a run reads the features at its own head), and an `Annotator` | As Speaker. Keyed by the group, so a scene never straddles the split. `data/original/speaker-state.cc` holds Model-annotated keeper scenes, one per Train or Validation group of both languages (`weksa scene put|dump`); a Test group has none and reads `NoScene` | As Speaker; a model-annotated scene is never Test evidence | -| Corpus item | Key `item::`, `n` canonical decimal >= 1 (no sign, no leading zero, from cut 2b); type `weksa.corpus_item` with text, language, speaker, split group, prior turns, source. Source `ModelWritten` carries a `ModelFamily` enum, the generator string as the fixture wrote it, and a prompt digest (cut 2b). The item carries no speaker state of its own: its speaker slug and group name the Speaker and Scene documents | Admitted once; the whole item is immutable (re-adding an identical item is a no-op, any difference is refused; a correction is a new item that `supersedes` the old); text already held under another key is refused unless the new item supersedes that key (cut 2b); withdrawn, never deleted, so old scores stay explainable | Importers and `weksa corpus add`, under `corpus-cast-plus-public-domain`; the source enum has no variant for Discord or any training-forbidden source (`corpus-clean`) | +| Corpus item | Key `item::`, `n` canonical decimal >= 1 (no sign, no leading zero, from cut 2b); type `weksa.corpus_item` with text, language, speaker, split group, prior turns, source and who placed its name spans (`names_by`, an `Annotator`, absent when unrecorded; cut `pd-name-spans`). Source `ModelWritten` carries a `ModelFamily` enum, the generator string as the fixture wrote it, and a prompt digest (cut 2b). The item carries no speaker state of its own: its speaker slug and group name the Speaker and Scene documents | Admitted once; the whole item is immutable (re-adding an identical item is a no-op, any difference is refused; a correction is a new item that `supersedes` the old); text already held under another key is refused unless the new item supersedes that key (cut 2b); withdrawn, never deleted, so old scores stay explainable | Importers and `weksa corpus add`, under `corpus-cast-plus-public-domain`; the source enum has no variant for Discord or any training-forbidden source (`corpus-clean`) | | Split assignment | Derived: `split(scheme, item.group)` | Fixed for a scheme; a new scheme is a new constant, never a reassignment. A model-written item whose group derives test is in no split; a superseding item keeps its predecessor's group | The hash function. Nobody assigns by hand, so there is no writer to forbid | | Loop round | Key `run: