diff --git a/crates/weksa/src/corpus.rs b/crates/weksa/src/corpus.rs index d67cc60..0beaa17 100644 --- a/crates/weksa/src/corpus.rs +++ b/crates/weksa/src/corpus.rs @@ -29,10 +29,19 @@ pub enum Source { OperatorApproved { note: String, }, + /// A public-domain work, with everything a reader needs to check the claim: + /// the edition's licence and the jurisdiction the public-domain status is + /// asserted in, who translated it (the translation is the text's copyright, + /// not the original's) and when the edition appeared. A work missing any + /// is refused at admission. PublicDomain { work: String, author: String, edition_url: String, + licence: String, + jurisdiction: String, + translator: String, + edition_year: u16, }, /// Labelled model output; never in the test split, never judged by its own family. /// `family` is `Unrecorded` when nothing records one. `generator` is the diff --git a/crates/weksa/src/daodejing.rs b/crates/weksa/src/daodejing.rs new file mode 100644 index 0000000..3a7f514 --- /dev/null +++ b/crates/weksa/src/daodejing.rs @@ -0,0 +1,326 @@ +//! The Dao De Jing test bed: inventory revision 3 of the `data/daodejing` +//! stores and the packets for the slice of James Legge's chapters in its corpus. +//! It follows `proof_packet`'s precedent: a hand-authored packet the stores +//! admit through the same doors as any other. +//! +//! The ids carry the leak labels the golf loop needs. A frame minted while +//! reading a Train chapter is generic evidence; one minted for a Validation or +//! Test chapter was written with that chapter's reference in view, so a score +//! on it measures the lowerer table's reach and not the interlingua's. + +use std::num::NonZeroU32; + +use crate::corpus::CorpusItemId; +use crate::inventory::{Change, Family, InvId, InventoryEntry, InventoryRevision}; +use crate::packet::{ + Constraints, Context, Discourse, EncoderId, FeatValue, FeatureValue, Grade, NodeId, Packet, + PacketId, PacketKind, Predication, Provenance, ProvenanceSource, Referent, RoleFill, + SCHEMA_VERSION, Stance, Trace, +}; + +const CONCEPTS: &[&str] = &[ + "sage", + "mind", + "belly", + "will", + "bone", + "empty", + "fill", + "weaken", + "strengthen", + "heaven", + "earth", + "is_long_enduring", + "continues_long", + "highest_excellence", + "water", + "true_words", + "seem_paradoxical", + "space_between_heaven_and_earth", + "bellows", + "compare", + "resemble", +]; + +/// Frame glosses, from `frame:4`. 4 is minted from a Train chapter, 5 and 6 +/// from Validation chapters, 7 and 8 from Test chapters. +const FRAMES: &[&str] = &[ + "act", + "intransitive", + "likeness", + "seems_paradoxical", + "may_be_compared", +]; + +/// Revision 3 of the Dao De Jing stores, after the seed and the proof revision. +pub fn revision() -> (InventoryRevision, Vec) { + let mut entries = Vec::new(); + for (family, glosses) in [(Family::Concept, CONCEPTS), (Family::Frame, FRAMES)] { + let first = if family == Family::Concept { 7 } else { 4 }; + for (i, gloss) in glosses.iter().enumerate() { + entries.push(InventoryEntry { + id: family.id(NonZeroU32::new(first + i as u32).expect("nonzero")), + gloss: (*gloss).to_string(), + value_kind: None, + applies_to: vec![], + introduced_rev: 3, + retired_rev: None, + }); + } + } + let changes = entries.iter().map(|e| Change::Add(e.id)).collect(); + ( + InventoryRevision { + n: 3, + parent: Some(2), + changes, + frozen: false, + }, + entries, + ) +} + +fn concept(gloss: &str) -> &'static str { + let at = CONCEPTS + .iter() + .position(|c| *c == gloss) + .expect("known concept"); + Box::leak(format!("concept:{}", 7 + at).into_boxed_str()) +} + +fn frame(gloss: &str) -> &'static str { + let at = FRAMES + .iter() + .position(|c| *c == gloss) + .expect("known frame"); + Box::leak(format!("frame:{}", 4 + at).into_boxed_str()) +} + +fn hex8(ascii: &str) -> String { + assert_eq!(ascii.len(), 8); + ascii.bytes().map(|b| format!("{b:02x}")).collect() +} + +struct Spec { + /// Eight ASCII characters; the packet id is their hex. + id: &'static str, + item: &'static str, + speech_act: &'static str, + /// `(node, referent kind id, concept gloss)`. + referents: &'static [(&'static str, &'static str, &'static str)], + /// `(node, concept gloss, frame gloss, [(role id, filler node)])`. + predications: &'static [( + &'static str, + &'static str, + &'static str, + &'static [(&'static str, &'static str)], + )], + /// The author's reading of the speaker's felt state: `(feature gloss, permille)`. + stance: &'static [(&'static str, u16)], + note: &'static str, +} + +const AGENT: &str = "role:1"; +const PATIENT: &str = "role:2"; +const EXPERIENCER: &str = "role:3"; +const GOAL: &str = "role:6"; +const PERSON: &str = "referent_kind:1"; +const ORGANISM: &str = "referent_kind:3"; +const PLACE: &str = "referent_kind:4"; +const ABSTRACT: &str = "referent_kind:11"; + +/// Feature ids of the seed: restraint is the tenth stance feature. +fn stance_feature(gloss: &str) -> InvId { + let n = [ + "anger", + "fear", + "sadness", + "joy", + "disgust", + "surprise", + "contempt", + "affection", + "shame", + "restraint", + ] + .iter() + .position(|g| *g == gloss) + .expect("stance feature"); + // Features before the stance block: polarity, tense, aspect, modality, + // evidentiality, hedge, certainty, irony. + InvId::parse(&format!("feature:{}", 9 + n)).expect("feature id") +} + +const SPECS: &[Spec] = &[ + // Chapter 3 section 2. Train. + Spec { + id: "dao-0302", + item: "item:daodejing:1", + speech_act: "speech_act:1", + referents: &[ + ("n1", PERSON, "sage"), + ("n2", ABSTRACT, "mind"), + ("n3", ORGANISM, "belly"), + ("n4", ABSTRACT, "will"), + ("n5", ORGANISM, "bone"), + ], + predications: &[ + ("n6", "empty", "act", &[(AGENT, "n1"), (PATIENT, "n2")]), + ("n7", "fill", "act", &[(AGENT, "n1"), (PATIENT, "n3")]), + ("n8", "weaken", "act", &[(AGENT, "n1"), (PATIENT, "n4")]), + ("n9", "strengthen", "act", &[(AGENT, "n1"), (PATIENT, "n5")]), + ], + stance: &[("restraint", 800)], + note: "Four acts of one agent; 'in the exercise of his government' and 'their' are not carried.", + }, + // Chapter 7 section 1, first sentence. Validation. + Spec { + id: "dao-0701", + item: "item:daodejing:2", + speech_act: "speech_act:1", + referents: &[("n1", PLACE, "heaven"), ("n2", PLACE, "earth")], + predications: &[ + ( + "n3", + "is_long_enduring", + "intransitive", + &[(EXPERIENCER, "n1")], + ), + ( + "n4", + "continues_long", + "intransitive", + &[(EXPERIENCER, "n2")], + ), + ], + stance: &[("restraint", 800)], + note: "Heaven and earth as places; whether they are entities is a ruling the ontology owes.", + }, + // Chapter 8 section 1, first sentence. Validation. + Spec { + id: "dao-0801", + item: "item:daodejing:3", + speech_act: "speech_act:1", + referents: &[ + ("n1", ABSTRACT, "highest_excellence"), + ("n2", ABSTRACT, "water"), + ], + predications: &[( + "n3", + "resemble", + "likeness", + &[(EXPERIENCER, "n1"), (GOAL, "n2")], + )], + stance: &[("restraint", 800), ("affection", 300)], + note: "The metaphor is a plain likeness predication; the tenor and vehicle are not marked.", + }, + // Chapter 78 section 4. Test. + Spec { + id: "dao-7804", + item: "item:daodejing:4", + speech_act: "speech_act:1", + referents: &[("n1", ABSTRACT, "true_words")], + predications: &[( + "n2", + "seem_paradoxical", + "seems_paradoxical", + &[(EXPERIENCER, "n1")], + )], + stance: &[("restraint", 800)], + note: "The whole paradox is in the referent's concept and the frame; the packet holds no more than the surface does.", + }, + // Chapter 5 section 2, first sentence. Test. + Spec { + id: "dao-0502", + item: "item:daodejing:5", + speech_act: "speech_act:2", + referents: &[ + ("n1", PLACE, "space_between_heaven_and_earth"), + ("n2", ARTIFACT, "bellows"), + ], + predications: &[( + "n3", + "compare", + "may_be_compared", + &[(EXPERIENCER, "n1"), (GOAL, "n2")], + )], + stance: &[("restraint", 800)], + note: "A rhetorical question; the speech act is ask and the negated 'may not' is dropped.", + }, +]; + +const ARTIFACT: &str = "referent_kind:5"; + +/// The packets of the slice, each minted for its corpus item. +pub fn packets() -> Vec { + SPECS.iter().map(build).collect() +} + +fn build(spec: &Spec) -> Packet { + let id = |s: &str| InvId::parse(s).expect("ids are well formed"); + let node = |s: &str| NodeId::parse(s).expect("nodes are well formed"); + Packet { + id: PacketId::parse(&format!("packet:{}", hex8(spec.id))).expect("packet id"), + schema_version: SCHEMA_VERSION, + inventory_rev: 3, + kind: PacketKind::CorpusExample, + provenance: Provenance { + source: ProvenanceSource::Authored, + corpus_item: CorpusItemId::parse(spec.item).expect("item key"), + encoder: EncoderId::parse(&format!("encoder:{}", hex8("claude-s"))).expect("encoder"), + confidence: Grade::new(600).expect("grade"), + }, + context: Context::default(), + discourse: Discourse { + speech_act: Some(id(spec.speech_act)), + addressee: None, + addressee_consent: None, + features: vec![], + }, + referents: spec + .referents + .iter() + .map(|(n, kind, gloss)| Referent { + id: node(n), + kind: id(kind), + concept: id(concept(gloss)), + entity: None, + features: vec![], + }) + .collect(), + predications: spec + .predications + .iter() + .map(|(n, c, f, roles)| Predication { + id: node(n), + concept: id(concept(c)), + frame: id(frame(f)), + roles: roles + .iter() + .map(|(role, filler)| RoleFill { + role: id(role), + filler: node(filler), + }) + .collect(), + features: vec![], + }) + .collect(), + relations: vec![], + constraints: Constraints::default(), + extensions: vec![], + trace: Trace { + notes: vec![spec.note.to_string()], + rejected_readings: vec![], + }, + stance: Stance { + features: spec + .stance + .iter() + .map(|(g, v)| FeatureValue { + feature: stance_feature(g), + value: FeatValue::Grade(Grade::new(*v).expect("grade")), + }) + .collect(), + }, + } +} diff --git a/crates/weksa/src/lib.rs b/crates/weksa/src/lib.rs index a0ef6c6..d826c0e 100644 --- a/crates/weksa/src/lib.rs +++ b/crates/weksa/src/lib.rs @@ -4,6 +4,7 @@ //! (chrF, meaning cosine, leak diagnostics). pub mod corpus; +pub mod daodejing; pub mod entity; pub mod form; pub mod inventory; diff --git a/crates/weksa/src/lower.rs b/crates/weksa/src/lower.rs index 299defb..619eb41 100644 --- a/crates/weksa/src/lower.rs +++ b/crates/weksa/src/lower.rs @@ -65,6 +65,33 @@ const CONCEPTS: &[Row] = &[ ("concept:3", "player", "player"), ("concept:4", "control", "control"), ("concept:5", "ship_mind", "ship mind"), + // The Dao De Jing test bed (`daodejing.rs`). Nouns, then the verbs a + // `{verb}` frame speaks. + ("concept:7", "sage", "sage"), + ("concept:8", "mind", "minds"), + ("concept:9", "belly", "bellies"), + ("concept:10", "will", "wills"), + ("concept:11", "bone", "bones"), + ("concept:12", "empty", "empties"), + ("concept:13", "fill", "fills"), + ("concept:14", "weaken", "weakens"), + ("concept:15", "strengthen", "strengthens"), + ("concept:16", "heaven", "heaven"), + ("concept:17", "earth", "earth"), + ("concept:18", "is_long_enduring", "is long-enduring"), + ("concept:19", "continues_long", "continues long"), + ("concept:20", "highest_excellence", "highest excellence"), + ("concept:21", "water", "water"), + ("concept:22", "true_words", "words that are strictly true"), + ("concept:23", "seem_paradoxical", "seem paradoxical"), + ( + "concept:24", + "space_between_heaven_and_earth", + "space between heaven and earth", + ), + ("concept:25", "bellows", "bellows"), + ("concept:26", "compare", "compare"), + ("concept:27", "resemble", "resemble"), ]; /// Frames: the clause a predication of that frame says. `{gloss}` is the filler @@ -81,6 +108,21 @@ const FRAMES: &[Row] = &[ "request_intervention", "{agent} asked {patient} to stop", ), + // The Dao De Jing test bed. Frame 4 was written from a Train chapter, 5 and + // 6 from Validation chapters, 7 and 8 from Test chapters. + ("frame:4", "act", "{agent} {verb} {patient}"), + ("frame:5", "intransitive", "{experiencer} {verb}"), + ("frame:6", "likeness", "{experiencer} is like {goal}"), + ( + "frame:7", + "seems_paradoxical", + "{experiencer} seem paradoxical", + ), + ( + "frame:8", + "may_be_compared", + "{experiencer} may be compared to {goal}", + ), ]; /// The frame of "{agent} asked {patient} to stop": the one frame that reads @@ -90,6 +132,7 @@ const REQUEST_FRAME: &str = "frame:3"; /// Speech acts: the opener of the first sentence. const SPEECH_ACTS: &[Row] = &[ ("speech_act:1", "assert", ""), + ("speech_act:2", "ask", ""), ("speech_act:5", "warn", "Warning: "), ]; @@ -353,6 +396,10 @@ fn clause( let close = rest[open..].find('}').map_or(rest.len(), |c| open + c); let name = &rest[open + 1..close]; rest = &rest[(close + 1).min(rest.len())..]; + if name == "verb" { + out.push_str(row(CONCEPTS, p.concept, inv, rev)?); + continue; + } let role = inv .entries() .find(|e| e.family() == Family::Role && e.gloss == name && e.live_at(rev)) diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index bc61c7e..b63fee7 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -76,6 +76,8 @@ enum Command { enum PacketCommand { /// Admit the nibu-proof-001 packet through `PacketStore::admit`. Proof { store: PathBuf }, + /// Admit the Dao De Jing slice's packets through `PacketStore::admit`. + Dao { store: PathBuf }, } #[derive(Args)] @@ -110,6 +112,8 @@ enum InventoryCommand { Seed, /// Admit revision 2, the concepts and frames the nibu-proof-001 packet names. Proof, + /// Admit revision 3 of the Dao De Jing test bed stores (after seed and proof). + Dao, } #[derive(Subcommand)] @@ -288,6 +292,16 @@ struct AddArgs { author: Option, #[arg(long)] edition_url: Option, + /// The edition's licence, for `public-domain`. + #[arg(long)] + licence: Option, + /// Where the public-domain status is asserted, for `public-domain`. + #[arg(long)] + jurisdiction: Option, + #[arg(long)] + translator: Option, + #[arg(long)] + edition_year: Option, #[arg(long = "model-family", value_enum)] family: Option, /// The generator string the fixture or run wrote, verbatim. @@ -312,7 +326,15 @@ impl AddArgs { SourceKind::CastAuthored => &["persona", "fixture", "author"], SourceKind::OperatorWritten => &[], SourceKind::OperatorApproved => &["note"], - SourceKind::PublicDomain => &["work", "author", "edition-url"], + SourceKind::PublicDomain => &[ + "work", + "author", + "edition-url", + "licence", + "jurisdiction", + "translator", + "edition-year", + ], SourceKind::ModelWritten => &["model-family", "generator", "prompt-digest"], }; let given = [ @@ -322,6 +344,10 @@ impl AddArgs { ("note", self.note.is_some()), ("work", self.work.is_some()), ("edition-url", self.edition_url.is_some()), + ("licence", self.licence.is_some()), + ("jurisdiction", self.jurisdiction.is_some()), + ("translator", self.translator.is_some()), + ("edition-year", self.edition_year.is_some()), ("model-family", self.family.is_some()), ("generator", self.generator.is_some()), ("prompt-digest", self.prompt_digest.is_some()), @@ -348,6 +374,12 @@ impl AddArgs { work: get(&self.work, "work")?, author: get(&self.author, "author")?, edition_url: get(&self.edition_url, "edition-url")?, + licence: get(&self.licence, "licence")?, + jurisdiction: get(&self.jurisdiction, "jurisdiction")?, + translator: get(&self.translator, "translator")?, + edition_year: self + .edition_year + .context("this source needs --edition-year")?, }, SourceKind::ModelWritten => Source::ModelWritten { family: self @@ -519,6 +551,27 @@ fn main() -> Result<()> { .map_err(refused)?; println!("admitted rev:2 with {count} entries"); } + Command::Inventory(InventoryCommand::Dao) => { + let (rev, entries) = weksa::daodejing::revision(); + let count = entries.len(); + InventoryStore::open(&cli.inventory)? + .admit_revision(rev, entries) + .map_err(refused)?; + println!("admitted rev:3 with {count} entries"); + } + Command::Packet(PacketCommand::Dao { store }) => { + let mut packets = PacketStore::open(&store)?; + let inventory = InventoryStore::open(&cli.inventory)?; + let corpus = CorpusStore::open(&cli.corpus)?; + let entities = EntityStore::open(&cli.entities)?; + for packet in weksa::daodejing::packets() { + let key = packet.id.to_string(); + packets + .admit(packet, &inventory, &corpus, &entities) + .map_err(refused)?; + println!("admitted {key}"); + } + } Command::Packet(PacketCommand::Proof { store }) => { let packet = proof_packet(); let key = packet.id.to_string(); diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index 764a7be..a32c77f 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -402,6 +402,18 @@ fn empty_attestation(source: &Source) -> Option<&'static str> { Source::PublicDomain { edition_url, .. } if empty(edition_url) => { Some("a public-domain source names its edition url") } + Source::PublicDomain { licence, .. } if empty(licence) => { + Some("a public-domain source names its licence") + } + Source::PublicDomain { jurisdiction, .. } if empty(jurisdiction) => { + Some("a public-domain source names its jurisdiction") + } + Source::PublicDomain { translator, .. } if empty(translator) => { + Some("a public-domain source names its translator") + } + Source::PublicDomain { + edition_year: 0, .. + } => Some("a public-domain source names its edition year"), Source::ModelWritten { generator, .. } if empty_some(generator) => { Some("a model-written source names a non-empty generator") } diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index b2e5b58..94ae9b3 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -210,6 +210,14 @@ fn model_written_and_public_domain_sources_name_their_provenance() { "a", "--edition-url", "https://example.org/w", + "--licence", + "Public domain", + "--jurisdiction", + "US", + "--translator", + "T. Translator", + "--edition-year", + "1891", ]); assert!(pd.status.success(), "{}", out(&pd)); let bad_turn = run(&[ @@ -566,6 +574,14 @@ fn source_flags_must_match_the_source() { "a", "--edition-url", "u", + "--licence", + "l", + "--jurisdiction", + "j", + "--translator", + "t", + "--edition-year", + "1891", "--note", "n", ], @@ -618,6 +634,78 @@ fn source_flags_must_match_the_source() { &["--work", "w", "--edition-url", "u"], "--author", ), + ( + "public-domain", + &[ + "--work", + "w", + "--author", + "a", + "--edition-url", + "u", + "--jurisdiction", + "j", + "--translator", + "t", + "--edition-year", + "1891", + ], + "--licence", + ), + ( + "public-domain", + &[ + "--work", + "w", + "--author", + "a", + "--edition-url", + "u", + "--licence", + "l", + "--translator", + "t", + "--edition-year", + "1891", + ], + "--jurisdiction", + ), + ( + "public-domain", + &[ + "--work", + "w", + "--author", + "a", + "--edition-url", + "u", + "--licence", + "l", + "--jurisdiction", + "j", + "--edition-year", + "1891", + ], + "--translator", + ), + ( + "public-domain", + &[ + "--work", + "w", + "--author", + "a", + "--edition-url", + "u", + "--licence", + "l", + "--jurisdiction", + "j", + "--translator", + "t", + ], + "--edition-year", + ), // Empty provenance strings. ( "model-written", diff --git a/crates/weksa/tests/daodejing.rs b/crates/weksa/tests/daodejing.rs new file mode 100644 index 0000000..7e5f9ae --- /dev/null +++ b/crates/weksa/tests/daodejing.rs @@ -0,0 +1,71 @@ +//! The Dao De Jing test bed (`data/daodejing`): its Legge items carry full +//! public-domain provenance, and its packets are admitted and lower to the +//! English the first golf-loop run scored (`docs/targets/dao-de-jing-test-bed.md`). + +use std::path::Path; + +use weksa::daodejing::packets; +use weksa::{CorpusStore, EntityStore, InventoryStore, PacketStore, Source, lower}; + +fn root() -> std::path::PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/daodejing") +} + +#[test] +fn every_legge_item_names_licence_jurisdiction_translator_and_edition_year() { + let items = CorpusStore::open(&root().join("corpus.cc")) + .unwrap() + .all() + .unwrap(); + assert_eq!(items.len(), 5); + for item in items { + let Source::PublicDomain { + licence, + jurisdiction, + translator, + edition_year, + .. + } = item.source + else { + panic!("{} is not a public-domain item", item.id); + }; + assert!( + !licence.is_empty() && !jurisdiction.is_empty(), + "{}", + item.id + ); + assert_eq!((translator.as_str(), edition_year), ("James Legge", 1891)); + } +} + +#[test] +fn the_slice_packets_are_admitted_and_lower_to_the_scored_english() { + let inventory = InventoryStore::open(&root().join("inventory.cc")).unwrap(); + let corpus = CorpusStore::open(&root().join("corpus.cc")).unwrap(); + let entities = EntityStore::open(&root().join("entity.cc")).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let mut store = PacketStore::open(&dir.path().join("p.cc")).unwrap(); + let inv = inventory.inventory().unwrap(); + let lowered: Vec = packets() + .into_iter() + .map(|p| { + let view = store.admit(p, &inventory, &corpus, &entities).unwrap(); + lower(&view, &inv, &entities).unwrap() + }) + .collect(); + assert_eq!( + lowered, + [ + "The sage empties the minds. The sage fills the bellies. The sage weakens the wills. The sage strengthens the bones.", + "The heaven is long-enduring. The earth continues long.", + "The highest excellence is like the water.", + "The words that are strictly true seem paradoxical.", + "The space between heaven and earth may be compared to the bellows.", + ] + ); + // The committed packet store holds exactly what the module builds. + let stored = PacketStore::open(&root().join("packets.cc")).unwrap(); + for p in packets() { + assert_eq!(stored.get(&p.id.to_string()).unwrap(), Some(p)); + } +} diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs index cf5ba4a..be9acc3 100644 --- a/crates/weksa/tests/stores.rs +++ b/crates/weksa/tests/stores.rs @@ -371,7 +371,7 @@ fn an_attestation_holds_something_readable() { let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); let s = |x: &str| x.to_string(); type Build = fn(String) -> Source; - let builders: [(Build, &'static str); 9] = [ + let builders: [(Build, &'static str); 12] = [ ( |v| Source::CastAuthored { persona: v, @@ -405,6 +405,10 @@ fn an_attestation_holds_something_readable() { work: v, author: "a".into(), edition_url: "u".into(), + licence: "l".into(), + jurisdiction: "j".into(), + translator: "t".into(), + edition_year: 1891, }, "a public-domain source names its work", ), @@ -413,6 +417,10 @@ fn an_attestation_holds_something_readable() { work: "w".into(), author: v, edition_url: "u".into(), + licence: "l".into(), + jurisdiction: "j".into(), + translator: "t".into(), + edition_year: 1891, }, "a public-domain source names its author", ), @@ -421,9 +429,49 @@ fn an_attestation_holds_something_readable() { work: "w".into(), author: "a".into(), edition_url: v, + licence: "l".into(), + jurisdiction: "j".into(), + translator: "t".into(), + edition_year: 1891, }, "a public-domain source names its edition url", ), + ( + |v| Source::PublicDomain { + work: "w".into(), + author: "a".into(), + edition_url: "u".into(), + licence: v, + jurisdiction: "j".into(), + translator: "t".into(), + edition_year: 1891, + }, + "a public-domain source names its licence", + ), + ( + |v| Source::PublicDomain { + work: "w".into(), + author: "a".into(), + edition_url: "u".into(), + licence: "l".into(), + jurisdiction: v, + translator: "t".into(), + edition_year: 1891, + }, + "a public-domain source names its jurisdiction", + ), + ( + |v| Source::PublicDomain { + work: "w".into(), + author: "a".into(), + edition_url: "u".into(), + licence: "l".into(), + jurisdiction: "j".into(), + translator: v, + edition_year: 1891, + }, + "a public-domain source names its translator", + ), ( |v| Source::ModelWritten { family: ModelFamily::Unrecorded, @@ -528,6 +576,10 @@ fn an_item_source_attests_with_non_empty_strings() { work: s(work), author: s(author), edition_url: s(url), + licence: s("l"), + jurisdiction: s("j"), + translator: s("t"), + edition_year: 1891, }; let cases: [(Source, &'static str); 9] = [ ( @@ -1292,3 +1344,26 @@ fn seed_reseeded() { replay.add(i, &entities).unwrap(); } } + +#[test] +fn a_public_domain_source_without_an_edition_year_is_refused() { + let d = dir(); + let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + let key = "item:pd:1"; + let source = |edition_year| Source::PublicDomain { + work: "w".into(), + author: "a".into(), + edition_url: "u".into(), + licence: "l".into(), + jurisdiction: "j".into(), + translator: "t".into(), + edition_year, + }; + assert_eq!( + refusal(store.add(item_in(key, "line", "scene-1", source(0)), &ents(&d))), + malformed(key, "a public-domain source names its edition year") + ); + store + .add(item_in(key, "line", "scene-1", source(1891)), &ents(&d)) + .unwrap(); +} diff --git a/data/daodejing/corpus.cc b/data/daodejing/corpus.cc new file mode 100644 index 0000000..45bcda5 --- /dev/null +++ b/data/daodejing/corpus.cc @@ -0,0 +1 @@ +“²cultcache.store.v1‘—±weksa.corpus_item±weksa.corpus_item´weksa.corpus_item.v1±weksa.corpus_itemÙV{"schemaName":"weksa.corpus_item","schemaVersion":"weksa.corpus_item.v1","members":[]}‘±weksa.corpus_item•”°item:daodejing:1±weksa.corpus_item´2026-11-07T02:09:00ZÅ4›°item:daodejing:1Ù’Therefore the sage, in the exercise of his government, empties their minds, fills their bellies, weakens their wills, and strengthens their bones.¥en-US¬entity:laozi³daodejing-legge-ch3¬PublicDomain—ÙBThe Tao Teh King, or the Tao and its Characteristics (Dao De Jing)¥LaoziÙ$https://www.gutenberg.org/ebooks/216ÙPublic domain (author d. c. 5th century BCE; translator d. 1897); Project Gutenberg eBook 216 text used without its trademark or licence matterÙ7United States and life+70 jurisdictions (Legge d. 1897)«James LeggeÍcª2026-11-07À”°item:daodejing:2±weksa.corpus_item´2026-11-07T02:09:00ZÅÓ›°item:daodejing:2Ù1Heaven is long-enduring and earth continues long.¥en-US¬entity:laozi³daodejing-legge-ch7¬PublicDomain—ÙBThe Tao Teh King, or the Tao and its Characteristics (Dao De Jing)¥LaoziÙ$https://www.gutenberg.org/ebooks/216ÙPublic domain (author d. c. 5th century BCE; translator d. 1897); Project Gutenberg eBook 216 text used without its trademark or licence matterÙ7United States and life+70 jurisdictions (Legge d. 1897)«James LeggeÍcª2026-11-07À”°item:daodejing:3±weksa.corpus_item´2026-11-07T02:09:00ZÅÑ›°item:daodejing:3Ù/The highest excellence is like (that of) water.¥en-US¬entity:laozi³daodejing-legge-ch8¬PublicDomain—ÙBThe Tao Teh King, or the Tao and its Characteristics (Dao De Jing)¥LaoziÙ$https://www.gutenberg.org/ebooks/216ÙPublic domain (author d. c. 5th century BCE; translator d. 1897); Project Gutenberg eBook 216 text used without its trademark or licence matterÙ7United States and life+70 jurisdictions (Legge d. 1897)«James LeggeÍcª2026-11-07À”°item:daodejing:4±weksa.corpus_item´2026-11-07T02:09:00ZÅ×›°item:daodejing:4Ù4Words that are strictly true seem to be paradoxical.¥en-US¬entity:laozi´daodejing-legge-ch78¬PublicDomain—ÙBThe Tao Teh King, or the Tao and its Characteristics (Dao De Jing)¥LaoziÙ$https://www.gutenberg.org/ebooks/216ÙPublic domain (author d. c. 5th century BCE; translator d. 1897); Project Gutenberg eBook 216 text used without its trademark or licence matterÙ7United States and life+70 jurisdictions (Legge d. 1897)«James LeggeÍcª2026-11-07À”°item:daodejing:5±weksa.corpus_item´2026-11-07T02:09:00ZÅæ›°item:daodejing:5ÙDMay not the space between heaven and earth be compared to a bellows?¥en-US¬entity:laozi³daodejing-legge-ch5¬PublicDomain—ÙBThe Tao Teh King, or the Tao and its Characteristics (Dao De Jing)¥LaoziÙ$https://www.gutenberg.org/ebooks/216ÙPublic domain (author d. c. 5th century BCE; translator d. 1897); Project Gutenberg eBook 216 text used without its trademark or licence matterÙ7United States and life+70 jurisdictions (Legge d. 1897)«James LeggeÍcª2026-11-07À \ No newline at end of file diff --git a/data/daodejing/entity.cc b/data/daodejing/entity.cc new file mode 100644 index 0000000..c11c6df --- /dev/null +++ b/data/daodejing/entity.cc @@ -0,0 +1 @@ +“²cultcache.store.v1‘—¬weksa.entity¬weksa.entity¯weksa.entity.v1¬weksa.entityÙL{"schemaName":"weksa.entity","schemaVersion":"weksa.entity.v1","members":[]}‘¬weksa.entity‘”¬entity:laozi¬weksa.entity´2026-11-07T02:09:00ZÄb“¬entity:laozi‘’¥en-US‘¥Laozi¬PublicDomain‘Ù4The Tao Teh King, or the Tao and its Characteristics \ No newline at end of file diff --git a/data/daodejing/inventory.cc b/data/daodejing/inventory.cc new file mode 100644 index 0000000..373eb3c Binary files /dev/null and b/data/daodejing/inventory.cc differ diff --git a/data/daodejing/packets.cc b/data/daodejing/packets.cc new file mode 100644 index 0000000..35b5bbe --- /dev/null +++ b/data/daodejing/packets.cc @@ -0,0 +1 @@ +“²cultcache.store.v1‘—¸weksa.interlingua_packet¸weksa.interlingua_packet»weksa.interlingua_packet.v1¸weksa.interlingua_packetÙd{"schemaName":"weksa.interlingua_packet","schemaVersion":"weksa.interlingua_packet.v1","members":[]}‘¸weksa.interlingua_packet•”·packet:64616f2d30333032¸weksa.interlingua_packet´2026-11-09T10:16:16ZÅSž·packet:64616f2d30333032­CorpusExample”¨Authored°item:daodejing:1¸encoder:636c617564652d73ÍX”ÀÀ”¬speech_act:1ÀÀ••¢n1¯referent_kind:1©concept:7À•¢n2°referent_kind:11©concept:8À•¢n3¯referent_kind:3©concept:9À•¢n4°referent_kind:11ªconcept:10À•¢n5¯referent_kind:3ªconcept:11À”•¢n6ªconcept:12§frame:4’’¦role:1¢n1’¦role:2¢n2•¢n7ªconcept:13§frame:4’’¦role:1¢n1’¦role:2¢n3•¢n8ªconcept:14§frame:4’’¦role:1¢n1’¦role:2¢n4•¢n9ªconcept:15§frame:4’’¦role:1¢n1’¦role:2¢n5“ÂÀ’‘ÙXFour acts of one agent; 'in the exercise of his government' and 'their' are not carried.‘‘’ªfeature:18¥GradeÍ ”·packet:64616f2d30353032¸weksa.interlingua_packet´2026-11-09T10:16:16ZÅ^ž·packet:64616f2d30353032­CorpusExample”¨Authored°item:daodejing:5¸encoder:636c617564652d73ÍX”ÀÀ”¬speech_act:2ÀÀ’•¢n1¯referent_kind:4ªconcept:24À•¢n2¯referent_kind:5ªconcept:25À‘•¢n3ªconcept:26§frame:8’’¦role:3¢n1’¦role:6¢n2“ÂÀ’‘ÙRA rhetorical question; the speech act is ask and the negated 'may not' is dropped.‘‘’ªfeature:18¥GradeÍ ”·packet:64616f2d30373031¸weksa.interlingua_packet´2026-11-09T10:16:16ZÅyž·packet:64616f2d30373031­CorpusExample”¨Authored°item:daodejing:2¸encoder:636c617564652d73ÍX”ÀÀ”¬speech_act:1ÀÀ’•¢n1¯referent_kind:4ªconcept:16À•¢n2¯referent_kind:4ªconcept:17À’•¢n3ªconcept:18§frame:5‘’¦role:3¢n1•¢n4ªconcept:19§frame:5‘’¦role:3¢n2“ÂÀ’‘ÙTHeaven and earth as places; whether they are entities is a ruling the ontology owes.‘‘’ªfeature:18¥GradeÍ ”·packet:64616f2d30383031¸weksa.interlingua_packet´2026-11-09T10:16:16ZÅwž·packet:64616f2d30383031­CorpusExample”¨Authored°item:daodejing:3¸encoder:636c617564652d73ÍX”ÀÀ”¬speech_act:1ÀÀ’•¢n1°referent_kind:11ªconcept:20À•¢n2°referent_kind:11ªconcept:21À‘•¢n3ªconcept:27§frame:6’’¦role:3¢n1’¦role:6¢n2“ÂÀ’‘ÙSThe metaphor is a plain likeness predication; the tenor and vehicle are not marked.‘’’ªfeature:18¥GradeÍ ’ªfeature:16¥GradeÍ,”·packet:64616f2d37383034¸weksa.interlingua_packet´2026-11-09T10:16:16ZÅNž·packet:64616f2d37383034­CorpusExample”¨Authored°item:daodejing:4¸encoder:636c617564652d73ÍX”ÀÀ”¬speech_act:1ÀÀ‘•¢n1°referent_kind:11ªconcept:22À‘•¢n2ªconcept:23§frame:7‘’¦role:3¢n1“ÂÀ’‘ÙmThe whole paradox is in the referent's concept and the frame; the packet holds no more than the surface does.‘‘’ªfeature:18¥GradeÍ \ No newline at end of file diff --git a/docs/targets/dao-de-jing-test-bed.md b/docs/targets/dao-de-jing-test-bed.md new file mode 100644 index 0000000..4f4de2e --- /dev/null +++ b/docs/targets/dao-de-jing-test-bed.md @@ -0,0 +1,116 @@ +# Dao De Jing test bed + +A lowering test bed on James Legge's public-domain translation of the Dao De +Jing, from Meta's analogy: the Dao has many varied human translations, so it is +a ready-made field of precedents for what one meaning can be lowered to. Meta +proposed it in #weksa:gamecult.org on 2026-11-06 (Matrix event +`$2_P0mbtbnnsP4I61UhVMlatuDSTjFXMOuGa4SSbhGmc`: public-domain translations only +as the scoring corpus, copyrighted ones cited without copying) and noted the +ontology consequence (event `$E4VHwS-zFEk9Fa-daFX8MkfoySn0iX0m1grltEJ9A8Y`: +model an imperial native term as a term with several context-tagged readings +plus a separately recorded chosen gloss per lowering). + +## What is here + +- `data/daodejing/`: its own stores (inventory revisions 1 to 3, one entity, five + corpus items, five packets), apart from the production stores in `data/`. + Revision 3 is `weksa::daodejing::revision()`; the packets are + `weksa::daodejing::packets()`. The scene revision (`scene_revision()`, also + numbered 3) is not applied in these stores and will need renumbering when it + is applied anywhere both meet. +- Rebuild: `weksa --corpus C --inventory I --entities E` with `inventory seed`, + `inventory proof`, `inventory dao`, `entity put entity:laozi ...`, + `corpus add ... --source public-domain` per item, then `packet dao `. +- Source: Project Gutenberg eBook 216, "The Tao Teh King, or the Tao and its + Characteristics", Laozi, tr. James Legge (Sacred Books of the East 39, 1891). + Gutenberg's licence (1.E.2) lets a work derived from texts not protected by US + copyright be used with no fee provided every Project Gutenberg reference is + removed from the work. The items hold only Legge's words, one numbered section + or sentence each; no Gutenberg header, licence text or trademark is stored, + and the eBook file itself is not committed. `edition_url` names the page the + text was retrieved from, as a citation of provenance and not as part of any + item's text. The legal-root question of whether a citation counts as a + "reference" is not ruled; if Gutenberg says it does, the field moves to the + Sacred Books of the East print edition. +- `Source::PublicDomain` now carries `licence`, `jurisdiction`, `translator` and + `edition_year`, and the store refuses an item missing any of them (an empty + string, or year 0). + +## The slice + +Corpus groups are chapters, so the derived split spreads the slice: + +| item | Legge | split | +|---|---|---| +| `item:daodejing:1` | ch. 3 s. 2 | Train | +| `item:daodejing:2` | ch. 7 s. 1, first sentence | Validation | +| `item:daodejing:3` | ch. 8 s. 1, first sentence | Validation | +| `item:daodejing:4` | ch. 78 s. 4 | Test | +| `item:daodejing:5` | ch. 5 s. 2, first sentence | Test | + +Each packet states neutral facts (referents, predications, a speech act) and a +stance (restraint 800 on all; affection 300 on the water likeness). The stance is +my reading of the sage's voice, not data. + +## First closeness number + +Scored with `weksa score` (chrF, and meaning as cosine under `qwen3-embedding:0.6b` +through the local Ollama), one lowered output against its own item. + +| item | split | chrF | meaning | lowered | +|---|---|---|---|---| +| 1 | Train | 44.18 | 0.831 | The sage empties the minds. The sage fills the bellies. The sage weakens the wills. The sage strengthens the bones. | +| 2 | Validation | 82.12 | 0.976 | The heaven is long-enduring. The earth continues long. | +| 3 | Validation | 74.99 | 0.974 | The highest excellence is like the water. | +| 4 | Test | 82.50 | 0.949 | The words that are strictly true seem paradoxical. | +| 5 | Test | 75.99 | 0.894 | The space between heaven and earth may be compared to the bellows. | + +Mean over the five: chrF 71.96, meaning 0.925. Mismatched baseline (each output +against the other four references, 20 pairs): chrF 14.1, meaning 0.458. So the +scorer separates the right reference from a wrong one by a wide margin; that is +all it shows here. + +**Do not read 72 as a measure of the interlingua.** The five outputs are the +ceiling of an author who saw the reference: + +- One author wrote the packets, the inventory entries and the lowerer rows with + Legge's sentence open. Frame 3 (`act`) was written from the Train item. Frames + 4 and 5 came from Validation items and frames 6 and 7 from Test items, each + written while reading that item. The Train item, the only one lowered by a + frame minted from Train, scored lowest (chrF 44.18). The scores rise with + how bespoke the frame is, the reverse of generalisation. The Validation and + Test numbers measure the lowerer table, not what the interlingua can carry. +- n = 5, one translation as reference, one embedding model, no variance. + +## What the scorer cannot see + +- **Stance.** The stance features are in the packets and the lowerer does not + render them, so the number is blind to the packet's voice entirely. It would + not change if every stance were deleted. +- **Dropped meaning.** Item 5 is a negated rhetorical question ("May not ... be + compared"); the lowering is a plain statement with no question mark and no + "not". Item 1 loses "Therefore", "in the exercise of his government" and "their". + Item 4 loses "to be". chrF scores the overlap and cannot say that the speech + act changed. The embedding cosine of 0.894 for item 5 is not much lower than + for the statement-for-statement items. +- **Table defects it forgives.** "The heaven" and "the water" are wrong English + (the lowerer prefixes "the" to every referent and has no proper-noun, + mass-noun or entity rendering). chrF takes a small penalty only. +- **Register and meaning of the Chinese.** The packets were read off Legge's + English, not the Chinese, so this is a round trip English to interlingua to + English. It measures how well the packet kept Legge's wording, not whether Legge + or the packet is right about the Dao. The term "Tao" never appears in the slice. + The multi-reading term modelling Meta described is untested. +- **Leak.** `weksa score` scores the whole output against the whole reference + with no mask, as the leak ruling says. Nothing here enforces that the + author did not use the reference; only the labels above say so. + +## What the run showed the interlingua needs + +- Negation and question force: `polarity` is refused by the lowerer ("features") + and `ask` has no punctuation row, so two of Laozi's habitual moves, the negated + claim and the rhetorical question, cannot be lowered at all. +- A frame per sentence shape is the lowerer's cost model today: each new shape + is one more table row and one more inventory frame. The Dao's aphorisms are + mostly copular and paradoxical, which suggests the next frames to mint + (attribution, contrast, negated identity) before more chapters are added. diff --git a/schemas/cultnet/weksa.corpus_item.schema.json b/schemas/cultnet/weksa.corpus_item.schema.json index 7b613be..c82c4da 100644 --- a/schemas/cultnet/weksa.corpus_item.schema.json +++ b/schemas/cultnet/weksa.corpus_item.schema.json @@ -170,6 +170,7 @@ ] }, { + "description": "A public-domain work, with everything a reader needs to check the claim:\nthe edition's licence and the jurisdiction the public-domain status is\nasserted in, who translated it (the translation is the text's copyright,\nnot the original's) and when the edition appeared. A work missing any\nis refused at admission.", "type": "object", "properties": { "PublicDomain": { @@ -181,6 +182,21 @@ "edition_url": { "type": "string" }, + "edition_year": { + "type": "integer", + "format": "uint16", + "maximum": 65535, + "minimum": 0 + }, + "jurisdiction": { + "type": "string" + }, + "licence": { + "type": "string" + }, + "translator": { + "type": "string" + }, "work": { "type": "string" } @@ -188,7 +204,11 @@ "required": [ "work", "author", - "edition_url" + "edition_url", + "licence", + "jurisdiction", + "translator", + "edition_year" ] } },