diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index e45e941..da6262d 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -71,10 +71,10 @@ A referent with an entity is named by one of that entity's forms on the entities Answer with one JSON object and nothing else: {\"text\": \"...\"}. No code fence, no commentary."; /// The instructions of a recall probe (`lowering-recall-diagnostic`): a -/// completion of a public-domain passage from its opening, asked of a model -/// that has no packet and no inventory. It measures recitation, not lowering. +/// completion of a passage from its opening, asked of a model that has no +/// packet and no inventory. It measures recitation, not lowering. pub const RECALL_INSTRUCTIONS: &str = "\ -Below is the opening of a passage from a published public-domain translation. Continue it exactly as that translation words it, from memory, until the passage ends. If you do not remember it, give your best guess in the same voice. +Below is the opening of a passage, with what is known of where it comes from. Continue it exactly as its source words it, from memory, until the passage ends. If you do not remember it, give your best guess in the same voice. Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuation only, not the opening. No code fence, no commentary."; /// The live Train items of a corpus, in key order. The only input minting @@ -547,13 +547,13 @@ impl Prompt { }) } - /// A recall probe: the work, author and translator of a public-domain item - /// and the `opening` of its text, nothing else. Only `RunStores::recall` - /// builds one. - pub(crate) fn recall(key: &str, work: &str, author: &str, translator: &str, opening: &str) -> Self { + /// A recall probe: `provenance` (header lines saying where the item is + /// published, or that it is not) and the `opening` of its text, nothing + /// else. Only `RunStores::recall` builds one. + pub(crate) fn recall(key: &str, provenance: &str, opening: &str) -> Self { Prompt { text: format!( - "{RECALL_INSTRUCTIONS}\n\nwork: {work}\nauthor: {author}\ntranslator: {translator}\n\nopening: {}", + "{RECALL_INSTRUCTIONS}\n\n{provenance}\n\nopening: {}", indented(opening) ), items: vec![key.to_string()], diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 4b65ce6..3808672 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -103,8 +103,8 @@ enum LoopCommand { #[arg(long, default_value = "http://127.0.0.1:11434")] ollama_url: String, }, - /// Probe how closely a fresh model recites the run's public-domain items - /// from their openings (chrF of its completion against the item's second + /// Probe how closely a fresh model recites the run's items from their + /// openings (chrF of its completion against the item's second /// half). Prompts and answers live in `/recall`; the command exits 3 /// until every prompt has an answer. A diagnostic, never a gate. Recall { root: PathBuf }, diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 3ae47dd..0d8f1d7 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -803,13 +803,13 @@ pub enum RoundOutcome { } /// A recall reading (`lowering-recall-diagnostic`): how closely a fresh model -/// completes each public-domain item from its opening alone, chrF against the +/// completes each live Train and Validation item from its opening alone, chrF against the /// item's own second half. Recorded nowhere: the answers stay in `/recall`. /// It is never a gate; a high reading says a round's scores on these items are /// recitation. #[derive(Clone, Debug, PartialEq)] pub struct RecallReading { - /// Public-domain items probed, Train and Validation. + /// Items probed, Train and Validation. pub n: u32, /// Items whose completion was a usable answer; a refused one counts 0. pub answered: u32, @@ -819,7 +819,7 @@ pub struct RecallReading { impl RecallReading { pub fn line(&self) -> String { format!( - "recall n={} answered={} chrf={:.2} (public-domain items, completion from the opening; a high chrf means round scores on them are recitation)", + "recall n={} answered={} chrf={:.2} (completion from the opening; a high chrf means round scores on these items are recitation; unrelated words score near 0 and an in-voice guess about 20)", self.n, self.answered, self.chrf ) } @@ -843,26 +843,27 @@ fn recall_split(text: &str) -> Option<(String, String)> { } impl RunStores { - /// Probes every live public-domain Train and Validation item of the pinned - /// corpus once, through `model`. + /// Probes every live Train and Validation item of the pinned corpus once, + /// through `model`. A public-domain item names its work, author and + /// translator; any other says `work: not published`. pub fn recall(&self, model: &mut dyn ModelPort) -> Result { let store = |_| RunError::Store; let mut probes = Vec::new(); for item in self.corpus.all().map_err(store)? { - let Source::PublicDomain { - work, - author, - translator, - .. - } = &item.source - else { - continue; + let provenance = match &item.source { + Source::PublicDomain { + work, + author, + translator, + .. + } => format!("work: {work}\nauthor: {author}\ntranslator: {translator}"), + _ => "work: not published".to_string(), }; if EncoderOffer::for_round(&item).is_err() { continue; } if let Some((opening, rest)) = recall_split(&item.text) { - let prompt = Prompt::recall(&item.id, work, author, translator, &opening); + let prompt = Prompt::recall(&item.id, &provenance, &opening); probes.push((prompt, rest)); } } diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index ae0b7fe..4d9dc1c 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -2988,7 +2988,7 @@ fn reciter(dao: &Dao, answer: fn(&str) -> String) -> Reciter { } #[test] -fn a_recall_probe_asks_for_the_second_half_of_each_live_public_domain_item_and_never_shows_it() { +fn a_recall_probe_asks_for_the_second_half_of_each_live_item_and_never_shows_it() { use weksa::run::RecallOutcome; let dao = dao(); let mut perfect = reciter(&dao, second_half); @@ -3047,13 +3047,13 @@ fn the_published_round_schema_does_not_require_a_field_older_records_lack() { } #[test] -fn a_recall_mean_is_over_every_probe_an_unreadable_answer_counts_0_and_only_live_public_domain_items_are_probed() { +fn a_recall_mean_is_over_every_probe_an_unreadable_answer_counts_0_and_a_withdrawn_item_is_not_probed() { use weksa::corpus::Source; use weksa::run::RecallOutcome; let dao = dao_over(|corpus, entities| { let entities = weksa::EntityStore::open(entities).unwrap(); let mut store = weksa::CorpusStore::open(corpus).unwrap(); - // The same words under other sources: not public domain, not probed. + // Another source: probed too, but it names no work. let mut written = store.get("item:daodejing:2").unwrap().unwrap(); written.id = "item:written:1".into(); written.text = "A line of some other source with words enough to split.".into(); @@ -3074,11 +3074,17 @@ fn a_recall_mean_is_over_every_probe_an_unreadable_answer_counts_0_and_only_live let full = both(&dao, None); assert_eq!( (full.n, full.answered), - (2, 2), - "neither the written item nor the withdrawn one is probed" + (3, 3), + "the written item is probed, the withdrawn one is not" ); - // One of two unreadable: the mean falls to a half. + // One of three unreadable: the mean falls to two thirds. let part = both(&dao, Some("Heaven is long-enduring")); - assert_eq!((part.n, part.answered), (2, 1)); - assert!((part.chrf - full.chrf / 2.0).abs() < 1e-9, "{}", part.chrf); + assert_eq!((part.n, part.answered), (3, 2)); + assert!((part.chrf - full.chrf * 2.0 / 3.0).abs() < 1e-9, "{}", part.chrf); + // A public-domain prompt names its work; any other says it is not published. + let mut m = reciter(&dao, second_half); + dao.run.recall(&mut m).unwrap(); + let named = m.seen.iter().filter(|t| t.contains("translator: James Legge")).count(); + let unnamed = m.seen.iter().filter(|t| t.contains("work: not published")).count(); + assert_eq!((named, unnamed), (2, 1)); } diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs new file mode 100644 index 0000000..4416a36 --- /dev/null +++ b/crates/weksa/tests/original_corpus.rs @@ -0,0 +1,67 @@ +//! The original corpus (`data/original`): passages written for the +//! non-memorised golf-loop target. It holds no public-domain text, so a model +//! cannot have it in pretraining, and every item says who wrote it. + +use std::path::Path; + +use weksa::corpus::{ModelFamily, Source}; +use weksa::split::{Split, SplitScheme}; +use weksa::{CorpusStore, EntityStore, InventoryStore}; + +fn stores() -> (CorpusStore, EntityStore) { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/original"); + let tmp = tempfile::tempdir().unwrap(); + for f in ["corpus.cc", "entity.cc", "inventory.cc"] { + std::fs::copy(dir.join(f), tmp.path().join(f)).unwrap(); + } + let keep = tmp.keep(); + ( + CorpusStore::open(&keep.join("corpus.cc")).unwrap(), + EntityStore::open(&keep.join("entity.cc")).unwrap(), + ) +} + +#[test] +fn every_original_item_records_its_author_and_none_is_public_domain() { + let (corpus, entities) = stores(); + let items = corpus.all().unwrap(); + assert_eq!(items.len(), 6); + for item in &items { + match &item.source { + Source::ModelWritten { + family, generator, .. + } => { + assert_eq!(*family, ModelFamily::Anthropic, "{}", item.id); + let generator = generator.as_deref().unwrap_or_default(); + assert!(generator.contains("claude-sonnet-5-5"), "{}", item.id); + } + other => panic!("{}: not model-written: {other:?}", item.id), + } + assert!(entities.get(&item.speaker).unwrap().is_some(), "{}", item.id); + } +} + +#[test] +fn the_original_items_spread_over_train_and_validation_and_none_is_in_test() { + let (corpus, _) = stores(); + let splits: Vec> = corpus + .all() + .unwrap() + .iter() + .map(|i| SplitScheme::V1.live(i)) + .collect(); + let count = |s| splits.iter().filter(|x| **x == Some(s)).count(); + assert_eq!((count(Split::Train), count(Split::Validation)), (3, 3)); + assert_eq!(count(Split::Test), 0); +} + +#[test] +fn the_original_inventory_is_the_dao_revision_three() { + let dao = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/daodejing/inventory.cc"); + let own = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/original/inventory.cc"); + assert_eq!(std::fs::read(dao).unwrap(), std::fs::read(&own).unwrap()); + let tmp = tempfile::tempdir().unwrap(); + let copy = tmp.path().join("inventory.cc"); + std::fs::copy(&own, ©).unwrap(); + assert!(InventoryStore::open(©).is_ok()); +} diff --git a/data/original/corpus.cc b/data/original/corpus.cc new file mode 100644 index 0000000..c11443f --- /dev/null +++ b/data/original/corpus.cc @@ -0,0 +1 @@ +“²cultcache.store.v1‘—±weksa.corpus_item±weksa.corpus_item´weksa.corpus_item.v1±weksa.corpus_itemÙV{"schemaName":"weksa.corpus_item","schemaVersion":"weksa.corpus_item.v1","members":[]}‘±weksa.corpus_item�–”¯item:original:1±weksa.corpus_item´2026-11-10T14:17:08ZÄ훯item:original:1Ù0A quiet mind is like still water in a deep bowl.¥en-US­entity:keeper­original-bowl��¬ModelWritten“©AnthropicÙ]claude-sonnet-5-5 (Weksa Self, original prose written for the non-memorised golf-loop target)Àª2026-11-10ÀÂ�”¯item:original:2±weksa.corpus_item´2026-11-10T14:17:08ZÅ›¯item:original:2ÙIThe river weakens the bank in spring and strengthens the roots in autumn.¥en-US­entity:keeper®original-river��¬ModelWritten“©AnthropicÙ]claude-sonnet-5-5 (Weksa Self, original prose written for the non-memorised golf-loop target)Àª2026-11-10ÀÂ�”¯item:original:3±weksa.corpus_item´2026-11-10T14:17:08ZÄô›¯item:original:3Ù7The sage waits at the door until the water has settled.¥en-US­entity:keeper­original-door��¬ModelWritten“©AnthropicÙ]claude-sonnet-5-5 (Weksa Self, original prose written for the non-memorised golf-loop target)Àª2026-11-10ÀÂ�”¯item:original:4±weksa.corpus_item´2026-11-10T14:17:08ZÄÿ›¯item:original:4ÙCThe old gardener empties the jar and fills it again with rainwater.¥en-US­entity:keeper¬original-jar��¬ModelWritten“©AnthropicÙ]claude-sonnet-5-5 (Weksa Self, original prose written for the non-memorised golf-loop target)Àª2026-11-10ÀÂ�”¯item:original:5±weksa.corpus_item´2026-11-10T14:17:08ZÅ›¯item:original:5ÙJUnder the same sky the earth keeps its patience, and the baker keeps hers.¥en-US­entity:keeper¬original-sky��¬ModelWritten“©AnthropicÙ]claude-sonnet-5-5 (Weksa Self, original prose written for the non-memorised golf-loop target)Àª2026-11-10ÀÂ�”¯item:original:6±weksa.corpus_item´2026-11-10T14:17:08ZÄ雯item:original:6Ù+A hungry belly argues louder than any will.¥en-US­entity:keeper®original-belly��¬ModelWritten“©AnthropicÙ]claude-sonnet-5-5 (Weksa Self, original prose written for the non-memorised golf-loop target)Àª2026-11-10ÀÂ� \ No newline at end of file diff --git a/data/original/entity.cc b/data/original/entity.cc new file mode 100644 index 0000000..c92e33d --- /dev/null +++ b/data/original/entity.cc @@ -0,0 +1 @@ +“²cultcache.store.v1‘—¬weksa.entity¬weksa.entity¯weksa.entity.v1¬weksa.entityÙL{"schemaName":"weksa.entity","schemaVersion":"weksa.entity.v1","members":[]}‘¬weksa.entity�‘”­entity:keeper¬weksa.entity´2026-11-10T14:17:08ZÄ:“­entity:keeper‘“¥en-US‘ªthe keeperÀ�¤Live‘®weksa-original \ No newline at end of file diff --git a/data/original/inventory.cc b/data/original/inventory.cc new file mode 100644 index 0000000..373eb3c Binary files /dev/null and b/data/original/inventory.cc differ