diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index dc1e19f..e45e941 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -70,6 +70,13 @@ Every id is the inventory's listed below and every feature value is as listed. T A referent with an entity is named by one of that entity's forms on the entities line. Answer with one JSON object and nothing else: {\"text\": \"...\"}. No code fence, no commentary."; +/// The instructions of a recall probe (`lowering-recall-diagnostic`): a +/// completion of a public-domain passage from its opening, asked of a model +/// that has no packet and no inventory. It measures recitation, not lowering. +pub const RECALL_INSTRUCTIONS: &str = "\ +Below is the opening of a passage from a published public-domain translation. Continue it exactly as that translation words it, from memory, until the passage ends. If you do not remember it, give your best guess in the same voice. +Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuation only, not the opening. No code fence, no commentary."; + /// The live Train items of a corpus, in key order. The only input minting /// reads; it cannot be built from a slice, and no accessor returns text or an /// item. @@ -540,6 +547,19 @@ impl Prompt { }) } + /// A recall probe: the work, author and translator of a public-domain item + /// and the `opening` of its text, nothing else. Only `RunStores::recall` + /// builds one. + pub(crate) fn recall(key: &str, work: &str, author: &str, translator: &str, opening: &str) -> Self { + Prompt { + text: format!( + "{RECALL_INSTRUCTIONS}\n\nwork: {work}\nauthor: {author}\ntranslator: {translator}\n\nopening: {}", + indented(opening) + ), + items: vec![key.to_string()], + } + } + /// 16 lowercase hex digits of FNV-1a 64 of the text: the name of the prompt /// in files and records. pub fn digest(&self) -> String { diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 06b3cce..4b65ce6 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -11,7 +11,7 @@ use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn}; use weksa::entity::{Entity, EntityOrigin, LanguageForms, Pronunciation}; use weksa::inventory::{proof_revision, seed}; use weksa::packet::proof_packet; -use weksa::run::{CycleOutcome, LowererReach, RoundOutcome, RunPaths, RunStores, cycle, round}; +use weksa::run::{CycleOutcome, LowererReach, RecallOutcome, RoundOutcome, RunPaths, RunStores, cycle, round}; use weksa::score::ScoreInput; use weksa::{ AnswerFiles, CorpusStore, EntityId, EntityStore, InventoryStore, OllamaEmbedder, PacketStore, @@ -103,6 +103,11 @@ enum LoopCommand { #[arg(long, default_value = "http://127.0.0.1:11434")] ollama_url: String, }, + /// Probe how closely a fresh model recites the run's public-domain items + /// from their openings (chrF of its completion against the item's second + /// half). Prompts and answers live in `/recall`; the command exits 3 + /// until every prompt has an answer. A diagnostic, never a gate. + Recall { root: PathBuf }, /// Run one edit-accept cycle of the run at ROOT. The stores are read only /// from the run's record. The editor's prompt, then the candidate round's /// encode prompts, then its lowering prompts are written in turn; the @@ -774,6 +779,20 @@ fn main() -> Result<()> { RoundOutcome::Recorded(r) => println!("{}", run.round_line(&r)), } } + Command::Loop(LoopCommand::Recall { root }) => { + let run = RunStores::open(&root)?; + let mut files = AnswerFiles::new(run.recall_dir(), run.record().model.clone()); + match run.recall(&mut files)? { + RecallOutcome::Unanswered { prompts } => { + for digest in &prompts { + println!("unanswered {digest}"); + } + println!("unanswered {} prompts", prompts.len()); + std::process::exit(3); + } + RecallOutcome::Read(reading) => println!("{}", reading.line()), + } + } Command::Loop(LoopCommand::Cycle { root, ollama_url }) => { let unreadable = |_| anyhow::anyhow!("a store would not read"); let mut run = RunStores::open(&root)?; diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 299ea72..3ae47dd 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -35,11 +35,11 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use crate::context::{EditFeedback, EncoderOffer, Prompt, TrainItems}; -use crate::corpus::{CorpusItem, CorpusItemId}; +use crate::corpus::{CorpusItem, CorpusItemId, Source}; use crate::encode::{EditDraft, LoweringDraft, PacketDraft, encoder_id}; use crate::form::{is_attestation, is_slug}; use crate::model::{ModelPort, PortError}; -use crate::score::{CHRF_VERSION, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate}; +use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate}; use crate::split::{Split, SplitScheme, fnv1a64, fnv1a64_of, hex16}; use crate::store::{ CorpusStore, EntityStore, InventoryStore, PacketStore, StoreError, open_cache_over, @@ -277,6 +277,7 @@ pub struct RoundRecord { /// The rules admission refused items under, in rule order; every Train /// and Validation item refused at `Stage::Admit` is counted once per rule. #[cultcache(key = 18)] + #[schemars(default)] admit_rules: Vec, } @@ -801,6 +802,105 @@ pub enum RoundOutcome { Recorded(RoundRecord), } +/// A recall reading (`lowering-recall-diagnostic`): how closely a fresh model +/// completes each public-domain item from its opening alone, chrF against the +/// item's own second half. Recorded nowhere: the answers stay in `/recall`. +/// It is never a gate; a high reading says a round's scores on these items are +/// recitation. +#[derive(Clone, Debug, PartialEq)] +pub struct RecallReading { + /// Public-domain items probed, Train and Validation. + pub n: u32, + /// Items whose completion was a usable answer; a refused one counts 0. + pub answered: u32, + pub chrf: f64, +} + +impl RecallReading { + pub fn line(&self) -> String { + format!( + "recall n={} answered={} chrf={:.2} (public-domain items, completion from the opening; a high chrf means round scores on them are recitation)", + self.n, self.answered, self.chrf + ) + } +} + +#[derive(Debug)] +pub enum RecallOutcome { + Unanswered { prompts: Vec }, + Read(RecallReading), +} + +/// The opening and the rest of `text`, split between words at about the +/// middle; `None` when it has fewer than four words. +fn recall_split(text: &str) -> Option<(String, String)> { + let words: Vec<&str> = text.split_whitespace().collect(); + if words.len() < 4 { + return None; + } + let (head, tail) = words.split_at(words.len() / 2); + Some((head.join(" "), tail.join(" "))) +} + +impl RunStores { + /// Probes every live public-domain Train and Validation item of the pinned + /// corpus once, through `model`. + pub fn recall(&self, model: &mut dyn ModelPort) -> Result { + let store = |_| RunError::Store; + let mut probes = Vec::new(); + for item in self.corpus.all().map_err(store)? { + let Source::PublicDomain { + work, + author, + translator, + .. + } = &item.source + else { + continue; + }; + if EncoderOffer::for_round(&item).is_err() { + continue; + } + if let Some((opening, rest)) = recall_split(&item.text) { + let prompt = Prompt::recall(&item.id, work, author, translator, &opening); + probes.push((prompt, rest)); + } + } + let mut unanswered = Vec::new(); + let mut scores = Vec::new(); + for (prompt, rest) in &probes { + match model.answer(prompt) { + Ok(answer) => { + let text = LoweringDraft::parse(&answer) + .ok() + .filter(LoweringDraft::usable) + .map(|d| d.text); + scores.push(text.map(|t| chrf(&t, rest))); + } + Err(PortError::Unanswered) => unanswered.push(prompt.digest()), + Err(PortError::Io(ErrorKind::InvalidData)) => scores.push(None), + Err(PortError::Io(kind)) => return Err(RunError::Io(kind)), + } + } + if !unanswered.is_empty() { + return Ok(RecallOutcome::Unanswered { prompts: unanswered }); + } + let n = scores.len() as u32; + let answered = scores.iter().flatten().count() as u32; + let chrf = if n == 0 { + 0.0 + } else { + scores.iter().map(|s| s.unwrap_or(0.0)).sum::() / f64::from(n) + }; + Ok(RecallOutcome::Read(RecallReading { n, answered, chrf })) + } + + /// `/recall`, where a recall's prompts and answers live. + pub fn recall_dir(&self) -> PathBuf { + self.root.join("recall") + } +} + /// The distinct rule codes an admission refusal broke: validator rules by /// code, a store refusal by its variant name. Nothing from the packet or the /// stores is kept; an I/O failure is `io`. diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 6269f99..ae0b7fe 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -2946,3 +2946,139 @@ fn admit_refusals_are_counted_by_rule_and_printed_with_the_label() { assert!(r.admit_rules().is_empty()); assert!(!dao.run.round_line(&r).contains("admit-rules")); } + +/// Answers a recall probe from the corpus: the item's own second half, a +/// fixed wrong text, or junk. +struct Reciter { + corpus: weksa::store::CorpusStore, + answer: fn(&str) -> String, + /// An item whose text holds this gets an answer file that is not UTF-8. + unreadable: Option<&'static str>, + seen: Vec, +} + +impl ModelPort for Reciter { + fn model(&self) -> &str { + "reciter" + } + fn answer(&mut self, prompt: &Prompt) -> Result { + self.seen.push(prompt.text().to_string()); + let key = &prompt.item_keys()[0]; + let item = self.corpus.get(key).unwrap().unwrap(); + if self.unreadable.is_some_and(|t| item.text.contains(t)) { + return Err(PortError::Io(std::io::ErrorKind::InvalidData)); + } + Ok((self.answer)(&item.text)) + } +} + +fn second_half(text: &str) -> String { + let words: Vec<&str> = text.split_whitespace().collect(); + let draft = serde_json::json!({ "text": words[words.len() / 2..].join(" ") }); + draft.to_string() +} + +fn reciter(dao: &Dao, answer: fn(&str) -> String) -> Reciter { + Reciter { + corpus: weksa::store::CorpusStore::open(&dao.corpus).unwrap(), + answer, + unreadable: None, + seen: vec![], + } +} + +#[test] +fn a_recall_probe_asks_for_the_second_half_of_each_live_public_domain_item_and_never_shows_it() { + use weksa::run::RecallOutcome; + let dao = dao(); + let mut perfect = reciter(&dao, second_half); + let RecallOutcome::Read(r) = dao.run.recall(&mut perfect).unwrap() else { + panic!("every prompt was answered"); + }; + // Items 1 (Train) and 2, 3 (Validation) are live; 4 and 5 are not offered. + assert_eq!((r.n, r.answered), (3, 3)); + assert!((r.chrf - 100.0).abs() < 1e-9, "{}", r.chrf); + assert_eq!(perfect.seen.len(), 3); + for (text, key) in perfect.seen.iter().zip(["item:daodejing:1", "item:daodejing:2", "item:daodejing:3"]) { + let item = perfect.corpus.get(key).unwrap().unwrap(); + let rest = item.text.split_whitespace().collect::>(); + let tail = rest[rest.len() / 2..].join(" "); + let head = rest[..rest.len() / 2].join(" "); + assert!(text.contains(&head), "{key}: the opening is shown"); + assert!(!text.contains(&tail), "{key}: the held half is not shown"); + } +} + +#[test] +fn a_recall_of_the_wrong_words_or_no_answer_reads_low_and_a_missing_answer_waits() { + use weksa::run::RecallOutcome; + let dao = dao(); + let mut wrong = reciter(&dao, |_| r#"{"text":"zzz qqq xxx"}"#.to_string()); + let RecallOutcome::Read(r) = dao.run.recall(&mut wrong).unwrap() else { + panic!("answered"); + }; + assert_eq!((r.n, r.answered), (3, 3)); + assert!(r.chrf < 10.0, "{}", r.chrf); + let mut junk = reciter(&dao, |_| "not json".to_string()); + let RecallOutcome::Read(r) = dao.run.recall(&mut junk).unwrap() else { + panic!("answered"); + }; + assert_eq!((r.n, r.answered, r.chrf), (3, 0, 0.0)); + // With no answers on disk the files port waits on all three prompts. + let mut files = AnswerFiles::new(dao.run.recall_dir(), "m".into()); + let RecallOutcome::Unanswered { prompts } = dao.run.recall(&mut files).unwrap() else { + panic!("nobody answered"); + }; + assert_eq!(prompts.len(), 3); + assert!(dao.run.recall_dir().join(format!("{}.prompt", prompts[0])).exists()); +} + +#[test] +fn the_published_round_schema_does_not_require_a_field_older_records_lack() { + let (_, json) = weksa::schema::published() + .into_iter() + .find(|(name, _)| name.contains("loop_round")) + .expect("the round schema is published"); + let schema: serde_json::Value = serde_json::from_str(&json).unwrap(); + let required = schema["required"].as_array().unwrap(); + assert!(required.iter().any(|r| r == "inventory_digest")); + assert!(!required.iter().any(|r| r == "admit_rules"), "{required:?}"); + assert!(schema["properties"]["admit_rules"].is_object()); +} + +#[test] +fn a_recall_mean_is_over_every_probe_an_unreadable_answer_counts_0_and_only_live_public_domain_items_are_probed() { + use weksa::corpus::Source; + use weksa::run::RecallOutcome; + let dao = dao_over(|corpus, entities| { + let entities = weksa::EntityStore::open(entities).unwrap(); + let mut store = weksa::CorpusStore::open(corpus).unwrap(); + // The same words under other sources: not public domain, not probed. + let mut written = store.get("item:daodejing:2").unwrap().unwrap(); + written.id = "item:written:1".into(); + written.text = "A line of some other source with words enough to split.".into(); + written.names = vec![]; + written.prior_turns = vec![]; + written.source = Source::OperatorWritten; + store.add(written, &entities).unwrap(); + store.withdraw("item:daodejing:3").unwrap(); + }); + let both = |dao: &Dao, unreadable: Option<&'static str>| { + let mut m = reciter(dao, second_half); + m.unreadable = unreadable; + let RecallOutcome::Read(r) = dao.run.recall(&mut m).unwrap() else { + panic!("answered"); + }; + r + }; + let full = both(&dao, None); + assert_eq!( + (full.n, full.answered), + (2, 2), + "neither the written item nor the withdrawn one is probed" + ); + // One of two unreadable: the mean falls to a half. + let part = both(&dao, Some("Heaven is long-enduring")); + assert_eq!((part.n, part.answered), (2, 1)); + assert!((part.chrf - full.chrf / 2.0).abs() < 1e-9, "{}", part.chrf); +} diff --git a/schemas/cultnet/weksa.loop_round.schema.json b/schemas/cultnet/weksa.loop_round.schema.json index dd940ad..f218bdb 100644 --- a/schemas/cultnet/weksa.loop_round.schema.json +++ b/schemas/cultnet/weksa.loop_round.schema.json @@ -7,6 +7,7 @@ "admit_rules": { "description": "The rules admission refused items under, in rule order; every Train\nand Validation item refused at `Stage::Admit` is counted once per rule.", "type": "array", + "default": [], "items": { "$ref": "#/$defs/RuleCount" } @@ -103,8 +104,7 @@ "validation_refused", "train_scored", "validation_scored", - "inventory_digest", - "admit_rules" + "inventory_digest" ], "$defs": { "Call": {