crates/weksa/src/context.rs

@@ -70,6 +70,13 @@ Every id is the inventory's listed below and every feature value is as listed. T
 A referent with an entity is named by one of that entity's forms on the entities line.
 Answer with one JSON object and nothing else: {\"text\": \"...\"}. No code fence, no commentary.";
 
+/// The instructions of a recall probe (`lowering-recall-diagnostic`): a
+/// completion of a public-domain passage from its opening, asked of a model
+/// that has no packet and no inventory. It measures recitation, not lowering.
+pub const RECALL_INSTRUCTIONS: &str = "\
+Below is the opening of a passage from a published public-domain translation. Continue it exactly as that translation words it, from memory, until the passage ends. If you do not remember it, give your best guess in the same voice.
+Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuation only, not the opening. No code fence, no commentary.";
+
 /// The live Train items of a corpus, in key order. The only input minting
 /// reads; it cannot be built from a slice, and no accessor returns text or an
 /// item.
@@ -540,6 +547,19 @@ impl Prompt {
         })
     }
 
+    /// A recall probe: the work, author and translator of a public-domain item
+    /// and the `opening` of its text, nothing else. Only `RunStores::recall`
+    /// builds one.
+    pub(crate) fn recall(key: &str, work: &str, author: &str, translator: &str, opening: &str) -> Self {
+        Prompt {
+            text: format!(
+                "{RECALL_INSTRUCTIONS}\n\nwork: {work}\nauthor: {author}\ntranslator: {translator}\n\nopening: {}",
+                indented(opening)
+            ),
+            items: vec![key.to_string()],
+        }
+    }
+
     /// 16 lowercase hex digits of FNV-1a 64 of the text: the name of the prompt
     /// in files and records.
     pub fn digest(&self) -> String {

crates/weksa/src/main.rs

@@ -11,7 +11,7 @@ use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn};
 use weksa::entity::{Entity, EntityOrigin, LanguageForms, Pronunciation};
 use weksa::inventory::{proof_revision, seed};
 use weksa::packet::proof_packet;
-use weksa::run::{CycleOutcome, LowererReach, RoundOutcome, RunPaths, RunStores, cycle, round};
+use weksa::run::{CycleOutcome, LowererReach, RecallOutcome, RoundOutcome, RunPaths, RunStores, cycle, round};
 use weksa::score::ScoreInput;
 use weksa::{
     AnswerFiles, CorpusStore, EntityId, EntityStore, InventoryStore, OllamaEmbedder, PacketStore,
@@ -103,6 +103,11 @@ enum LoopCommand {
         #[arg(long, default_value = "http://127.0.0.1:11434")]
         ollama_url: String,
     },
+    /// Probe how closely a fresh model recites the run's public-domain items
+    /// from their openings (chrF of its completion against the item's second
+    /// half). Prompts and answers live in `<ROOT>/recall`; the command exits 3
+    /// until every prompt has an answer. A diagnostic, never a gate.
+    Recall { root: PathBuf },
     /// Run one edit-accept cycle of the run at ROOT. The stores are read only
     /// from the run's record. The editor's prompt, then the candidate round's
     /// encode prompts, then its lowering prompts are written in turn; the
@@ -774,6 +779,20 @@ fn main() -> Result<()> {
                 RoundOutcome::Recorded(r) => println!("{}", run.round_line(&r)),
             }
         }
+        Command::Loop(LoopCommand::Recall { root }) => {
+            let run = RunStores::open(&root)?;
+            let mut files = AnswerFiles::new(run.recall_dir(), run.record().model.clone());
+            match run.recall(&mut files)? {
+                RecallOutcome::Unanswered { prompts } => {
+                    for digest in &prompts {
+                        println!("unanswered {digest}");
+                    }
+                    println!("unanswered {} prompts", prompts.len());
+                    std::process::exit(3);
+                }
+                RecallOutcome::Read(reading) => println!("{}", reading.line()),
+            }
+        }
         Command::Loop(LoopCommand::Cycle { root, ollama_url }) => {
             let unreadable = |_| anyhow::anyhow!("a store would not read");
             let mut run = RunStores::open(&root)?;

crates/weksa/src/run.rs

@@ -35,11 +35,11 @@ use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 
 use crate::context::{EditFeedback, EncoderOffer, Prompt, TrainItems};
-use crate::corpus::{CorpusItem, CorpusItemId};
+use crate::corpus::{CorpusItem, CorpusItemId, Source};
 use crate::encode::{EditDraft, LoweringDraft, PacketDraft, encoder_id};
 use crate::form::{is_attestation, is_slug};
 use crate::model::{ModelPort, PortError};
-use crate::score::{CHRF_VERSION, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate};
+use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate};
 use crate::split::{Split, SplitScheme, fnv1a64, fnv1a64_of, hex16};
 use crate::store::{
     CorpusStore, EntityStore, InventoryStore, PacketStore, StoreError, open_cache_over,
@@ -277,6 +277,7 @@ pub struct RoundRecord {
     /// The rules admission refused items under, in rule order; every Train
     /// and Validation item refused at `Stage::Admit` is counted once per rule.
     #[cultcache(key = 18)]
+    #[schemars(default)]
     admit_rules: Vec<RuleCount>,
 }
 
@@ -801,6 +802,105 @@ pub enum RoundOutcome {
     Recorded(RoundRecord),
 }
 
+/// A recall reading (`lowering-recall-diagnostic`): how closely a fresh model
+/// completes each public-domain item from its opening alone, chrF against the
+/// item's own second half. Recorded nowhere: the answers stay in `<root>/recall`.
+/// It is never a gate; a high reading says a round's scores on these items are
+/// recitation.
+#[derive(Clone, Debug, PartialEq)]
+pub struct RecallReading {
+    /// Public-domain items probed, Train and Validation.
+    pub n: u32,
+    /// Items whose completion was a usable answer; a refused one counts 0.
+    pub answered: u32,
+    pub chrf: f64,
+}
+
+impl RecallReading {
+    pub fn line(&self) -> String {
+        format!(
+            "recall n={} answered={} chrf={:.2} (public-domain items, completion from the opening; a high chrf means round scores on them are recitation)",
+            self.n, self.answered, self.chrf
+        )
+    }
+}
+
+#[derive(Debug)]
+pub enum RecallOutcome {
+    Unanswered { prompts: Vec<String> },
+    Read(RecallReading),
+}
+
+/// The opening and the rest of `text`, split between words at about the
+/// middle; `None` when it has fewer than four words.
+fn recall_split(text: &str) -> Option<(String, String)> {
+    let words: Vec<&str> = text.split_whitespace().collect();
+    if words.len() < 4 {
+        return None;
+    }
+    let (head, tail) = words.split_at(words.len() / 2);
+    Some((head.join(" "), tail.join(" ")))
+}
+
+impl RunStores {
+    /// Probes every live public-domain Train and Validation item of the pinned
+    /// corpus once, through `model`.
+    pub fn recall(&self, model: &mut dyn ModelPort) -> Result<RecallOutcome, RunError> {
+        let store = |_| RunError::Store;
+        let mut probes = Vec::new();
+        for item in self.corpus.all().map_err(store)? {
+            let Source::PublicDomain {
+                work,
+                author,
+                translator,
+                ..
+            } = &item.source
+            else {
+                continue;
+            };
+            if EncoderOffer::for_round(&item).is_err() {
+                continue;
+            }
+            if let Some((opening, rest)) = recall_split(&item.text) {
+                let prompt = Prompt::recall(&item.id, work, author, translator, &opening);
+                probes.push((prompt, rest));
+            }
+        }
+        let mut unanswered = Vec::new();
+        let mut scores = Vec::new();
+        for (prompt, rest) in &probes {
+            match model.answer(prompt) {
+                Ok(answer) => {
+                    let text = LoweringDraft::parse(&answer)
+                        .ok()
+                        .filter(LoweringDraft::usable)
+                        .map(|d| d.text);
+                    scores.push(text.map(|t| chrf(&t, rest)));
+                }
+                Err(PortError::Unanswered) => unanswered.push(prompt.digest()),
+                Err(PortError::Io(ErrorKind::InvalidData)) => scores.push(None),
+                Err(PortError::Io(kind)) => return Err(RunError::Io(kind)),
+            }
+        }
+        if !unanswered.is_empty() {
+            return Ok(RecallOutcome::Unanswered { prompts: unanswered });
+        }
+        let n = scores.len() as u32;
+        let answered = scores.iter().flatten().count() as u32;
+        let chrf = if n == 0 {
+            0.0
+        } else {
+            scores.iter().map(|s| s.unwrap_or(0.0)).sum::<f64>() / f64::from(n)
+        };
+        Ok(RecallOutcome::Read(RecallReading { n, answered, chrf }))
+    }
+
+    /// `<root>/recall`, where a recall's prompts and answers live.
+    pub fn recall_dir(&self) -> PathBuf {
+        self.root.join("recall")
+    }
+}
+
 /// The distinct rule codes an admission refusal broke: validator rules by
 /// code, a store refusal by its variant name. Nothing from the packet or the
 /// stores is kept; an I/O failure is `io`.

crates/weksa/tests/golf_loop.rs

@@ -2946,3 +2946,139 @@ fn admit_refusals_are_counted_by_rule_and_printed_with_the_label() {
     assert!(r.admit_rules().is_empty());
     assert!(!dao.run.round_line(&r).contains("admit-rules"));
 }
+
+/// Answers a recall probe from the corpus: the item's own second half, a
+/// fixed wrong text, or junk.
+struct Reciter {
+    corpus: weksa::store::CorpusStore,
+    answer: fn(&str) -> String,
+    /// An item whose text holds this gets an answer file that is not UTF-8.
+    unreadable: Option<&'static str>,
+    seen: Vec<String>,
+}
+
+impl ModelPort for Reciter {
+    fn model(&self) -> &str {
+        "reciter"
+    }
+    fn answer(&mut self, prompt: &Prompt) -> Result<String, PortError> {
+        self.seen.push(prompt.text().to_string());
+        let key = &prompt.item_keys()[0];
+        let item = self.corpus.get(key).unwrap().unwrap();
+        if self.unreadable.is_some_and(|t| item.text.contains(t)) {
+            return Err(PortError::Io(std::io::ErrorKind::InvalidData));
+        }
+        Ok((self.answer)(&item.text))
+    }
+}
+
+fn second_half(text: &str) -> String {
+    let words: Vec<&str> = text.split_whitespace().collect();
+    let draft = serde_json::json!({ "text": words[words.len() / 2..].join(" ") });
+    draft.to_string()
+}
+
+fn reciter(dao: &Dao, answer: fn(&str) -> String) -> Reciter {
+    Reciter {
+        corpus: weksa::store::CorpusStore::open(&dao.corpus).unwrap(),
+        answer,
+        unreadable: None,
+        seen: vec![],
+    }
+}
+
+#[test]
+fn a_recall_probe_asks_for_the_second_half_of_each_live_public_domain_item_and_never_shows_it() {
+    use weksa::run::RecallOutcome;
+    let dao = dao();
+    let mut perfect = reciter(&dao, second_half);
+    let RecallOutcome::Read(r) = dao.run.recall(&mut perfect).unwrap() else {
+        panic!("every prompt was answered");
+    };
+    // Items 1 (Train) and 2, 3 (Validation) are live; 4 and 5 are not offered.
+    assert_eq!((r.n, r.answered), (3, 3));
+    assert!((r.chrf - 100.0).abs() < 1e-9, "{}", r.chrf);
+    assert_eq!(perfect.seen.len(), 3);
+    for (text, key) in perfect.seen.iter().zip(["item:daodejing:1", "item:daodejing:2", "item:daodejing:3"]) {
+        let item = perfect.corpus.get(key).unwrap().unwrap();
+        let rest = item.text.split_whitespace().collect::<Vec<_>>();
+        let tail = rest[rest.len() / 2..].join(" ");
+        let head = rest[..rest.len() / 2].join(" ");
+        assert!(text.contains(&head), "{key}: the opening is shown");
+        assert!(!text.contains(&tail), "{key}: the held half is not shown");
+    }
+}
+
+#[test]
+fn a_recall_of_the_wrong_words_or_no_answer_reads_low_and_a_missing_answer_waits() {
+    use weksa::run::RecallOutcome;
+    let dao = dao();
+    let mut wrong = reciter(&dao, |_| r#"{"text":"zzz qqq xxx"}"#.to_string());
+    let RecallOutcome::Read(r) = dao.run.recall(&mut wrong).unwrap() else {
+        panic!("answered");
+    };
+    assert_eq!((r.n, r.answered), (3, 3));
+    assert!(r.chrf < 10.0, "{}", r.chrf);
+    let mut junk = reciter(&dao, |_| "not json".to_string());
+    let RecallOutcome::Read(r) = dao.run.recall(&mut junk).unwrap() else {
+        panic!("answered");
+    };
+    assert_eq!((r.n, r.answered, r.chrf), (3, 0, 0.0));
+    // With no answers on disk the files port waits on all three prompts.
+    let mut files = AnswerFiles::new(dao.run.recall_dir(), "m".into());
+    let RecallOutcome::Unanswered { prompts } = dao.run.recall(&mut files).unwrap() else {
+        panic!("nobody answered");
+    };
+    assert_eq!(prompts.len(), 3);
+    assert!(dao.run.recall_dir().join(format!("{}.prompt", prompts[0])).exists());
+}
+
+#[test]
+fn the_published_round_schema_does_not_require_a_field_older_records_lack() {
+    let (_, json) = weksa::schema::published()
+        .into_iter()
+        .find(|(name, _)| name.contains("loop_round"))
+        .expect("the round schema is published");
+    let schema: serde_json::Value = serde_json::from_str(&json).unwrap();
+    let required = schema["required"].as_array().unwrap();
+    assert!(required.iter().any(|r| r == "inventory_digest"));
+    assert!(!required.iter().any(|r| r == "admit_rules"), "{required:?}");
+    assert!(schema["properties"]["admit_rules"].is_object());
+}
+
+#[test]
+fn a_recall_mean_is_over_every_probe_an_unreadable_answer_counts_0_and_only_live_public_domain_items_are_probed() {
+    use weksa::corpus::Source;
+    use weksa::run::RecallOutcome;
+    let dao = dao_over(|corpus, entities| {
+        let entities = weksa::EntityStore::open(entities).unwrap();
+        let mut store = weksa::CorpusStore::open(corpus).unwrap();
+        // The same words under other sources: not public domain, not probed.
+        let mut written = store.get("item:daodejing:2").unwrap().unwrap();
+        written.id = "item:written:1".into();
+        written.text = "A line of some other source with words enough to split.".into();
+        written.names = vec![];
+        written.prior_turns = vec![];
+        written.source = Source::OperatorWritten;
+        store.add(written, &entities).unwrap();
+        store.withdraw("item:daodejing:3").unwrap();
+    });
+    let both = |dao: &Dao, unreadable: Option<&'static str>| {
+        let mut m = reciter(dao, second_half);
+        m.unreadable = unreadable;
+        let RecallOutcome::Read(r) = dao.run.recall(&mut m).unwrap() else {
+            panic!("answered");
+        };
+        r
+    };
+    let full = both(&dao, None);
+    assert_eq!(
+        (full.n, full.answered),
+        (2, 2),
+        "neither the written item nor the withdrawn one is probed"
+    );
+    // One of two unreadable: the mean falls to a half.
+    let part = both(&dao, Some("Heaven is long-enduring"));
+    assert_eq!((part.n, part.answered), (2, 1));
+    assert!((part.chrf - full.chrf / 2.0).abs() < 1e-9, "{}", part.chrf);
+}

schemas/cultnet/weksa.loop_round.schema.json

@@ -7,6 +7,7 @@
     "admit_rules": {
       "description": "The rules admission refused items under, in rule order; every Train\nand Validation item refused at `Stage::Admit` is counted once per rule.",
       "type": "array",
+      "default": [],
       "items": {
         "$ref": "#/$defs/RuleCount"
       }
@@ -103,8 +104,7 @@
     "validation_refused",
     "train_scored",
     "validation_scored",
-    "inventory_digest",
-    "admit_rules"
+    "inventory_digest"
   ],
   "$defs": {
     "Call": {