From 3d3ca915ea805aa7fc036de56b3f33e6d721d9ad Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 14:21:31 +0000 Subject: [PATCH] Encode prompt states the id form; recall line states the real chrF floor; probe test over data/original Five of six encoders in the first original-corpus round wrote bare integer ids and were refused at parse (encoder-prompt-concept-ids). The prompt now says ids are written in full with their family prefix. Soul s1 on the original corpus: unrelated English scores chrF 13 to 18, not near 0, so the recall line says so; a test runs the probe over the real original items. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/context.rs | 2 +- crates/weksa/src/run.rs | 2 +- crates/weksa/tests/context.rs | 18 ++++++++++ crates/weksa/tests/original_corpus.rs | 47 +++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 2 deletions(-) diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index da6262d..3c73b0c 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -59,7 +59,7 @@ Fields: - predications: a list of {id, concept, frame, roles, features}; roles is a list of {role, filler}, filler a node id. - relations: a list of {relation, from, to, features}. - stance: {features}; the speaker's felt affect and restraint, never the affect the line displays. -Node ids are n1, n2, and so on, one per referent and predication. Every other id comes from the inventory listed below and must be live at its revision. An entity comes only from the entities line. +Node ids are n1, n2, and so on, one per referent and predication. Every other id comes from the inventory listed below and must be live at its revision. Write each id in full, exactly as the inventory lines spell it, family prefix included: concept:7, never 7. An entity comes only from the entities line. A feature is {feature, value}; value is {\"Grade\": 0 to 1000}, {\"Option\": label} or {\"Flag\": true or false}, as the inventory lists the feature's kind."; /// The instructions a lowering prompt starts with: the lowerer's whole brief. diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 0d8f1d7..77c3642 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -819,7 +819,7 @@ pub struct RecallReading { impl RecallReading { pub fn line(&self) -> String { format!( - "recall n={} answered={} chrf={:.2} (completion from the opening; a high chrf means round scores on these items are recitation; unrelated words score near 0 and an in-voice guess about 20)", + "recall n={} answered={} chrf={:.2} (completion from the opening; unrelated English already scores chrf 13 to 18, so a reading near that floor is no recitation and one well above it is)", self.n, self.answered, self.chrf ) } diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs index fe351d7..04ebe35 100644 --- a/crates/weksa/tests/context.rs +++ b/crates/weksa/tests/context.rs @@ -686,3 +686,21 @@ mod lowering { } } } + +/// First real rounds: five of six encoders wrote bare integers (`"concept": 7`) +/// where the inventory lists `concept:7`, and the draft reader refused them. +/// The prompt says the form outright, and the inventory lines it points to +/// are spelled in that form. +#[test] +fn the_encode_prompt_states_the_id_form_the_inventory_lines_use() { + assert!(ENCODE.contains("family prefix included: concept:7, never 7")); + let (text, _) = encode_with_prior("Hello there.", "Hi."); + let listed = text + .lines() + .skip_while(|l| !l.starts_with("inventory rev:")) + .skip(1); + for line in listed { + let id = line.split(": ").next().unwrap(); + assert!(id.contains(':'), "an inventory line spells its id in full: {line}"); + } +} diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 4416a36..51e4c35 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -65,3 +65,50 @@ fn the_original_inventory_is_the_dao_revision_three() { std::fs::copy(&own, ©).unwrap(); assert!(InventoryStore::open(©).is_ok()); } + +/// The probe over the real original corpus: every item is offered, none names a +/// work, and a model that answers with unrelated words reads at the noise floor. +#[test] +fn the_recall_probe_covers_all_six_original_items_and_names_no_work() { + use weksa::model::{ModelPort, PortError}; + use weksa::run::{RecallOutcome, RunPaths, RunStores}; + struct Unrelated(Vec); + impl ModelPort for Unrelated { + fn model(&self) -> &str { + "unrelated" + } + fn answer(&mut self, p: &weksa::context::Prompt) -> Result { + self.0.push(p.text().to_string()); + Ok(r#"{"text":"stone harbours remember distant lanterns"}"#.to_string()) + } + } + let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/original"); + let tmp = tempfile::tempdir().unwrap(); + let path = |f: &str| { + std::fs::copy(src.join(f), tmp.path().join(f)).unwrap(); + tmp.path().join(f) + }; + let (c, i, e) = (path("corpus.cc"), path("inventory.cc"), path("entity.cc")); + let run = RunStores::create( + &tmp.path().join("run"), + "orig", + RunPaths { + corpus: &c, + inventory: &i, + entities: &e, + }, + "m", + "bag", + weksa::run::LowererReach::BriefedSubagent, + ) + .unwrap(); + let mut model = Unrelated(vec![]); + let RecallOutcome::Read(r) = run.recall(&mut model).unwrap() else { + panic!("answered"); + }; + assert_eq!((r.n, r.answered), (6, 6)); + assert!(r.chrf < 25.0, "{}", r.chrf); + assert_eq!(model.0.len(), 6); + assert!(model.0.iter().all(|t| t.contains("work: not published"))); + assert!(model.0.iter().all(|t| !t.contains("translator:"))); +}