From e0e3c93d979ab94f51429f779b8654dd564fdeb0 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 18:26:31 +0000 Subject: [PATCH 1/4] Stance ablation: replay, drop and permute a packet's stance beside recall RecordedAnswers is the one reader of .answer files (AnswerFiles reads through it). RunStores::stance_ablation asks a fresh lowerer the round's own lowering prompt again (replay), with the stance emptied (drop) and with the next item's stance in its split (permute), for live Train and Validation packets only, and reads how far the lowering moves and what the item's own stance adds to fidelity. A diagnostic in /ablation-: records nothing, never writes round-. `weksa loop ablate --round n`; exits 3 until answered. Touches no pinned prompt-source file. The source tripwire now allows the ablation as a second Prompt::lower call site. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/main.rs | 50 +++- crates/weksa/src/model.rs | 40 ++- crates/weksa/src/run.rs | 290 +++++++++++++++++++- crates/weksa/tests/cli.rs | 71 +++++ crates/weksa/tests/golf_loop.rs | 452 +++++++++++++++++++++++++++++++- 5 files changed, 892 insertions(+), 11 deletions(-) diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 5b88c14..a92c010 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -11,11 +11,14 @@ use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn}; use weksa::entity::{Entity, EntityOrigin, LanguageForms, Pronunciation}; use weksa::inventory::{proof_revision, seed}; use weksa::packet::proof_packet; -use weksa::run::{CycleOutcome, LowererReach, RecallOutcome, RoundOutcome, RunPaths, RunStores, cycle, round}; +use weksa::model::RecordedAnswers; +use weksa::run::{ + AblationOutcome, CycleOutcome, LowererReach, RecallOutcome, RoundOutcome, RunPaths, RunStores, cycle, round, +}; use weksa::score::ScoreInput; use weksa::{ AnswerFiles, CorpusStore, EntityId, EntityStore, InventoryStore, OllamaEmbedder, PacketStore, - Scorer, SplitScheme, StoreError, lower, pronounce, validate, + Scorer, Split, SplitScheme, StoreError, lower, pronounce, validate, }; #[derive(Parser)] @@ -112,6 +115,19 @@ enum LoopCommand { /// half). Prompts and answers live in `/recall`; the command exits 3 /// until every prompt has an answer. A diagnostic, never a gate. Recall { root: PathBuf }, + /// Ask whether the lowerer uses a packet's stance: for each live Train and + /// Validation packet of round ROUND, ask again with the stance as it is, + /// emptied, and swapped for the next item's, and read how far the lowering + /// moves from the round's own answer. Prompts and answers live in + /// `/ablation-`; the command exits 3 until every prompt has an + /// answer. A diagnostic, never a gate; Test is never touched. + Ablate { + root: PathBuf, + #[arg(long)] + round: u32, + #[arg(long, default_value = "http://127.0.0.1:11434")] + ollama_url: String, + }, /// Run one edit-accept cycle of the run at ROOT. The stores are read only /// from the run's record. The editor's prompt, then the candidate round's /// encode prompts, then its lowering prompts are written in turn; the @@ -785,6 +801,36 @@ fn main() -> Result<()> { RoundOutcome::Recorded(r) => println!("{}", run.round_line(&r)), } } + Command::Loop(LoopCommand::Ablate { + root, + round, + ollama_url, + }) => { + let run = RunStores::open(&root)?; + let model = run.record().model.clone(); + let mut recorded = + RecordedAnswers::new(root.join(format!("round-{round}")), model.clone()); + let mut files = AnswerFiles::new(run.ablation_dir(round), model); + let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder)); + match run.stance_ablation(round, &mut recorded, &mut files, &scorer)? { + AblationOutcome::Unanswered { prompts } => { + for digest in &prompts { + println!("unanswered {digest}"); + } + println!("unanswered {} prompts", prompts.len()); + std::process::exit(3); + } + AblationOutcome::Read(readings) => { + for (split, reading) in readings { + let name = match split { + Split::Train => "train", + _ => "validation", + }; + println!("{name} {}", reading.line()); + } + } + } + } Command::Loop(LoopCommand::Recall { root }) => { let run = RunStores::open(&root)?; let mut files = AnswerFiles::new(run.recall_dir(), run.record().model.clone()); diff --git a/crates/weksa/src/model.rs b/crates/weksa/src/model.rs index 2f83176..1a79449 100644 --- a/crates/weksa/src/model.rs +++ b/crates/weksa/src/model.rs @@ -48,6 +48,37 @@ impl AnswerFiles { } } +/// Answers read from one directory and nothing else: `.answer` is the +/// reply to the prompt whose digest is ``. A prompt with no file is +/// [`PortError::Unanswered`]; this port never creates or writes anything. The +/// one reader of answer files: [`AnswerFiles`] reads through it. +pub struct RecordedAnswers { + dir: PathBuf, + model: String, +} + +impl RecordedAnswers { + pub fn new(dir: PathBuf, model: String) -> Self { + Self { dir, model } + } +} + +impl ModelPort for RecordedAnswers { + fn model(&self) -> &str { + &self.model + } + + fn answer(&mut self, prompt: &Prompt) -> Result { + match std::fs::read(self.dir.join(format!("{}.answer", prompt.digest()))) { + Ok(bytes) => { + String::from_utf8(bytes).map_err(|_| PortError::Io(ErrorKind::InvalidData)) + } + Err(e) if e.kind() == ErrorKind::NotFound => Err(PortError::Unanswered), + Err(e) => Err(PortError::Io(e.kind())), + } + } +} + impl ModelPort for AnswerFiles { fn model(&self) -> &str { &self.model @@ -56,12 +87,9 @@ impl ModelPort for AnswerFiles { fn answer(&mut self, prompt: &Prompt) -> Result { let io = |e: std::io::Error| PortError::Io(e.kind()); let digest = prompt.digest(); - match std::fs::read(self.dir.join(format!("{digest}.answer"))) { - Ok(bytes) => { - return String::from_utf8(bytes).map_err(|_| PortError::Io(ErrorKind::InvalidData)); - } - Err(e) if e.kind() == ErrorKind::NotFound => {} - Err(e) => return Err(io(e)), + match RecordedAnswers::new(self.dir.clone(), self.model.clone()).answer(prompt) { + Err(PortError::Unanswered) => {} + read => return read, } std::fs::create_dir_all(&self.dir).map_err(io)?; let path = self.dir.join(format!("{digest}.prompt")); diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 943894d..958edf1 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -22,6 +22,9 @@ //! `ModelPort` (`Prompt::lower`), in two answer phases; the CLI's `lower` still //! uses the template table. The prompt source pin covers `context.rs`, //! `split.rs` and `lower.rs`. +//! +//! `RunStores::stance_ablation`, like `recall`, is a diagnostic in its own +//! directory: it records nothing and never writes its round's directory. use std::collections::{BTreeMap, BTreeSet}; use std::fmt; @@ -39,6 +42,7 @@ use crate::corpus::{CorpusItem, CorpusItemId, ModelFamily, Source}; use crate::encode::{EditDraft, LoweringDraft, PacketDraft, encoder_id}; use crate::form::{is_attestation, is_slug}; use crate::model::{ModelPort, PortError}; +use crate::packet::{Packet, Stance}; use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate}; use crate::split::{Split, SplitScheme, fnv1a64, fnv1a64_of, hex16}; use crate::store::{ @@ -950,6 +954,290 @@ impl RunStores { } } +/// What the lowerer's answer did to each variant of a packet's stance, per +/// split. Means only; no key, text or stance value. +#[derive(Clone, Debug, PartialEq)] +pub struct AblationReading { + /// Items whose recorded and replayed lowerings were both usable. + pub n: u32, + /// Answers dropped as unusable, replay or variant. + pub unusable: u32, + /// Mean agreement of the replay with the recorded lowering: the noise + /// floor a variant is read against. + pub replay: Agreement, + pub drop: VariantReading, + pub permute: VariantReading, +} + +#[derive(Clone, Debug, PartialEq, Default)] +pub struct Agreement { + pub chrf: f64, + pub meaning: f64, +} + +/// One variant's reading: agreement with the recorded lowering, pairs whose +/// chrF fell below the replay's, and the mean fidelity the item's own stance +/// adds (`score(item, replay) - score(item, variant)`). +#[derive(Clone, Debug, PartialEq, Default)] +pub struct VariantReading { + pub n: u32, + pub chrf: f64, + pub meaning: f64, + pub moved: u32, + pub ref_gain_chrf: f64, + pub ref_gain_meaning: f64, +} + +impl AblationReading { + pub fn line(&self) -> String { + let variant = |name: &str, v: &VariantReading| { + format!( + "{name} n={} chrf={:.2} meaning={:.3} moved={} ref_gain_chrf={:.2} ref_gain_meaning={:.3}", + v.n, v.chrf, v.meaning, v.moved, v.ref_gain_chrf, v.ref_gain_meaning + ) + }; + format!( + "ablation n={} unusable={} replay chrf={:.2} meaning={:.3} | {} | {}", + self.n, + self.unusable, + self.replay.chrf, + self.replay.meaning, + variant("drop", &self.drop), + variant("permute", &self.permute), + ) + } +} + +#[derive(Debug)] +pub enum AblationOutcome { + Unanswered { prompts: Vec }, + /// Train then Validation. + Read(Vec<(Split, AblationReading)>), +} + +/// What the port made of one lowering prompt. +enum Asked { + Text(String), + Unusable, + Waiting, +} + +fn asked(port: &mut dyn ModelPort, prompt: &Prompt) -> Result { + match port.answer(prompt) { + Ok(a) => Ok(LoweringDraft::parse(&a) + .ok() + .filter(LoweringDraft::usable) + .map_or(Asked::Unusable, |d| Asked::Text(d.text))), + Err(PortError::Io(ErrorKind::InvalidData)) => Ok(Asked::Unusable), + Err(PortError::Unanswered) => Ok(Asked::Waiting), + Err(PortError::Io(kind)) => Err(RunError::Io(kind)), + } +} + +fn mean(xs: impl Iterator) -> f64 { + let (sum, n) = xs.fold((0.0, 0u32), |(s, n), x| (s + x, n + 1)); + if n == 0 { 0.0 } else { sum / f64::from(n) } +} + +type Scored = (ItemScore, ItemScore); + +/// One item's reference text, recorded lowering, replay, and drop and permute +/// lowerings. +type Lowered = (String, String, String, [Option; 2]); + +impl RunStores { + /// `/ablation-`, where a stance ablation's prompts, answers and + /// variant packet stores live. + pub fn ablation_dir(&self, n: u32) -> PathBuf { + self.root.join(format!("ablation-{n}")) + } + + /// A diagnostic like [`recall`](Self::recall), never a gate: for each live + /// Train and Validation packet of plain round `n`, asks the lowerer again + /// (`model`, the replay), with the packet's stance emptied (drop) and with + /// the next item's stance in its split (permute), and reads how far the + /// lowering moves against the round's own answer (`recorded`). Test and + /// withdrawn items are never asked. Records nothing and writes nothing in + /// `round-`; the variant stores are rebuilt each call. An answer that + /// is not a usable lowering drops its pair and is counted in `unusable`. + pub fn stance_ablation( + &self, + n: u32, + recorded: &mut dyn ModelPort, + model: &mut dyn ModelPort, + scorer: &Scorer, + ) -> Result { + let store = |_| RunError::Store; + check_drift(self, model, scorer)?; + self.rounds() + .map_err(store)? + .into_iter() + .find(|r| r.n() == n && r.cycle().is_none()) + .ok_or(RunError::Malformed("round"))?; + let inventory = self.inventory.inventory().map_err(store)?; + + let dir = self.ablation_dir(n); + std::fs::create_dir_all(&dir).map_err(|e| RunError::Io(e.kind()))?; + for name in ["base", "drop", "permute"] { + for suffix in ["cc", "cc.lock"] { + let _ = std::fs::remove_file(dir.join(format!("{name}.{suffix}"))); + } + } + let mut stores = [ + PacketStore::open(&dir.join("base.cc")).map_err(store)?, + PacketStore::open(&dir.join("drop.cc")).map_err(store)?, + PacketStore::open(&dir.join("permute.cc")).map_err(store)?, + ]; + + // Live Train and Validation packets, in item key order within a split. + let round_packets = PacketStore::open(&self.root.join(format!("round-{n}")).join("packets.cc")) + .map_err(store)? + .all() + .map_err(store)?; + let mut splits: [(Split, Vec<(CorpusItem, Packet)>); 2] = + [(Split::Train, Vec::new()), (Split::Validation, Vec::new())]; + for packet in round_packets { + let item = self + .corpus + .get(packet.provenance.corpus_item.as_str()) + .map_err(store)? + .ok_or(RunError::Store)?; + for (split, held) in &mut splits { + if SplitScheme::V1.live(&item) == Some(*split) { + held.push((item.clone(), packet.clone())); + } + } + } + + let mut waiting: Vec = Vec::new(); + let mut readings = Vec::new(); + // Texts per split: (item text, recorded, replay, [drop, permute]). + let mut texts: Vec> = Vec::new(); + let mut unusable = [0u32; 2]; + for (at, (_, held)) in splits.iter_mut().enumerate() { + held.sort_by(|a, b| a.0.id.as_str().cmp(b.0.id.as_str())); + let mut rows = Vec::new(); + for (i, (item, packet)) in held.iter().enumerate() { + let packet_item = packet.provenance.corpus_item.clone(); + let mut prompt_of = |slot: usize, packet: Packet| -> Result { + let view = stores[slot] + .admit(packet, &self.inventory, &self.corpus, &self.entities) + .map_err(|_| RunError::Store)?; + Prompt::lower(&view, &packet_item, &inventory, &self.entities) + .map_err(|_| RunError::Store) + }; + let base = prompt_of(0, packet.clone())?; + let mut variants: [Option; 2] = [None, None]; + if !packet.stance.features.is_empty() { + let mut p = packet.clone(); + p.stance = Stance::default(); + variants[0] = Some(prompt_of(1, p)?); + } + if held.len() > 1 { + let mut p = packet.clone(); + p.stance = held[(i + 1) % held.len()].1.stance.clone(); + let prompt = prompt_of(2, p)?; + if prompt.digest() != base.digest() { + variants[1] = Some(prompt); + } + } + // The round's own answer must exist; a missing one is not a gap to ask for. + let recorded_text = match asked(recorded, &base)? { + Asked::Waiting => return Err(RunError::Store), + other => other, + }; + let replay = asked(model, &base)?; + let mut got: [Option; 2] = [None, None]; + for (slot, prompt) in got.iter_mut().zip(&variants) { + let Some(prompt) = prompt else { continue }; + match asked(model, prompt)? { + Asked::Text(t) => *slot = Some(t), + Asked::Unusable => unusable[at] += 1, + Asked::Waiting => { + let d = prompt.digest(); + if !waiting.contains(&d) { + waiting.push(d); + } + } + } + } + match (recorded_text, replay) { + (Asked::Text(b), Asked::Text(r)) => { + rows.push((item.text.clone(), b, r, got)); + } + (_, Asked::Waiting) => { + let d = base.digest(); + if !waiting.contains(&d) { + waiting.push(d); + } + } + _ => unusable[at] += 1, + } + } + texts.push(rows); + } + if !waiting.is_empty() { + return Ok(AblationOutcome::Unanswered { prompts: waiting }); + } + + // One score call: for each item, the replay then each variant, each + // against the recorded lowering (agreement) and the item (fidelity). + let mut inputs: Vec = Vec::new(); + for (reference, base, replay, got) in texts.iter().flatten() { + for output in std::iter::once(replay).chain(got.iter().flatten()) { + inputs.push(ScoreInput { reference: base, output }); + inputs.push(ScoreInput { reference, output }); + } + } + let mut scores = if inputs.is_empty() { + Vec::new() + } else { + scorer.score(&inputs).map_err(|_| RunError::Embed)? + } + .into_iter(); + let mut pair = || -> Result { + Ok((scores.next().ok_or(RunError::Store)?, scores.next().ok_or(RunError::Store)?)) + }; + for (at, ((split, _), rows)) in splits.iter().zip(&texts).enumerate() { + let mut replay: Vec = Vec::new(); + let mut variants: [Vec<(Scored, Scored)>; 2] = [Vec::new(), Vec::new()]; + for (_, _, _, got) in rows { + let r = pair()?; + for (k, g) in got.iter().enumerate() { + if g.is_some() { + variants[k].push((r.clone(), pair()?)); + } + } + replay.push(r); + } + let variant = |v: &[(Scored, Scored)]| VariantReading { + n: v.len() as u32, + chrf: mean(v.iter().map(|(_, x)| x.0.chrf)), + meaning: mean(v.iter().filter_map(|(_, x)| x.0.meaning)), + moved: v.iter().filter(|(r, x)| x.0.chrf < r.0.chrf).count() as u32, + ref_gain_chrf: mean(v.iter().map(|(r, x)| r.1.chrf - x.1.chrf)), + ref_gain_meaning: mean( + v.iter().filter_map(|(r, x)| Some(r.1.meaning? - x.1.meaning?)), + ), + }; + readings.push(( + *split, + AblationReading { + n: rows.len() as u32, + unusable: unusable[at], + replay: Agreement { + chrf: mean(replay.iter().map(|r| r.0.chrf)), + meaning: mean(replay.iter().filter_map(|r| r.0.meaning)), + }, + drop: variant(&variants[0]), + permute: variant(&variants[1]), + }, + )); + } + Ok(AblationOutcome::Read(readings)) + } +} + /// The distinct rule codes an admission refusal broke: validator rules by /// code, a store refusal by its variant name. Nothing from the packet or the /// stores is kept; an I/O failure is `io`. @@ -1008,7 +1296,7 @@ pub fn round( } /// The model and embedder the run pinned are the ones answering. -fn check_drift( +fn check_drift( run: &RunStores, model: &M, scorer: &Scorer, diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 4113a82..3d38bba 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -1633,3 +1633,74 @@ fn loop_open_requires_the_lowerer_reach_and_pins_it() { weksa::run::LowererReach::BriefedSubagent ); } + +/// Answers the Dao encode prompts with the Dao packets and every lowering with +/// one sentence. +struct DaoModel; + +impl weksa::model::ModelPort for DaoModel { + fn model(&self) -> &str { + "fake-model" + } + fn answer(&mut self, prompt: &weksa::context::Prompt) -> Result { + if prompt.text().starts_with(weksa::context::LOWER_INSTRUCTIONS) { + return Ok(r#"{"text":"The highest excellence is like the water."}"#.into()); + } + let n: usize = prompt.item_keys()[0].rsplit(':').next().unwrap().parse().unwrap(); + Ok(serde_json::to_string(&weksa::PacketDraft::of(&weksa::daodejing::packets()[n - 1])).unwrap()) + } +} + +#[test] +fn loop_ablate_waits_then_prints_one_line_per_split() { + let d = tempfile::tempdir().unwrap(); + open_dao_run(d.path()); + let url = fake_embedder(); + let root = d.path().join("canaryroot"); + + // Record round 1 the way `loop round` does, then put its lowering answers + // where the relay would have. + let mut run = weksa::run::RunStores::open(&root).unwrap(); + let scorer = weksa::Scorer::new(weksa::OllamaEmbedder::new(&url, &run.record().embedder)); + let weksa::run::RoundOutcome::Recorded(r) = weksa::run::round(&mut run, &mut DaoModel, &scorer).unwrap() + else { + panic!("answered") + }; + for call in &r.calls()[3..] { + std::fs::write( + root.join(format!("round-1/{}.answer", call.prompt)), + r#"{"text":"The highest excellence is like the water."}"#, + ) + .unwrap(); + } + + let ablate = || weksa(d.path(), &["loop", "ablate", "canaryroot", "--round", "1", "--ollama-url", &url]); + let first = ablate(); + assert_eq!(first.status.code(), Some(3), "{}", out(&first)); + let text = out(&first); + let digests: Vec<&str> = text + .lines() + .filter_map(|l| l.strip_prefix("unanswered ")) + .filter(|d| d.len() == 16) + .collect(); + // Three replays, three drops and the Validation permutes (2 items). + assert_eq!(digests.len(), 3 + 3 + 2, "{text}"); + for digest in &digests { + assert!(root.join(format!("ablation-1/{digest}.prompt")).is_file()); + std::fs::write( + root.join(format!("ablation-1/{digest}.answer")), + r#"{"text":"The highest excellence is like the water."}"#, + ) + .unwrap(); + } + let second = ablate(); + assert!(second.status.success(), "{}", out(&second)); + let text = out(&second); + let lines: Vec<&str> = text.lines().collect(); + assert_eq!(lines.len(), 2, "{text}"); + assert!(lines[0].starts_with("train ablation n=1 "), "{text}"); + assert!(lines[1].starts_with("validation ablation n=2 "), "{text}"); + for leaked in ["item:", "packet:", "highest excellence", "canaryroot"] { + assert!(!text.contains(leaked), "{text}"); + } +} diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 0929e53..c9c4bd6 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -2598,9 +2598,19 @@ fn source_tripwires() { let count = |needle: &str| code.iter().filter(|l| l.contains(needle)).count(); assert!(!run.contains("pub fn new")); // The round does not call the template table: the model lowers. - assert_eq!(count("lower(&"), 0, "run.rs calls the table"); + assert_eq!( + code.iter() + .filter(|l| l.contains("lower(&") && !l.contains("Prompt::lower(")) + .count(), + 0, + "run.rs calls the table" + ); assert_eq!(count("crate::lower::lower"), 0, "run.rs imports the table"); - assert_eq!(count("Prompt::lower("), 1, "one lowering call site"); + assert_eq!( + count("Prompt::lower("), + 2, + "two lowering call sites: the round's and the stance ablation's replay of it" + ); // Prompt::lower takes the view, the item's key, the inventory and the entity // port: no corpus item, train set, offer or store. let start = context.find("pub fn lower(").unwrap(); @@ -3560,3 +3570,441 @@ fn own_family_counts_offered_items_of_the_answerer_family() { assert_eq!(mixed_family_round(ModelFamily::Google), 0); assert_eq!(mixed_family_round(ModelFamily::Unrecorded), 0); } + +// ---- stance ablation (stance-to-lowerer) ---- + +/// A lowerer by function: the text it answers for an item key and the prompt +/// text, recording every prompt it was shown. +struct Lowerer { + f: fn(&str, &str) -> String, + seen: Vec<(String, String, String)>, +} + +impl Lowerer { + fn new(f: fn(&str, &str) -> String) -> Self { + Self { f, seen: vec![] } + } + /// The digests asked, once each, in the order first asked. + fn digests(&self) -> Vec { + let mut out: Vec = Vec::new(); + for (d, ..) in &self.seen { + if !out.contains(d) { + out.push(d.clone()); + } + } + out + } +} + +impl ModelPort for Lowerer { + fn model(&self) -> &str { + "fake-model" + } + fn answer(&mut self, prompt: &Prompt) -> Result { + let key = prompt.item_keys()[0].clone(); + self.seen.push((prompt.digest(), key.clone(), prompt.text().to_string())); + Ok(serde_json::json!({ "text": (self.f)(&key, prompt.text()) }).to_string()) + } +} + +const WRITTEN: &str = "item:written:1"; + +fn english(key: &str) -> String { + let mut table = table_english(); + table.insert( + WRITTEN.into(), + "Heaven endures through the long years without end.".into(), + ); + table[key].clone() +} + +/// The packet line's stance, as the prompt prints it: the packet line is one +/// JSON object ending in the stance. +fn stance_of(text: &str) -> String { + let line = text.lines().find(|l| l.starts_with("packet: ")).unwrap(); + line[line.find("\"stance\":").unwrap()..].to_string() +} + +/// Says the same thing whatever the stance. +fn blind(key: &str, _text: &str) -> String { + english(key) +} + +/// Appends the stance it was shown. +fn reader(key: &str, text: &str) -> String { + format!("{} {}", english(key), stance_of(text)) +} + +/// Does better with the item's own stance: the real sentence for it, nonsense +/// for any other, empty included. +fn helped(key: &str, text: &str) -> String { + let r800 = r#""stance":{"features":[{"feature":"feature:18","value":{"Grade":800}}]}}"#; + let own = match key { + "item:daodejing:3" => r#""stance":{"features":[{"feature":"feature:18","value":{"Grade":800}},{"feature":"feature:16","value":{"Grade":300}}]}}"#, + WRITTEN => r#""stance":{"features":[{"feature":"feature:18","value":{"Grade":500}}]}}"#, + _ => r800, + }; + if stance_of(text) == own { english(key) } else { "zzz qqq xxx".into() } +} + +fn grade_stance(features: &[(&str, u16)]) -> weksa::packet::Stance { + weksa::packet::Stance { + features: features + .iter() + .map(|(f, v)| weksa::packet::FeatureValue { + feature: f.parse().unwrap(), + value: weksa::packet::FeatValue::Grade(common::grade(*v)), + }) + .collect(), + } +} + +/// A Dao run with a third Validation item (`item:written:1`, key order after +/// items 2 and 3) and round 1 recorded over it, every item encoded to its Dao +/// packet. Stances: item 1 restraint 800; item 2 restraint 800; item 3 +/// restraint 800 and affection 300; the written item restraint 500. +fn recorded_dao(edit: impl FnOnce(&mut HashMap)) -> (Dao, RoundRecord) { + let mut dao = dao_over(|corpus, entities| { + let entities = weksa::EntityStore::open(entities).unwrap(); + let mut store = CorpusStore::open(corpus).unwrap(); + let mut written = store.get("item:daodejing:2").unwrap().unwrap(); + written.id = WRITTEN.into(); + written.text = "Heaven endures through the long years without end.".into(); + written.source = weksa::corpus::Source::OperatorWritten; + store.add(written, &entities).unwrap(); + }); + let mut model = fake(&good_answers()); + let mut extra = draft_of(2); + extra.referents.truncate(1); + extra.predications.truncate(1); + extra.stance = grade_stance(&[("feature:18", 500)]); + model.answers.insert(WRITTEN.into(), json(&extra)); + edit(&mut model.answers); + model.lowered.insert(WRITTEN.into(), english(WRITTEN)); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + (dao, r) +} + +fn ablate( + dao: &Dao, + recorded: &mut dyn ModelPort, + model: &mut dyn ModelPort, +) -> Result { + dao.run.stance_ablation(1, recorded, model, &Scorer::new(Bag)) +} + +fn read( + outcome: weksa::run::AblationOutcome, +) -> Vec<(weksa::Split, weksa::run::AblationReading)> { + match outcome { + weksa::run::AblationOutcome::Read(r) => r, + weksa::run::AblationOutcome::Unanswered { prompts } => panic!("unanswered {prompts:?}"), + } +} + +#[test] +fn recorded_answers_read_and_never_write() { + use std::io::ErrorKind; + use weksa::model::RecordedAnswers; + let p = prompt(); + let dir = tempfile::tempdir().unwrap(); + let mut port = RecordedAnswers::new(dir.path().to_path_buf(), "answerer".into()); + assert_eq!(port.model(), "answerer"); + // A miss is Unanswered and the directory is as it was. + assert_eq!(port.answer(&p), Err(PortError::Unanswered)); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0); + // A directory that does not exist is a miss too, and is not made. + let mut absent = RecordedAnswers::new(dir.path().join("nope"), "m".into()); + assert_eq!(absent.answer(&p), Err(PortError::Unanswered)); + assert!(!dir.path().join("nope").exists()); + // A hit is the file's text as written. + let file = dir.path().join(format!("{}.answer", p.digest())); + std::fs::write(&file, " {\"x\": 1}\n").unwrap(); + assert_eq!(port.answer(&p).unwrap(), " {\"x\": 1}\n"); + std::fs::write(&file, [0xff, 0xfe]).unwrap(); + assert_eq!(port.answer(&p), Err(PortError::Io(ErrorKind::InvalidData))); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1); +} + +#[test] +fn a_stance_ablation_asks_replay_drop_and_permute_for_train_and_validation_only() { + let (dao, r) = recorded_dao(|_| {}); + let round_digests: Vec = r.calls()[4..].iter().map(|c| c.prompt.clone()).collect(); + assert_eq!(round_digests.len(), 4, "the round lowered four items"); + let mut recorded_port = Lowerer::new(reader); + let mut model = Lowerer::new(reader); + read(ablate(&dao, &mut recorded_port, &mut model).unwrap()); + + // The base prompts are the round's own, byte for byte. + let recorded_digests = recorded_port.digests(); + assert_eq!(recorded_digests, round_digests); + let asked = model.digests(); + for d in &round_digests { + assert!(asked.contains(d), "the replay asks the round's prompt {d}"); + } + // Per item: replay and drop for each; permute for each Validation item, and + // none for the lone Train item. + let per_key = |key: &str| model.seen.iter().filter(|s| s.1 == key).count(); + assert_eq!(per_key("item:daodejing:1"), 2); + for key in ["item:daodejing:2", "item:daodejing:3", WRITTEN] { + assert_eq!(per_key(key), 3, "{key}"); + } + // Test items are never asked. + for (_, key, text) in &model.seen { + assert!(key != "item:daodejing:4" && key != "item:daodejing:5", "{key}"); + assert!(!text.contains("item:daodejing:4")); + } + // A drop prompt prints the empty stance, a replay the item's own. + let empty = model + .seen + .iter() + .filter(|s| stance_of(&s.2) == "\"stance\":{\"features\":[]}}") + .count(); + assert_eq!(empty, 4, "one drop per item"); +} + +#[test] +fn a_lowerer_blind_to_stance_reads_no_movement_and_one_that_reads_it_does() { + let (dao, _) = recorded_dao(|_| {}); + let one = |f: fn(&str, &str) -> String| { + let (mut rec, mut model) = (Lowerer::new(f), Lowerer::new(f)); + read(ablate(&dao, &mut rec, &mut model).unwrap()) + }; + for (split, r) in one(blind) { + assert_eq!(r.unusable, 0); + assert_eq!((r.drop.moved, r.permute.moved), (0, 0), "{split:?}: {}", r.line()); + assert!((r.replay.chrf - 100.0).abs() < 1e-9); + assert!((r.drop.chrf - r.replay.chrf).abs() < 1e-9); + if r.permute.n > 0 { + assert!((r.permute.chrf - r.replay.chrf).abs() < 1e-9); + } + assert_eq!(r.drop.ref_gain_chrf, 0.0); + } + let reading = one(reader); + let (train, validation) = (&reading[0].1, &reading[1].1); + assert_eq!((reading[0].0, reading[1].0), (weksa::Split::Train, weksa::Split::Validation)); + assert_eq!((train.n, validation.n), (1, 3)); + assert_eq!((train.drop.n, train.drop.moved), (1, 1)); + assert_eq!((train.permute.n, train.permute.moved), (0, 0)); + assert_eq!((validation.drop.n, validation.drop.moved), (3, 3)); + assert_eq!((validation.permute.n, validation.permute.moved), (3, 3)); + assert!((validation.replay.chrf - 100.0).abs() < 1e-9); + assert!(validation.drop.chrf < validation.replay.chrf); + assert!(validation.permute.chrf < validation.replay.chrf); +} + +#[test] +fn the_item_s_own_stance_helps_fidelity_only_for_a_lowerer_that_uses_it() { + let (dao, _) = recorded_dao(|_| {}); + let one = |f: fn(&str, &str) -> String| { + let (mut rec, mut model) = (Lowerer::new(f), Lowerer::new(f)); + read(ablate(&dao, &mut rec, &mut model).unwrap()) + }; + for (split, r) in one(helped) { + assert!(r.drop.ref_gain_chrf > 10.0, "{split:?}: {}", r.line()); + assert!(r.drop.ref_gain_meaning > 0.0, "{split:?}: {}", r.line()); + if r.permute.n > 0 { + assert!(r.permute.ref_gain_chrf > 10.0, "{split:?}: {}", r.line()); + } + assert_eq!(r.drop.moved, r.drop.n); + } + for (split, r) in one(blind) { + assert_eq!((r.drop.ref_gain_chrf, r.drop.ref_gain_meaning), (0.0, 0.0), "{split:?}"); + } +} + +#[test] +fn a_packet_without_stance_asks_no_drop_and_a_test_packet_in_the_round_is_never_asked() { + let (dao, _) = recorded_dao(|answers| { + let mut d = draft_of(3); + d.stance = Default::default(); + answers.insert("item:daodejing:3".into(), json(&d)); + }); + // A packet for a Test item, put into the round's own store the way no round + // would: the ablation reads splits from the corpus, not from the store. + { + let corpus = CorpusStore::open(&dao.corpus).unwrap(); + let inv = dao.run.inventory(); + let mut packet = packets().remove(3); + packet.inventory_rev = 3; + PacketStore::open(&dao.root.join("round-1/packets.cc")) + .unwrap() + .admit(packet, inv, &corpus, dao.run.entities()) + .unwrap(); + } + let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(reader)); + read(ablate(&dao, &mut rec, &mut model).unwrap()); + let bare = "\"stance\":{\"features\":[]}}"; + let drops_of_3 = model + .seen + .iter() + .filter(|s| s.1 == "item:daodejing:3" && stance_of(&s.2) == bare) + .count(); + // Its replay already shows the empty stance; there is no drop beside it. + assert_eq!(drops_of_3, 1); + assert!(model.seen.iter().all(|s| s.1 != "item:daodejing:4"), "a Test item is never asked"); + assert!(rec.seen.iter().all(|s| s.1 != "item:daodejing:4")); +} + +#[test] +fn permute_takes_the_next_items_stance_in_its_split_and_skips_equal_ones() { + let (dao, _) = recorded_dao(|_| {}); + let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(reader)); + read(ablate(&dao, &mut rec, &mut model).unwrap()); + // The stance each Validation item's permute prompt shows: the next item's + // in key order (2, 3, written:1), cyclic; item 1 is alone in Train. + let stance = |text: &str| stance_of(text); + let of = |key: &str| -> Vec { + model + .seen + .iter() + .filter(|s| s.1 == key) + .map(|s| stance(&s.2)) + .collect() + }; + let (s2, s3, sw) = ( + stance(&rec.seen.iter().find(|s| s.1 == "item:daodejing:2").unwrap().2), + stance(&rec.seen.iter().find(|s| s.1 == "item:daodejing:3").unwrap().2), + stance(&rec.seen.iter().find(|s| s.1 == WRITTEN).unwrap().2), + ); + assert!(s2 != s3 && s3 != sw && sw != s2); + assert!(of("item:daodejing:2").contains(&s3)); + assert!(of("item:daodejing:3").contains(&sw)); + assert!(of(WRITTEN).contains(&s2), "the last wraps to the first"); + assert!(!of("item:daodejing:2").contains(&sw), "not the previous or the other"); + // Item 1 is alone in Train: it never takes a stance across splits. + assert_eq!(of("item:daodejing:1").len(), 2, "replay and drop only"); + + // Items 2 and 3 with equal stances: item 2's permute is its own prompt, so + // it asks none; the others still do. + let (dao, _) = recorded_dao(|answers| { + let mut d = draft_of(3); + d.stance = draft_of(2).stance; + answers.insert("item:daodejing:3".into(), json(&d)); + }); + let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(reader)); + let r = read(ablate(&dao, &mut rec, &mut model).unwrap()); + let per_key = |key: &str| model.seen.iter().filter(|s| s.1 == key).count(); + assert_eq!(per_key("item:daodejing:2"), 2, "replay and drop: its next holds the stance it has"); + assert_eq!(per_key("item:daodejing:3"), 3); + assert_eq!(per_key(WRITTEN), 3); + assert_eq!(r[1].1.permute.n, 2, "{}", r[1].1.line()); +} + +#[test] +fn a_stance_ablation_records_nothing_and_writes_nothing_in_its_round() { + use weksa::model::RecordedAnswers; + let (dao, r) = recorded_dao(|_| {}); + let round1 = dao.root.join("round-1"); + for c in &r.calls()[4..] { + std::fs::write( + round1.join(format!("{}.answer", c.prompt)), + serde_json::json!({ "text": english(&c.items[0]) }).to_string(), + ) + .unwrap(); + } + let snapshot = |dir: &Path| -> Vec<(String, Vec)> { + let mut files: Vec<_> = std::fs::read_dir(dir) + .unwrap() + .map(|e| { + let e = e.unwrap(); + let bytes = if e.path().is_dir() { vec![] } else { std::fs::read(e.path()).unwrap() }; + (e.file_name().to_string_lossy().into_owned(), bytes) + }) + .collect(); + files.sort(); + files.retain(|f| f.0 != "ablation-1"); + files + }; + let before = (snapshot(&round1), snapshot(&dao.root), dao.run.rounds().unwrap(), dao.run.decisions().unwrap()); + let go = || { + let mut rec = RecordedAnswers::new(round1.clone(), "fake-model".into()); + let mut files = weksa::model::AnswerFiles::new(dao.run.ablation_dir(1), "fake-model".into()); + // Nothing answers yet: the replay and variant prompts are written. + dao.run.stance_ablation(1, &mut rec, &mut files, &Scorer::new(Bag)) + }; + let weksa::run::AblationOutcome::Unanswered { prompts } = go().unwrap() else { + panic!("nobody answered") + }; + // Four replays (the round's own prompts), four drops, three permutes (item 1 + // is alone in Train). + assert_eq!(prompts.len(), 4 + 4 + 3, "{prompts:?}"); + for d in &prompts { + assert!(dao.run.ablation_dir(1).join(format!("{d}.prompt")).is_file()); + } + // Answer every ask from a fake and read; then again over the same files. + for d in &prompts { + let text = std::fs::read_to_string(dao.run.ablation_dir(1).join(format!("{d}.prompt"))).unwrap(); + std::fs::write( + dao.run.ablation_dir(1).join(format!("{d}.answer")), + serde_json::json!({ "text": format!("answer {}", &text[..8]) }).to_string(), + ) + .unwrap(); + } + let first = read(go().unwrap()); + let again = read(go().unwrap()); + assert_eq!(first, again); + let after = (snapshot(&round1), snapshot(&dao.root), dao.run.rounds().unwrap(), dao.run.decisions().unwrap()); + assert!(before.0 == after.0, "round-1 is unchanged byte for byte"); + assert!(before.1 == after.1, "the run root's files are unchanged"); + assert_eq!(before.2, after.2); + assert_eq!(before.3, after.3); +} + +#[test] +fn a_stance_ablation_refuses_a_cycle_round_and_missing_round_answers() { + use weksa::model::RecordedAnswers; + let (dao, _) = recorded_dao(|_| {}); + // No round-2, and round 1 holds no answer files. + let mut model = Lowerer::new(reader); + let mut rec = RecordedAnswers::new(dao.root.join("round-1"), "fake-model".into()); + assert!(matches!( + dao.run.stance_ablation(2, &mut rec, &mut model, &Scorer::new(Bag)), + Err(RunError::Malformed("round")) + )); + assert!(matches!( + ablate(&dao, &mut rec, &mut model), + Err(RunError::Store) + )); + assert!( + std::fs::read_dir(dao.root.join("round-1")) + .unwrap() + .all(|e| !e.unwrap().file_name().to_string_lossy().ends_with(".prompt")), + "no prompt is written into the round" + ); + // Drift is refused as a round refuses it. + struct Named(Lowerer); + impl ModelPort for Named { + fn model(&self) -> &str { + "other-model" + } + fn answer(&mut self, p: &Prompt) -> Result { + self.0.answer(p) + } + } + let mut named = Named(Lowerer::new(reader)); + let mut rec2 = Lowerer::new(reader); + assert!(matches!( + ablate(&dao, &mut rec2, &mut named), + Err(RunError::Drift(DriftOf::Model)) + )); + assert!(matches!( + dao.run.stance_ablation(1, &mut rec2, &mut Lowerer::new(reader), &Scorer::new(OtherEmbedder)), + Err(RunError::Drift(DriftOf::Embedder)) + )); +} + +#[test] +fn a_stance_ablation_refuses_a_cycle_round() { + let mut dao = dao(); + two_rounds(&mut dao); + run_cycle(&mut dao, &mut cycle_fake(ONE_CONCEPT)); + let rounds = dao.run.rounds().unwrap(); + let candidate = rounds.iter().find(|r| r.cycle().is_some()).expect("the cycle's round"); + let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(reader)); + assert!(matches!( + dao.run.stance_ablation(candidate.n(), &mut rec, &mut model, &Scorer::new(Bag)), + Err(RunError::Malformed("round")) + )); + assert!(model.seen.is_empty() && rec.seen.is_empty()); +} -- 2.47.3 From 633bf831dc3e4cfaeb3549b6653d760187f72551 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 18:27:23 +0000 Subject: [PATCH 2/4] Pin that the lowering prompt moves with the stance alone One dao packet admitted with its own stance, an anger-and-restraint stance and none: three digests, each packet line ending in exactly that stance's JSON, the rest of the text byte-equal, the stance features glossed in the one inventory listing. No prompt change; the lowerer already reads the stance. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/context.rs | 44 +++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs index d6e2efe..32d3306 100644 --- a/crates/weksa/tests/context.rs +++ b/crates/weksa/tests/context.rs @@ -642,6 +642,50 @@ mod lowering { assert!(!prompt.text().contains(item.as_str())); } + #[test] + fn lower_prompt_moves_with_the_stance_alone() { + use weksa::packet::{FeatValue, FeatureValue, Grade, Stance}; + let (_, inv, entities, item) = dao(2); + let store = InventoryStore::open(&data("daodejing/inventory.cc")).unwrap(); + let own = packets().remove(2).stance; + let grade = |feature: &str, v: u16| FeatureValue { + feature: feature.parse().unwrap(), + value: FeatValue::Grade(Grade::new(v).unwrap()), + }; + let angry = Stance { + features: vec![grade("feature:9", 800), grade("feature:18", 900)], + }; + let prompt_of = |stance: Stance| { + let mut packet = packets().remove(2); + packet.stance = stance.clone(); + let view = admitted(packet, &store, "daodejing/corpus.cc", &entities); + let prompt = Prompt::lower(&view, &item, &inv, &entities).unwrap(); + (stance, prompt) + }; + let three = [prompt_of(own), prompt_of(angry), prompt_of(Stance::default())]; + + // Three stances, three prompts. + let digests: std::collections::BTreeSet = + three.iter().map(|(_, p)| p.digest()).collect(); + assert_eq!(digests.len(), 3); + // Each packet line ends with exactly that stance's JSON, and the prompt + // with the packet line taken out is the same text for all three. + let without_packet_line = |text: &str| -> String { + text.lines().filter(|l| !l.starts_with("packet: ")).collect::>().join("\n") + }; + let bare = without_packet_line(three[0].1.text()); + for (stance, prompt) in &three { + let line = prompt.text().lines().find(|l| l.starts_with("packet: ")).unwrap(); + let json = serde_json::to_string(stance).unwrap(); + assert!(line.ends_with(&format!(r#","stance":{json}}}"#)), "{line}"); + assert_eq!(without_packet_line(prompt.text()), bare); + } + // The stance's feature ids are glossed in the one inventory listing. + let text = three[1].1.text(); + assert!(text.contains("\nfeature:9: anger [grade; slots Stance]\n")); + assert!(text.contains("\nfeature:18: restraint [grade; slots Stance]\n")); + } + #[test] fn an_entity_referent_prints_its_en_us_forms_citation_first() { let cast = InventoryStore::open(&data("interlingua/inventory.cc")).unwrap(); -- 2.47.3 From 00feb1d31071fcebfdacdbee55a48a335091a8f1 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 18:28:14 +0000 Subject: [PATCH 3/4] Lowering brief states the grade scale LOWER_INSTRUCTIONS gains 'a grade is permille, 0 to 1000' (the encode brief already states it). The pinned lowering test requires the sentence and asserts the item's speaker entity id and group are in no prompt text. This edits context.rs, a PROMPT_SOURCE_DIGEST input: the digest moves and runs opened before this commit stop opening; open a fresh run to ablate under the new brief. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/context.rs | 2 +- crates/weksa/tests/context.rs | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index 1a648a0..8e2cbfc 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -66,7 +66,7 @@ A feature is {feature, value}; value is {\"Grade\": 0 to 1000}, {\"Option\": lab /// Fixed per run by `PROMPT_SOURCE_DIGEST`. pub const LOWER_INSTRUCTIONS: &str = "\ Say the packet below in English (en-US), as one utterance of its speaker: all of its meaning and nothing it does not carry. -Every id is the inventory's listed below and every feature value is as listed. The stance is the speaker's felt affect and restraint: it shapes how the utterance is said, not what it says. +Every id is the inventory's listed below and every feature value is as listed; a grade is permille, 0 to 1000. The stance is the speaker's felt affect and restraint: it shapes how the utterance is said, not what it says. A referent with an entity is named by one of that entity's forms on the entities line. Answer with one JSON object and nothing else: {\"text\": \"...\"}. No code fence, no commentary."; diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs index 32d3306..5979a26 100644 --- a/crates/weksa/tests/context.rs +++ b/crates/weksa/tests/context.rs @@ -630,6 +630,7 @@ mod lowering { for needed in [ "en-US", "all of its meaning and nothing it does not carry", + "a grade is permille, 0 to 1000", "felt affect and restraint", "shapes how the utterance is said, not what it says", "forms on the entities line", @@ -640,6 +641,17 @@ mod lowering { // The one call it makes is its item; the key is in no text. assert_eq!(prompt.item_keys(), [item.as_str()]); assert!(!prompt.text().contains(item.as_str())); + // The view's speaker and group are private: the item's speaker entity + // id and its group are in no text. + let held = CorpusStore::open(&data("daodejing/corpus.cc")) + .unwrap() + .get(item.as_str()) + .unwrap() + .unwrap(); + assert!(!held.group.is_empty()); + for private in [held.speaker.as_str(), held.group.as_str()] { + assert!(!prompt.text().contains(private), "{private}"); + } } #[test] -- 2.47.3 From e8a9bbc5abce05a336890a87cf18e6b13eb92569 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 19:01:24 +0000 Subject: [PATCH 4/4] Ablation reads none for an empty set, pins its line, names a missing round answer Soul s1 findings: an empty mean printed 0.00 beside n=0 (now Option, printed none, like SplitMean); the printed line and whole ablate output are pinned by exact string, with drop and permute each carrying their own n; a missing recorded round answer is RunError::Unrecorded and says the prompt digest drifted or the head inventory moved, not that a store would not read; the unusable count is pinned for variant and item sides. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/run.rs | 45 +++++++---- crates/weksa/tests/cli.rs | 24 +++++- crates/weksa/tests/golf_loop.rs | 131 ++++++++++++++++++++++++++++---- 3 files changed, 167 insertions(+), 33 deletions(-) diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 958edf1..dbbf6ad 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -455,6 +455,9 @@ pub enum RunError { NoFloor, /// A round has no Validation reading. NoValidation, + /// The round holds no answer to a lowering prompt the ablation rebuilt: the + /// prompt's digest drifted or the head inventory moved since the round. + Unrecorded, } impl fmt::Display for RunError { @@ -470,6 +473,9 @@ impl fmt::Display for RunError { RunError::NoInventory => f.write_str("the inventory holds no revision"), RunError::Io(kind) => write!(f, "io error {kind:?}"), RunError::Store => f.write_str("a store would not read or write"), + RunError::Unrecorded => f.write_str( + "the round holds no answer to a lowering prompt rebuilt now: the prompt digest drifted or the head inventory moved since the round", + ), RunError::Embed => f.write_str("the embedder failed"), RunError::NoFloor => f.write_str("the run has no noise floor: record two rounds first"), RunError::NoValidation => f.write_str("a reading has no validation item"), @@ -969,10 +975,12 @@ pub struct AblationReading { pub permute: VariantReading, } +/// Means over an empty set are `None`, printed `none`: no measurement, never +/// a zero. #[derive(Clone, Debug, PartialEq, Default)] pub struct Agreement { - pub chrf: f64, - pub meaning: f64, + pub chrf: Option, + pub meaning: Option, } /// One variant's reading: agreement with the recorded lowering, pairs whose @@ -981,27 +989,36 @@ pub struct Agreement { #[derive(Clone, Debug, PartialEq, Default)] pub struct VariantReading { pub n: u32, - pub chrf: f64, - pub meaning: f64, + pub chrf: Option, + pub meaning: Option, pub moved: u32, - pub ref_gain_chrf: f64, - pub ref_gain_meaning: f64, + pub ref_gain_chrf: Option, + pub ref_gain_meaning: Option, +} + +fn show(x: Option, places: usize) -> String { + x.map_or("none".to_string(), |x| format!("{x:.places$}")) } impl AblationReading { pub fn line(&self) -> String { let variant = |name: &str, v: &VariantReading| { format!( - "{name} n={} chrf={:.2} meaning={:.3} moved={} ref_gain_chrf={:.2} ref_gain_meaning={:.3}", - v.n, v.chrf, v.meaning, v.moved, v.ref_gain_chrf, v.ref_gain_meaning + "{name} n={} chrf={} meaning={} moved={} ref_gain_chrf={} ref_gain_meaning={}", + v.n, + show(v.chrf, 2), + show(v.meaning, 3), + v.moved, + show(v.ref_gain_chrf, 2), + show(v.ref_gain_meaning, 3) ) }; format!( - "ablation n={} unusable={} replay chrf={:.2} meaning={:.3} | {} | {}", + "ablation n={} unusable={} replay chrf={} meaning={} | {} | {}", self.n, self.unusable, - self.replay.chrf, - self.replay.meaning, + show(self.replay.chrf, 2), + show(self.replay.meaning, 3), variant("drop", &self.drop), variant("permute", &self.permute), ) @@ -1034,9 +1051,9 @@ fn asked(port: &mut dyn ModelPort, prompt: &Prompt) -> Result { } } -fn mean(xs: impl Iterator) -> f64 { +fn mean(xs: impl Iterator) -> Option { let (sum, n) = xs.fold((0.0, 0u32), |(s, n), x| (s + x, n + 1)); - if n == 0 { 0.0 } else { sum / f64::from(n) } + (n > 0).then(|| sum / f64::from(n)) } type Scored = (ItemScore, ItemScore); @@ -1143,7 +1160,7 @@ impl RunStores { } // The round's own answer must exist; a missing one is not a gap to ask for. let recorded_text = match asked(recorded, &base)? { - Asked::Waiting => return Err(RunError::Store), + Asked::Waiting => return Err(RunError::Unrecorded), other => other, }; let replay = asked(model, &base)?; diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 3d38bba..f5562cd 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -1696,11 +1696,27 @@ fn loop_ablate_waits_then_prints_one_line_per_split() { let second = ablate(); assert!(second.status.success(), "{}", out(&second)); let text = out(&second); - let lines: Vec<&str> = text.lines().collect(); - assert_eq!(lines.len(), 2, "{text}"); - assert!(lines[0].starts_with("train ablation n=1 "), "{text}"); - assert!(lines[1].starts_with("validation ablation n=2 "), "{text}"); + assert_eq!( + text, + "train ablation n=1 unusable=0 replay chrf=100.00 meaning=1.000 | drop n=1 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000 | permute n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none\nvalidation ablation n=2 unusable=0 replay chrf=100.00 meaning=1.000 | drop n=2 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000 | permute n=2 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000\n" + ); for leaked in ["item:", "packet:", "highest excellence", "canaryroot"] { assert!(!text.contains(leaked), "{text}"); } } + +#[test] +fn loop_ablate_without_the_rounds_answers_says_so() { + let d = tempfile::tempdir().unwrap(); + open_dao_run(d.path()); + let url = fake_embedder(); + let root = d.path().join("canaryroot"); + let mut run = weksa::run::RunStores::open(&root).unwrap(); + let scorer = weksa::Scorer::new(weksa::OllamaEmbedder::new(&url, &run.record().embedder)); + weksa::run::round(&mut run, &mut DaoModel, &scorer).unwrap(); + let o = weksa(d.path(), &["loop", "ablate", "canaryroot", "--round", "1", "--ollama-url", &url]); + assert!(!o.status.success()); + let text = out(&o); + assert!(text.contains("holds no answer") && text.contains("head inventory"), "{text}"); + assert!(!text.contains("would not read"), "{text}"); +} diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index c9c4bd6..4a1c7ee 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -3773,12 +3773,12 @@ fn a_lowerer_blind_to_stance_reads_no_movement_and_one_that_reads_it_does() { for (split, r) in one(blind) { assert_eq!(r.unusable, 0); assert_eq!((r.drop.moved, r.permute.moved), (0, 0), "{split:?}: {}", r.line()); - assert!((r.replay.chrf - 100.0).abs() < 1e-9); - assert!((r.drop.chrf - r.replay.chrf).abs() < 1e-9); + assert!((r.replay.chrf.unwrap() - 100.0).abs() < 1e-9); + assert!((r.drop.chrf.unwrap() - r.replay.chrf.unwrap()).abs() < 1e-9); if r.permute.n > 0 { - assert!((r.permute.chrf - r.replay.chrf).abs() < 1e-9); + assert!((r.permute.chrf.unwrap() - r.replay.chrf.unwrap()).abs() < 1e-9); } - assert_eq!(r.drop.ref_gain_chrf, 0.0); + assert_eq!(r.drop.ref_gain_chrf, Some(0.0)); } let reading = one(reader); let (train, validation) = (&reading[0].1, &reading[1].1); @@ -3788,9 +3788,23 @@ fn a_lowerer_blind_to_stance_reads_no_movement_and_one_that_reads_it_does() { assert_eq!((train.permute.n, train.permute.moved), (0, 0)); assert_eq!((validation.drop.n, validation.drop.moved), (3, 3)); assert_eq!((validation.permute.n, validation.permute.moved), (3, 3)); - assert!((validation.replay.chrf - 100.0).abs() < 1e-9); - assert!(validation.drop.chrf < validation.replay.chrf); - assert!(validation.permute.chrf < validation.replay.chrf); + // A lone Train item has no permute: no measurement, not total disagreement. + assert_eq!( + train.permute, + weksa::run::VariantReading::default(), + "{}", + train.line() + ); + assert!( + train.line().contains( + "| permute n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none" + ), + "{}", + train.line() + ); + assert!((validation.replay.chrf.unwrap() - 100.0).abs() < 1e-9); + assert!(validation.drop.chrf.unwrap() < validation.replay.chrf.unwrap()); + assert!(validation.permute.chrf.unwrap() < validation.replay.chrf.unwrap()); } #[test] @@ -3801,15 +3815,15 @@ fn the_item_s_own_stance_helps_fidelity_only_for_a_lowerer_that_uses_it() { read(ablate(&dao, &mut rec, &mut model).unwrap()) }; for (split, r) in one(helped) { - assert!(r.drop.ref_gain_chrf > 10.0, "{split:?}: {}", r.line()); - assert!(r.drop.ref_gain_meaning > 0.0, "{split:?}: {}", r.line()); + assert!(r.drop.ref_gain_chrf.unwrap() > 10.0, "{split:?}: {}", r.line()); + assert!(r.drop.ref_gain_meaning.unwrap() > 0.0, "{split:?}: {}", r.line()); if r.permute.n > 0 { - assert!(r.permute.ref_gain_chrf > 10.0, "{split:?}: {}", r.line()); + assert!(r.permute.ref_gain_chrf.unwrap() > 10.0, "{split:?}: {}", r.line()); } assert_eq!(r.drop.moved, r.drop.n); } for (split, r) in one(blind) { - assert_eq!((r.drop.ref_gain_chrf, r.drop.ref_gain_meaning), (0.0, 0.0), "{split:?}"); + assert_eq!((r.drop.ref_gain_chrf, r.drop.ref_gain_meaning), (Some(0.0), Some(0.0)), "{split:?}"); } } @@ -3962,10 +3976,14 @@ fn a_stance_ablation_refuses_a_cycle_round_and_missing_round_answers() { dao.run.stance_ablation(2, &mut rec, &mut model, &Scorer::new(Bag)), Err(RunError::Malformed("round")) )); - assert!(matches!( - ablate(&dao, &mut rec, &mut model), - Err(RunError::Store) - )); + let Err(missing) = ablate(&dao, &mut rec, &mut model) else { + panic!("a missing round answer is refused") + }; + assert!(matches!(missing, RunError::Unrecorded)); + // It says what happened, not that a store would not read. + let said = missing.to_string(); + assert!(said.contains("holds no answer") && said.contains("digest") && said.contains("head inventory"), "{said}"); + assert!(!said.contains("would not read"), "{said}"); assert!( std::fs::read_dir(dao.root.join("round-1")) .unwrap() @@ -4008,3 +4026,86 @@ fn a_stance_ablation_refuses_a_cycle_round() { )); assert!(model.seen.is_empty() && rec.seen.is_empty()); } + +/// The real sentence for the item's own stance, an empty text (not a usable +/// lowering) for any other. +fn mute_unless_own(key: &str, text: &str) -> String { + let said = helped(key, text); + if said == "zzz qqq xxx" { String::new() } else { said } +} + +#[test] +fn unusable_variant_answers_are_counted_and_read_as_none_not_zero() { + let (dao, _) = recorded_dao(|_| {}); + let (mut rec, mut model) = (Lowerer::new(mute_unless_own), Lowerer::new(mute_unless_own)); + let reading = read(ablate(&dao, &mut rec, &mut model).unwrap()); + let (train, validation) = (&reading[0].1, &reading[1].1); + // Every variant answer is unusable: Train has one drop, Validation three + // drops and three permutes. The items themselves (replay) still read. + assert_eq!((train.n, train.unusable), (1, 1)); + assert_eq!((validation.n, validation.unusable), (3, 6)); + for r in [train, validation] { + assert!((r.replay.chrf.unwrap() - 100.0).abs() < 1e-9); + assert_eq!(r.drop.n, 0); + assert_eq!(r.permute.n, 0); + assert_eq!(r.drop, weksa::run::VariantReading::default()); + assert_eq!(r.permute, weksa::run::VariantReading::default()); + } + assert_eq!( + train.line(), + "ablation n=1 unusable=1 replay chrf=100.00 meaning=1.000 | drop n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none | permute n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none" + ); +} + +#[test] +fn the_line_names_each_variant_with_its_own_n_and_the_splits_n() { + use weksa::run::{AblationReading, Agreement, VariantReading}; + let reading = AblationReading { + n: 7, + unusable: 2, + replay: Agreement { chrf: Some(91.5), meaning: Some(0.9) }, + drop: VariantReading { + n: 5, + chrf: Some(80.25), + meaning: Some(0.8), + moved: 4, + ref_gain_chrf: Some(3.5), + ref_gain_meaning: Some(-0.125), + }, + permute: VariantReading { + n: 3, + chrf: Some(70.0), + meaning: None, + moved: 1, + ref_gain_chrf: None, + ref_gain_meaning: Some(0.5), + }, + }; + assert_eq!( + reading.line(), + "ablation n=7 unusable=2 replay chrf=91.50 meaning=0.900 | drop n=5 chrf=80.25 meaning=0.800 moved=4 ref_gain_chrf=3.50 ref_gain_meaning=-0.125 | permute n=3 chrf=70.00 meaning=none moved=1 ref_gain_chrf=none ref_gain_meaning=0.500" + ); +} + +fn nothing_usable(_: &str, _: &str) -> String { + String::new() +} + +#[test] +fn a_split_where_no_replay_is_usable_reads_none_throughout() { + let (dao, _) = recorded_dao(|_| {}); + let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(nothing_usable)); + let reading = read(ablate(&dao, &mut rec, &mut model).unwrap()); + // Train: the item and its drop; Validation: three items, three drops, three permutes. + assert_eq!((reading[0].1.n, reading[0].1.unusable), (0, 2)); + assert_eq!((reading[1].1.n, reading[1].1.unusable), (0, 9)); + for (_, r) in &reading { + assert_eq!(r.replay, weksa::run::Agreement::default()); + assert_eq!(r.drop, weksa::run::VariantReading::default()); + assert_eq!(r.permute, weksa::run::VariantReading::default()); + } + assert_eq!( + reading[0].1.line(), + "ablation n=0 unusable=2 replay chrf=none meaning=none | drop n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none | permute n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none" + ); +} -- 2.47.3