diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index 1a648a0..8e2cbfc 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -66,7 +66,7 @@ A feature is {feature, value}; value is {\"Grade\": 0 to 1000}, {\"Option\": lab /// Fixed per run by `PROMPT_SOURCE_DIGEST`. pub const LOWER_INSTRUCTIONS: &str = "\ Say the packet below in English (en-US), as one utterance of its speaker: all of its meaning and nothing it does not carry. -Every id is the inventory's listed below and every feature value is as listed. The stance is the speaker's felt affect and restraint: it shapes how the utterance is said, not what it says. +Every id is the inventory's listed below and every feature value is as listed; a grade is permille, 0 to 1000. The stance is the speaker's felt affect and restraint: it shapes how the utterance is said, not what it says. A referent with an entity is named by one of that entity's forms on the entities line. Answer with one JSON object and nothing else: {\"text\": \"...\"}. No code fence, no commentary."; diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 5b88c14..a92c010 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -11,11 +11,14 @@ use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn}; use weksa::entity::{Entity, EntityOrigin, LanguageForms, Pronunciation}; use weksa::inventory::{proof_revision, seed}; use weksa::packet::proof_packet; -use weksa::run::{CycleOutcome, LowererReach, RecallOutcome, RoundOutcome, RunPaths, RunStores, cycle, round}; +use weksa::model::RecordedAnswers; +use weksa::run::{ + AblationOutcome, CycleOutcome, LowererReach, RecallOutcome, RoundOutcome, RunPaths, RunStores, cycle, round, +}; use weksa::score::ScoreInput; use weksa::{ AnswerFiles, CorpusStore, EntityId, EntityStore, InventoryStore, OllamaEmbedder, PacketStore, - Scorer, SplitScheme, StoreError, lower, pronounce, validate, + Scorer, Split, SplitScheme, StoreError, lower, pronounce, validate, }; #[derive(Parser)] @@ -112,6 +115,19 @@ enum LoopCommand { /// half). Prompts and answers live in `/recall`; the command exits 3 /// until every prompt has an answer. A diagnostic, never a gate. Recall { root: PathBuf }, + /// Ask whether the lowerer uses a packet's stance: for each live Train and + /// Validation packet of round ROUND, ask again with the stance as it is, + /// emptied, and swapped for the next item's, and read how far the lowering + /// moves from the round's own answer. Prompts and answers live in + /// `/ablation-`; the command exits 3 until every prompt has an + /// answer. A diagnostic, never a gate; Test is never touched. + Ablate { + root: PathBuf, + #[arg(long)] + round: u32, + #[arg(long, default_value = "http://127.0.0.1:11434")] + ollama_url: String, + }, /// Run one edit-accept cycle of the run at ROOT. The stores are read only /// from the run's record. The editor's prompt, then the candidate round's /// encode prompts, then its lowering prompts are written in turn; the @@ -785,6 +801,36 @@ fn main() -> Result<()> { RoundOutcome::Recorded(r) => println!("{}", run.round_line(&r)), } } + Command::Loop(LoopCommand::Ablate { + root, + round, + ollama_url, + }) => { + let run = RunStores::open(&root)?; + let model = run.record().model.clone(); + let mut recorded = + RecordedAnswers::new(root.join(format!("round-{round}")), model.clone()); + let mut files = AnswerFiles::new(run.ablation_dir(round), model); + let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder)); + match run.stance_ablation(round, &mut recorded, &mut files, &scorer)? { + AblationOutcome::Unanswered { prompts } => { + for digest in &prompts { + println!("unanswered {digest}"); + } + println!("unanswered {} prompts", prompts.len()); + std::process::exit(3); + } + AblationOutcome::Read(readings) => { + for (split, reading) in readings { + let name = match split { + Split::Train => "train", + _ => "validation", + }; + println!("{name} {}", reading.line()); + } + } + } + } Command::Loop(LoopCommand::Recall { root }) => { let run = RunStores::open(&root)?; let mut files = AnswerFiles::new(run.recall_dir(), run.record().model.clone()); diff --git a/crates/weksa/src/model.rs b/crates/weksa/src/model.rs index 2f83176..1a79449 100644 --- a/crates/weksa/src/model.rs +++ b/crates/weksa/src/model.rs @@ -48,6 +48,37 @@ impl AnswerFiles { } } +/// Answers read from one directory and nothing else: `.answer` is the +/// reply to the prompt whose digest is ``. A prompt with no file is +/// [`PortError::Unanswered`]; this port never creates or writes anything. The +/// one reader of answer files: [`AnswerFiles`] reads through it. +pub struct RecordedAnswers { + dir: PathBuf, + model: String, +} + +impl RecordedAnswers { + pub fn new(dir: PathBuf, model: String) -> Self { + Self { dir, model } + } +} + +impl ModelPort for RecordedAnswers { + fn model(&self) -> &str { + &self.model + } + + fn answer(&mut self, prompt: &Prompt) -> Result { + match std::fs::read(self.dir.join(format!("{}.answer", prompt.digest()))) { + Ok(bytes) => { + String::from_utf8(bytes).map_err(|_| PortError::Io(ErrorKind::InvalidData)) + } + Err(e) if e.kind() == ErrorKind::NotFound => Err(PortError::Unanswered), + Err(e) => Err(PortError::Io(e.kind())), + } + } +} + impl ModelPort for AnswerFiles { fn model(&self) -> &str { &self.model @@ -56,12 +87,9 @@ impl ModelPort for AnswerFiles { fn answer(&mut self, prompt: &Prompt) -> Result { let io = |e: std::io::Error| PortError::Io(e.kind()); let digest = prompt.digest(); - match std::fs::read(self.dir.join(format!("{digest}.answer"))) { - Ok(bytes) => { - return String::from_utf8(bytes).map_err(|_| PortError::Io(ErrorKind::InvalidData)); - } - Err(e) if e.kind() == ErrorKind::NotFound => {} - Err(e) => return Err(io(e)), + match RecordedAnswers::new(self.dir.clone(), self.model.clone()).answer(prompt) { + Err(PortError::Unanswered) => {} + read => return read, } std::fs::create_dir_all(&self.dir).map_err(io)?; let path = self.dir.join(format!("{digest}.prompt")); diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 943894d..dbbf6ad 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -22,6 +22,9 @@ //! `ModelPort` (`Prompt::lower`), in two answer phases; the CLI's `lower` still //! uses the template table. The prompt source pin covers `context.rs`, //! `split.rs` and `lower.rs`. +//! +//! `RunStores::stance_ablation`, like `recall`, is a diagnostic in its own +//! directory: it records nothing and never writes its round's directory. use std::collections::{BTreeMap, BTreeSet}; use std::fmt; @@ -39,6 +42,7 @@ use crate::corpus::{CorpusItem, CorpusItemId, ModelFamily, Source}; use crate::encode::{EditDraft, LoweringDraft, PacketDraft, encoder_id}; use crate::form::{is_attestation, is_slug}; use crate::model::{ModelPort, PortError}; +use crate::packet::{Packet, Stance}; use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate}; use crate::split::{Split, SplitScheme, fnv1a64, fnv1a64_of, hex16}; use crate::store::{ @@ -451,6 +455,9 @@ pub enum RunError { NoFloor, /// A round has no Validation reading. NoValidation, + /// The round holds no answer to a lowering prompt the ablation rebuilt: the + /// prompt's digest drifted or the head inventory moved since the round. + Unrecorded, } impl fmt::Display for RunError { @@ -466,6 +473,9 @@ impl fmt::Display for RunError { RunError::NoInventory => f.write_str("the inventory holds no revision"), RunError::Io(kind) => write!(f, "io error {kind:?}"), RunError::Store => f.write_str("a store would not read or write"), + RunError::Unrecorded => f.write_str( + "the round holds no answer to a lowering prompt rebuilt now: the prompt digest drifted or the head inventory moved since the round", + ), RunError::Embed => f.write_str("the embedder failed"), RunError::NoFloor => f.write_str("the run has no noise floor: record two rounds first"), RunError::NoValidation => f.write_str("a reading has no validation item"), @@ -950,6 +960,301 @@ impl RunStores { } } +/// What the lowerer's answer did to each variant of a packet's stance, per +/// split. Means only; no key, text or stance value. +#[derive(Clone, Debug, PartialEq)] +pub struct AblationReading { + /// Items whose recorded and replayed lowerings were both usable. + pub n: u32, + /// Answers dropped as unusable, replay or variant. + pub unusable: u32, + /// Mean agreement of the replay with the recorded lowering: the noise + /// floor a variant is read against. + pub replay: Agreement, + pub drop: VariantReading, + pub permute: VariantReading, +} + +/// Means over an empty set are `None`, printed `none`: no measurement, never +/// a zero. +#[derive(Clone, Debug, PartialEq, Default)] +pub struct Agreement { + pub chrf: Option, + pub meaning: Option, +} + +/// One variant's reading: agreement with the recorded lowering, pairs whose +/// chrF fell below the replay's, and the mean fidelity the item's own stance +/// adds (`score(item, replay) - score(item, variant)`). +#[derive(Clone, Debug, PartialEq, Default)] +pub struct VariantReading { + pub n: u32, + pub chrf: Option, + pub meaning: Option, + pub moved: u32, + pub ref_gain_chrf: Option, + pub ref_gain_meaning: Option, +} + +fn show(x: Option, places: usize) -> String { + x.map_or("none".to_string(), |x| format!("{x:.places$}")) +} + +impl AblationReading { + pub fn line(&self) -> String { + let variant = |name: &str, v: &VariantReading| { + format!( + "{name} n={} chrf={} meaning={} moved={} ref_gain_chrf={} ref_gain_meaning={}", + v.n, + show(v.chrf, 2), + show(v.meaning, 3), + v.moved, + show(v.ref_gain_chrf, 2), + show(v.ref_gain_meaning, 3) + ) + }; + format!( + "ablation n={} unusable={} replay chrf={} meaning={} | {} | {}", + self.n, + self.unusable, + show(self.replay.chrf, 2), + show(self.replay.meaning, 3), + variant("drop", &self.drop), + variant("permute", &self.permute), + ) + } +} + +#[derive(Debug)] +pub enum AblationOutcome { + Unanswered { prompts: Vec }, + /// Train then Validation. + Read(Vec<(Split, AblationReading)>), +} + +/// What the port made of one lowering prompt. +enum Asked { + Text(String), + Unusable, + Waiting, +} + +fn asked(port: &mut dyn ModelPort, prompt: &Prompt) -> Result { + match port.answer(prompt) { + Ok(a) => Ok(LoweringDraft::parse(&a) + .ok() + .filter(LoweringDraft::usable) + .map_or(Asked::Unusable, |d| Asked::Text(d.text))), + Err(PortError::Io(ErrorKind::InvalidData)) => Ok(Asked::Unusable), + Err(PortError::Unanswered) => Ok(Asked::Waiting), + Err(PortError::Io(kind)) => Err(RunError::Io(kind)), + } +} + +fn mean(xs: impl Iterator) -> Option { + let (sum, n) = xs.fold((0.0, 0u32), |(s, n), x| (s + x, n + 1)); + (n > 0).then(|| sum / f64::from(n)) +} + +type Scored = (ItemScore, ItemScore); + +/// One item's reference text, recorded lowering, replay, and drop and permute +/// lowerings. +type Lowered = (String, String, String, [Option; 2]); + +impl RunStores { + /// `/ablation-`, where a stance ablation's prompts, answers and + /// variant packet stores live. + pub fn ablation_dir(&self, n: u32) -> PathBuf { + self.root.join(format!("ablation-{n}")) + } + + /// A diagnostic like [`recall`](Self::recall), never a gate: for each live + /// Train and Validation packet of plain round `n`, asks the lowerer again + /// (`model`, the replay), with the packet's stance emptied (drop) and with + /// the next item's stance in its split (permute), and reads how far the + /// lowering moves against the round's own answer (`recorded`). Test and + /// withdrawn items are never asked. Records nothing and writes nothing in + /// `round-`; the variant stores are rebuilt each call. An answer that + /// is not a usable lowering drops its pair and is counted in `unusable`. + pub fn stance_ablation( + &self, + n: u32, + recorded: &mut dyn ModelPort, + model: &mut dyn ModelPort, + scorer: &Scorer, + ) -> Result { + let store = |_| RunError::Store; + check_drift(self, model, scorer)?; + self.rounds() + .map_err(store)? + .into_iter() + .find(|r| r.n() == n && r.cycle().is_none()) + .ok_or(RunError::Malformed("round"))?; + let inventory = self.inventory.inventory().map_err(store)?; + + let dir = self.ablation_dir(n); + std::fs::create_dir_all(&dir).map_err(|e| RunError::Io(e.kind()))?; + for name in ["base", "drop", "permute"] { + for suffix in ["cc", "cc.lock"] { + let _ = std::fs::remove_file(dir.join(format!("{name}.{suffix}"))); + } + } + let mut stores = [ + PacketStore::open(&dir.join("base.cc")).map_err(store)?, + PacketStore::open(&dir.join("drop.cc")).map_err(store)?, + PacketStore::open(&dir.join("permute.cc")).map_err(store)?, + ]; + + // Live Train and Validation packets, in item key order within a split. + let round_packets = PacketStore::open(&self.root.join(format!("round-{n}")).join("packets.cc")) + .map_err(store)? + .all() + .map_err(store)?; + let mut splits: [(Split, Vec<(CorpusItem, Packet)>); 2] = + [(Split::Train, Vec::new()), (Split::Validation, Vec::new())]; + for packet in round_packets { + let item = self + .corpus + .get(packet.provenance.corpus_item.as_str()) + .map_err(store)? + .ok_or(RunError::Store)?; + for (split, held) in &mut splits { + if SplitScheme::V1.live(&item) == Some(*split) { + held.push((item.clone(), packet.clone())); + } + } + } + + let mut waiting: Vec = Vec::new(); + let mut readings = Vec::new(); + // Texts per split: (item text, recorded, replay, [drop, permute]). + let mut texts: Vec> = Vec::new(); + let mut unusable = [0u32; 2]; + for (at, (_, held)) in splits.iter_mut().enumerate() { + held.sort_by(|a, b| a.0.id.as_str().cmp(b.0.id.as_str())); + let mut rows = Vec::new(); + for (i, (item, packet)) in held.iter().enumerate() { + let packet_item = packet.provenance.corpus_item.clone(); + let mut prompt_of = |slot: usize, packet: Packet| -> Result { + let view = stores[slot] + .admit(packet, &self.inventory, &self.corpus, &self.entities) + .map_err(|_| RunError::Store)?; + Prompt::lower(&view, &packet_item, &inventory, &self.entities) + .map_err(|_| RunError::Store) + }; + let base = prompt_of(0, packet.clone())?; + let mut variants: [Option; 2] = [None, None]; + if !packet.stance.features.is_empty() { + let mut p = packet.clone(); + p.stance = Stance::default(); + variants[0] = Some(prompt_of(1, p)?); + } + if held.len() > 1 { + let mut p = packet.clone(); + p.stance = held[(i + 1) % held.len()].1.stance.clone(); + let prompt = prompt_of(2, p)?; + if prompt.digest() != base.digest() { + variants[1] = Some(prompt); + } + } + // The round's own answer must exist; a missing one is not a gap to ask for. + let recorded_text = match asked(recorded, &base)? { + Asked::Waiting => return Err(RunError::Unrecorded), + other => other, + }; + let replay = asked(model, &base)?; + let mut got: [Option; 2] = [None, None]; + for (slot, prompt) in got.iter_mut().zip(&variants) { + let Some(prompt) = prompt else { continue }; + match asked(model, prompt)? { + Asked::Text(t) => *slot = Some(t), + Asked::Unusable => unusable[at] += 1, + Asked::Waiting => { + let d = prompt.digest(); + if !waiting.contains(&d) { + waiting.push(d); + } + } + } + } + match (recorded_text, replay) { + (Asked::Text(b), Asked::Text(r)) => { + rows.push((item.text.clone(), b, r, got)); + } + (_, Asked::Waiting) => { + let d = base.digest(); + if !waiting.contains(&d) { + waiting.push(d); + } + } + _ => unusable[at] += 1, + } + } + texts.push(rows); + } + if !waiting.is_empty() { + return Ok(AblationOutcome::Unanswered { prompts: waiting }); + } + + // One score call: for each item, the replay then each variant, each + // against the recorded lowering (agreement) and the item (fidelity). + let mut inputs: Vec = Vec::new(); + for (reference, base, replay, got) in texts.iter().flatten() { + for output in std::iter::once(replay).chain(got.iter().flatten()) { + inputs.push(ScoreInput { reference: base, output }); + inputs.push(ScoreInput { reference, output }); + } + } + let mut scores = if inputs.is_empty() { + Vec::new() + } else { + scorer.score(&inputs).map_err(|_| RunError::Embed)? + } + .into_iter(); + let mut pair = || -> Result { + Ok((scores.next().ok_or(RunError::Store)?, scores.next().ok_or(RunError::Store)?)) + }; + for (at, ((split, _), rows)) in splits.iter().zip(&texts).enumerate() { + let mut replay: Vec = Vec::new(); + let mut variants: [Vec<(Scored, Scored)>; 2] = [Vec::new(), Vec::new()]; + for (_, _, _, got) in rows { + let r = pair()?; + for (k, g) in got.iter().enumerate() { + if g.is_some() { + variants[k].push((r.clone(), pair()?)); + } + } + replay.push(r); + } + let variant = |v: &[(Scored, Scored)]| VariantReading { + n: v.len() as u32, + chrf: mean(v.iter().map(|(_, x)| x.0.chrf)), + meaning: mean(v.iter().filter_map(|(_, x)| x.0.meaning)), + moved: v.iter().filter(|(r, x)| x.0.chrf < r.0.chrf).count() as u32, + ref_gain_chrf: mean(v.iter().map(|(r, x)| r.1.chrf - x.1.chrf)), + ref_gain_meaning: mean( + v.iter().filter_map(|(r, x)| Some(r.1.meaning? - x.1.meaning?)), + ), + }; + readings.push(( + *split, + AblationReading { + n: rows.len() as u32, + unusable: unusable[at], + replay: Agreement { + chrf: mean(replay.iter().map(|r| r.0.chrf)), + meaning: mean(replay.iter().filter_map(|r| r.0.meaning)), + }, + drop: variant(&variants[0]), + permute: variant(&variants[1]), + }, + )); + } + Ok(AblationOutcome::Read(readings)) + } +} + /// The distinct rule codes an admission refusal broke: validator rules by /// code, a store refusal by its variant name. Nothing from the packet or the /// stores is kept; an I/O failure is `io`. @@ -1008,7 +1313,7 @@ pub fn round( } /// The model and embedder the run pinned are the ones answering. -fn check_drift( +fn check_drift( run: &RunStores, model: &M, scorer: &Scorer, diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 4113a82..f5562cd 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -1633,3 +1633,90 @@ fn loop_open_requires_the_lowerer_reach_and_pins_it() { weksa::run::LowererReach::BriefedSubagent ); } + +/// Answers the Dao encode prompts with the Dao packets and every lowering with +/// one sentence. +struct DaoModel; + +impl weksa::model::ModelPort for DaoModel { + fn model(&self) -> &str { + "fake-model" + } + fn answer(&mut self, prompt: &weksa::context::Prompt) -> Result { + if prompt.text().starts_with(weksa::context::LOWER_INSTRUCTIONS) { + return Ok(r#"{"text":"The highest excellence is like the water."}"#.into()); + } + let n: usize = prompt.item_keys()[0].rsplit(':').next().unwrap().parse().unwrap(); + Ok(serde_json::to_string(&weksa::PacketDraft::of(&weksa::daodejing::packets()[n - 1])).unwrap()) + } +} + +#[test] +fn loop_ablate_waits_then_prints_one_line_per_split() { + let d = tempfile::tempdir().unwrap(); + open_dao_run(d.path()); + let url = fake_embedder(); + let root = d.path().join("canaryroot"); + + // Record round 1 the way `loop round` does, then put its lowering answers + // where the relay would have. + let mut run = weksa::run::RunStores::open(&root).unwrap(); + let scorer = weksa::Scorer::new(weksa::OllamaEmbedder::new(&url, &run.record().embedder)); + let weksa::run::RoundOutcome::Recorded(r) = weksa::run::round(&mut run, &mut DaoModel, &scorer).unwrap() + else { + panic!("answered") + }; + for call in &r.calls()[3..] { + std::fs::write( + root.join(format!("round-1/{}.answer", call.prompt)), + r#"{"text":"The highest excellence is like the water."}"#, + ) + .unwrap(); + } + + let ablate = || weksa(d.path(), &["loop", "ablate", "canaryroot", "--round", "1", "--ollama-url", &url]); + let first = ablate(); + assert_eq!(first.status.code(), Some(3), "{}", out(&first)); + let text = out(&first); + let digests: Vec<&str> = text + .lines() + .filter_map(|l| l.strip_prefix("unanswered ")) + .filter(|d| d.len() == 16) + .collect(); + // Three replays, three drops and the Validation permutes (2 items). + assert_eq!(digests.len(), 3 + 3 + 2, "{text}"); + for digest in &digests { + assert!(root.join(format!("ablation-1/{digest}.prompt")).is_file()); + std::fs::write( + root.join(format!("ablation-1/{digest}.answer")), + r#"{"text":"The highest excellence is like the water."}"#, + ) + .unwrap(); + } + let second = ablate(); + assert!(second.status.success(), "{}", out(&second)); + let text = out(&second); + assert_eq!( + text, + "train ablation n=1 unusable=0 replay chrf=100.00 meaning=1.000 | drop n=1 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000 | permute n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none\nvalidation ablation n=2 unusable=0 replay chrf=100.00 meaning=1.000 | drop n=2 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000 | permute n=2 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000\n" + ); + for leaked in ["item:", "packet:", "highest excellence", "canaryroot"] { + assert!(!text.contains(leaked), "{text}"); + } +} + +#[test] +fn loop_ablate_without_the_rounds_answers_says_so() { + let d = tempfile::tempdir().unwrap(); + open_dao_run(d.path()); + let url = fake_embedder(); + let root = d.path().join("canaryroot"); + let mut run = weksa::run::RunStores::open(&root).unwrap(); + let scorer = weksa::Scorer::new(weksa::OllamaEmbedder::new(&url, &run.record().embedder)); + weksa::run::round(&mut run, &mut DaoModel, &scorer).unwrap(); + let o = weksa(d.path(), &["loop", "ablate", "canaryroot", "--round", "1", "--ollama-url", &url]); + assert!(!o.status.success()); + let text = out(&o); + assert!(text.contains("holds no answer") && text.contains("head inventory"), "{text}"); + assert!(!text.contains("would not read"), "{text}"); +} diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs index d6e2efe..5979a26 100644 --- a/crates/weksa/tests/context.rs +++ b/crates/weksa/tests/context.rs @@ -630,6 +630,7 @@ mod lowering { for needed in [ "en-US", "all of its meaning and nothing it does not carry", + "a grade is permille, 0 to 1000", "felt affect and restraint", "shapes how the utterance is said, not what it says", "forms on the entities line", @@ -640,6 +641,61 @@ mod lowering { // The one call it makes is its item; the key is in no text. assert_eq!(prompt.item_keys(), [item.as_str()]); assert!(!prompt.text().contains(item.as_str())); + // The view's speaker and group are private: the item's speaker entity + // id and its group are in no text. + let held = CorpusStore::open(&data("daodejing/corpus.cc")) + .unwrap() + .get(item.as_str()) + .unwrap() + .unwrap(); + assert!(!held.group.is_empty()); + for private in [held.speaker.as_str(), held.group.as_str()] { + assert!(!prompt.text().contains(private), "{private}"); + } + } + + #[test] + fn lower_prompt_moves_with_the_stance_alone() { + use weksa::packet::{FeatValue, FeatureValue, Grade, Stance}; + let (_, inv, entities, item) = dao(2); + let store = InventoryStore::open(&data("daodejing/inventory.cc")).unwrap(); + let own = packets().remove(2).stance; + let grade = |feature: &str, v: u16| FeatureValue { + feature: feature.parse().unwrap(), + value: FeatValue::Grade(Grade::new(v).unwrap()), + }; + let angry = Stance { + features: vec![grade("feature:9", 800), grade("feature:18", 900)], + }; + let prompt_of = |stance: Stance| { + let mut packet = packets().remove(2); + packet.stance = stance.clone(); + let view = admitted(packet, &store, "daodejing/corpus.cc", &entities); + let prompt = Prompt::lower(&view, &item, &inv, &entities).unwrap(); + (stance, prompt) + }; + let three = [prompt_of(own), prompt_of(angry), prompt_of(Stance::default())]; + + // Three stances, three prompts. + let digests: std::collections::BTreeSet = + three.iter().map(|(_, p)| p.digest()).collect(); + assert_eq!(digests.len(), 3); + // Each packet line ends with exactly that stance's JSON, and the prompt + // with the packet line taken out is the same text for all three. + let without_packet_line = |text: &str| -> String { + text.lines().filter(|l| !l.starts_with("packet: ")).collect::>().join("\n") + }; + let bare = without_packet_line(three[0].1.text()); + for (stance, prompt) in &three { + let line = prompt.text().lines().find(|l| l.starts_with("packet: ")).unwrap(); + let json = serde_json::to_string(stance).unwrap(); + assert!(line.ends_with(&format!(r#","stance":{json}}}"#)), "{line}"); + assert_eq!(without_packet_line(prompt.text()), bare); + } + // The stance's feature ids are glossed in the one inventory listing. + let text = three[1].1.text(); + assert!(text.contains("\nfeature:9: anger [grade; slots Stance]\n")); + assert!(text.contains("\nfeature:18: restraint [grade; slots Stance]\n")); } #[test] diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 0929e53..4a1c7ee 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -2598,9 +2598,19 @@ fn source_tripwires() { let count = |needle: &str| code.iter().filter(|l| l.contains(needle)).count(); assert!(!run.contains("pub fn new")); // The round does not call the template table: the model lowers. - assert_eq!(count("lower(&"), 0, "run.rs calls the table"); + assert_eq!( + code.iter() + .filter(|l| l.contains("lower(&") && !l.contains("Prompt::lower(")) + .count(), + 0, + "run.rs calls the table" + ); assert_eq!(count("crate::lower::lower"), 0, "run.rs imports the table"); - assert_eq!(count("Prompt::lower("), 1, "one lowering call site"); + assert_eq!( + count("Prompt::lower("), + 2, + "two lowering call sites: the round's and the stance ablation's replay of it" + ); // Prompt::lower takes the view, the item's key, the inventory and the entity // port: no corpus item, train set, offer or store. let start = context.find("pub fn lower(").unwrap(); @@ -3560,3 +3570,542 @@ fn own_family_counts_offered_items_of_the_answerer_family() { assert_eq!(mixed_family_round(ModelFamily::Google), 0); assert_eq!(mixed_family_round(ModelFamily::Unrecorded), 0); } + +// ---- stance ablation (stance-to-lowerer) ---- + +/// A lowerer by function: the text it answers for an item key and the prompt +/// text, recording every prompt it was shown. +struct Lowerer { + f: fn(&str, &str) -> String, + seen: Vec<(String, String, String)>, +} + +impl Lowerer { + fn new(f: fn(&str, &str) -> String) -> Self { + Self { f, seen: vec![] } + } + /// The digests asked, once each, in the order first asked. + fn digests(&self) -> Vec { + let mut out: Vec = Vec::new(); + for (d, ..) in &self.seen { + if !out.contains(d) { + out.push(d.clone()); + } + } + out + } +} + +impl ModelPort for Lowerer { + fn model(&self) -> &str { + "fake-model" + } + fn answer(&mut self, prompt: &Prompt) -> Result { + let key = prompt.item_keys()[0].clone(); + self.seen.push((prompt.digest(), key.clone(), prompt.text().to_string())); + Ok(serde_json::json!({ "text": (self.f)(&key, prompt.text()) }).to_string()) + } +} + +const WRITTEN: &str = "item:written:1"; + +fn english(key: &str) -> String { + let mut table = table_english(); + table.insert( + WRITTEN.into(), + "Heaven endures through the long years without end.".into(), + ); + table[key].clone() +} + +/// The packet line's stance, as the prompt prints it: the packet line is one +/// JSON object ending in the stance. +fn stance_of(text: &str) -> String { + let line = text.lines().find(|l| l.starts_with("packet: ")).unwrap(); + line[line.find("\"stance\":").unwrap()..].to_string() +} + +/// Says the same thing whatever the stance. +fn blind(key: &str, _text: &str) -> String { + english(key) +} + +/// Appends the stance it was shown. +fn reader(key: &str, text: &str) -> String { + format!("{} {}", english(key), stance_of(text)) +} + +/// Does better with the item's own stance: the real sentence for it, nonsense +/// for any other, empty included. +fn helped(key: &str, text: &str) -> String { + let r800 = r#""stance":{"features":[{"feature":"feature:18","value":{"Grade":800}}]}}"#; + let own = match key { + "item:daodejing:3" => r#""stance":{"features":[{"feature":"feature:18","value":{"Grade":800}},{"feature":"feature:16","value":{"Grade":300}}]}}"#, + WRITTEN => r#""stance":{"features":[{"feature":"feature:18","value":{"Grade":500}}]}}"#, + _ => r800, + }; + if stance_of(text) == own { english(key) } else { "zzz qqq xxx".into() } +} + +fn grade_stance(features: &[(&str, u16)]) -> weksa::packet::Stance { + weksa::packet::Stance { + features: features + .iter() + .map(|(f, v)| weksa::packet::FeatureValue { + feature: f.parse().unwrap(), + value: weksa::packet::FeatValue::Grade(common::grade(*v)), + }) + .collect(), + } +} + +/// A Dao run with a third Validation item (`item:written:1`, key order after +/// items 2 and 3) and round 1 recorded over it, every item encoded to its Dao +/// packet. Stances: item 1 restraint 800; item 2 restraint 800; item 3 +/// restraint 800 and affection 300; the written item restraint 500. +fn recorded_dao(edit: impl FnOnce(&mut HashMap)) -> (Dao, RoundRecord) { + let mut dao = dao_over(|corpus, entities| { + let entities = weksa::EntityStore::open(entities).unwrap(); + let mut store = CorpusStore::open(corpus).unwrap(); + let mut written = store.get("item:daodejing:2").unwrap().unwrap(); + written.id = WRITTEN.into(); + written.text = "Heaven endures through the long years without end.".into(); + written.source = weksa::corpus::Source::OperatorWritten; + store.add(written, &entities).unwrap(); + }); + let mut model = fake(&good_answers()); + let mut extra = draft_of(2); + extra.referents.truncate(1); + extra.predications.truncate(1); + extra.stance = grade_stance(&[("feature:18", 500)]); + model.answers.insert(WRITTEN.into(), json(&extra)); + edit(&mut model.answers); + model.lowered.insert(WRITTEN.into(), english(WRITTEN)); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + (dao, r) +} + +fn ablate( + dao: &Dao, + recorded: &mut dyn ModelPort, + model: &mut dyn ModelPort, +) -> Result { + dao.run.stance_ablation(1, recorded, model, &Scorer::new(Bag)) +} + +fn read( + outcome: weksa::run::AblationOutcome, +) -> Vec<(weksa::Split, weksa::run::AblationReading)> { + match outcome { + weksa::run::AblationOutcome::Read(r) => r, + weksa::run::AblationOutcome::Unanswered { prompts } => panic!("unanswered {prompts:?}"), + } +} + +#[test] +fn recorded_answers_read_and_never_write() { + use std::io::ErrorKind; + use weksa::model::RecordedAnswers; + let p = prompt(); + let dir = tempfile::tempdir().unwrap(); + let mut port = RecordedAnswers::new(dir.path().to_path_buf(), "answerer".into()); + assert_eq!(port.model(), "answerer"); + // A miss is Unanswered and the directory is as it was. + assert_eq!(port.answer(&p), Err(PortError::Unanswered)); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0); + // A directory that does not exist is a miss too, and is not made. + let mut absent = RecordedAnswers::new(dir.path().join("nope"), "m".into()); + assert_eq!(absent.answer(&p), Err(PortError::Unanswered)); + assert!(!dir.path().join("nope").exists()); + // A hit is the file's text as written. + let file = dir.path().join(format!("{}.answer", p.digest())); + std::fs::write(&file, " {\"x\": 1}\n").unwrap(); + assert_eq!(port.answer(&p).unwrap(), " {\"x\": 1}\n"); + std::fs::write(&file, [0xff, 0xfe]).unwrap(); + assert_eq!(port.answer(&p), Err(PortError::Io(ErrorKind::InvalidData))); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1); +} + +#[test] +fn a_stance_ablation_asks_replay_drop_and_permute_for_train_and_validation_only() { + let (dao, r) = recorded_dao(|_| {}); + let round_digests: Vec = r.calls()[4..].iter().map(|c| c.prompt.clone()).collect(); + assert_eq!(round_digests.len(), 4, "the round lowered four items"); + let mut recorded_port = Lowerer::new(reader); + let mut model = Lowerer::new(reader); + read(ablate(&dao, &mut recorded_port, &mut model).unwrap()); + + // The base prompts are the round's own, byte for byte. + let recorded_digests = recorded_port.digests(); + assert_eq!(recorded_digests, round_digests); + let asked = model.digests(); + for d in &round_digests { + assert!(asked.contains(d), "the replay asks the round's prompt {d}"); + } + // Per item: replay and drop for each; permute for each Validation item, and + // none for the lone Train item. + let per_key = |key: &str| model.seen.iter().filter(|s| s.1 == key).count(); + assert_eq!(per_key("item:daodejing:1"), 2); + for key in ["item:daodejing:2", "item:daodejing:3", WRITTEN] { + assert_eq!(per_key(key), 3, "{key}"); + } + // Test items are never asked. + for (_, key, text) in &model.seen { + assert!(key != "item:daodejing:4" && key != "item:daodejing:5", "{key}"); + assert!(!text.contains("item:daodejing:4")); + } + // A drop prompt prints the empty stance, a replay the item's own. + let empty = model + .seen + .iter() + .filter(|s| stance_of(&s.2) == "\"stance\":{\"features\":[]}}") + .count(); + assert_eq!(empty, 4, "one drop per item"); +} + +#[test] +fn a_lowerer_blind_to_stance_reads_no_movement_and_one_that_reads_it_does() { + let (dao, _) = recorded_dao(|_| {}); + let one = |f: fn(&str, &str) -> String| { + let (mut rec, mut model) = (Lowerer::new(f), Lowerer::new(f)); + read(ablate(&dao, &mut rec, &mut model).unwrap()) + }; + for (split, r) in one(blind) { + assert_eq!(r.unusable, 0); + assert_eq!((r.drop.moved, r.permute.moved), (0, 0), "{split:?}: {}", r.line()); + assert!((r.replay.chrf.unwrap() - 100.0).abs() < 1e-9); + assert!((r.drop.chrf.unwrap() - r.replay.chrf.unwrap()).abs() < 1e-9); + if r.permute.n > 0 { + assert!((r.permute.chrf.unwrap() - r.replay.chrf.unwrap()).abs() < 1e-9); + } + assert_eq!(r.drop.ref_gain_chrf, Some(0.0)); + } + let reading = one(reader); + let (train, validation) = (&reading[0].1, &reading[1].1); + assert_eq!((reading[0].0, reading[1].0), (weksa::Split::Train, weksa::Split::Validation)); + assert_eq!((train.n, validation.n), (1, 3)); + assert_eq!((train.drop.n, train.drop.moved), (1, 1)); + assert_eq!((train.permute.n, train.permute.moved), (0, 0)); + assert_eq!((validation.drop.n, validation.drop.moved), (3, 3)); + assert_eq!((validation.permute.n, validation.permute.moved), (3, 3)); + // A lone Train item has no permute: no measurement, not total disagreement. + assert_eq!( + train.permute, + weksa::run::VariantReading::default(), + "{}", + train.line() + ); + assert!( + train.line().contains( + "| permute n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none" + ), + "{}", + train.line() + ); + assert!((validation.replay.chrf.unwrap() - 100.0).abs() < 1e-9); + assert!(validation.drop.chrf.unwrap() < validation.replay.chrf.unwrap()); + assert!(validation.permute.chrf.unwrap() < validation.replay.chrf.unwrap()); +} + +#[test] +fn the_item_s_own_stance_helps_fidelity_only_for_a_lowerer_that_uses_it() { + let (dao, _) = recorded_dao(|_| {}); + let one = |f: fn(&str, &str) -> String| { + let (mut rec, mut model) = (Lowerer::new(f), Lowerer::new(f)); + read(ablate(&dao, &mut rec, &mut model).unwrap()) + }; + for (split, r) in one(helped) { + assert!(r.drop.ref_gain_chrf.unwrap() > 10.0, "{split:?}: {}", r.line()); + assert!(r.drop.ref_gain_meaning.unwrap() > 0.0, "{split:?}: {}", r.line()); + if r.permute.n > 0 { + assert!(r.permute.ref_gain_chrf.unwrap() > 10.0, "{split:?}: {}", r.line()); + } + assert_eq!(r.drop.moved, r.drop.n); + } + for (split, r) in one(blind) { + assert_eq!((r.drop.ref_gain_chrf, r.drop.ref_gain_meaning), (Some(0.0), Some(0.0)), "{split:?}"); + } +} + +#[test] +fn a_packet_without_stance_asks_no_drop_and_a_test_packet_in_the_round_is_never_asked() { + let (dao, _) = recorded_dao(|answers| { + let mut d = draft_of(3); + d.stance = Default::default(); + answers.insert("item:daodejing:3".into(), json(&d)); + }); + // A packet for a Test item, put into the round's own store the way no round + // would: the ablation reads splits from the corpus, not from the store. + { + let corpus = CorpusStore::open(&dao.corpus).unwrap(); + let inv = dao.run.inventory(); + let mut packet = packets().remove(3); + packet.inventory_rev = 3; + PacketStore::open(&dao.root.join("round-1/packets.cc")) + .unwrap() + .admit(packet, inv, &corpus, dao.run.entities()) + .unwrap(); + } + let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(reader)); + read(ablate(&dao, &mut rec, &mut model).unwrap()); + let bare = "\"stance\":{\"features\":[]}}"; + let drops_of_3 = model + .seen + .iter() + .filter(|s| s.1 == "item:daodejing:3" && stance_of(&s.2) == bare) + .count(); + // Its replay already shows the empty stance; there is no drop beside it. + assert_eq!(drops_of_3, 1); + assert!(model.seen.iter().all(|s| s.1 != "item:daodejing:4"), "a Test item is never asked"); + assert!(rec.seen.iter().all(|s| s.1 != "item:daodejing:4")); +} + +#[test] +fn permute_takes_the_next_items_stance_in_its_split_and_skips_equal_ones() { + let (dao, _) = recorded_dao(|_| {}); + let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(reader)); + read(ablate(&dao, &mut rec, &mut model).unwrap()); + // The stance each Validation item's permute prompt shows: the next item's + // in key order (2, 3, written:1), cyclic; item 1 is alone in Train. + let stance = |text: &str| stance_of(text); + let of = |key: &str| -> Vec { + model + .seen + .iter() + .filter(|s| s.1 == key) + .map(|s| stance(&s.2)) + .collect() + }; + let (s2, s3, sw) = ( + stance(&rec.seen.iter().find(|s| s.1 == "item:daodejing:2").unwrap().2), + stance(&rec.seen.iter().find(|s| s.1 == "item:daodejing:3").unwrap().2), + stance(&rec.seen.iter().find(|s| s.1 == WRITTEN).unwrap().2), + ); + assert!(s2 != s3 && s3 != sw && sw != s2); + assert!(of("item:daodejing:2").contains(&s3)); + assert!(of("item:daodejing:3").contains(&sw)); + assert!(of(WRITTEN).contains(&s2), "the last wraps to the first"); + assert!(!of("item:daodejing:2").contains(&sw), "not the previous or the other"); + // Item 1 is alone in Train: it never takes a stance across splits. + assert_eq!(of("item:daodejing:1").len(), 2, "replay and drop only"); + + // Items 2 and 3 with equal stances: item 2's permute is its own prompt, so + // it asks none; the others still do. + let (dao, _) = recorded_dao(|answers| { + let mut d = draft_of(3); + d.stance = draft_of(2).stance; + answers.insert("item:daodejing:3".into(), json(&d)); + }); + let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(reader)); + let r = read(ablate(&dao, &mut rec, &mut model).unwrap()); + let per_key = |key: &str| model.seen.iter().filter(|s| s.1 == key).count(); + assert_eq!(per_key("item:daodejing:2"), 2, "replay and drop: its next holds the stance it has"); + assert_eq!(per_key("item:daodejing:3"), 3); + assert_eq!(per_key(WRITTEN), 3); + assert_eq!(r[1].1.permute.n, 2, "{}", r[1].1.line()); +} + +#[test] +fn a_stance_ablation_records_nothing_and_writes_nothing_in_its_round() { + use weksa::model::RecordedAnswers; + let (dao, r) = recorded_dao(|_| {}); + let round1 = dao.root.join("round-1"); + for c in &r.calls()[4..] { + std::fs::write( + round1.join(format!("{}.answer", c.prompt)), + serde_json::json!({ "text": english(&c.items[0]) }).to_string(), + ) + .unwrap(); + } + let snapshot = |dir: &Path| -> Vec<(String, Vec)> { + let mut files: Vec<_> = std::fs::read_dir(dir) + .unwrap() + .map(|e| { + let e = e.unwrap(); + let bytes = if e.path().is_dir() { vec![] } else { std::fs::read(e.path()).unwrap() }; + (e.file_name().to_string_lossy().into_owned(), bytes) + }) + .collect(); + files.sort(); + files.retain(|f| f.0 != "ablation-1"); + files + }; + let before = (snapshot(&round1), snapshot(&dao.root), dao.run.rounds().unwrap(), dao.run.decisions().unwrap()); + let go = || { + let mut rec = RecordedAnswers::new(round1.clone(), "fake-model".into()); + let mut files = weksa::model::AnswerFiles::new(dao.run.ablation_dir(1), "fake-model".into()); + // Nothing answers yet: the replay and variant prompts are written. + dao.run.stance_ablation(1, &mut rec, &mut files, &Scorer::new(Bag)) + }; + let weksa::run::AblationOutcome::Unanswered { prompts } = go().unwrap() else { + panic!("nobody answered") + }; + // Four replays (the round's own prompts), four drops, three permutes (item 1 + // is alone in Train). + assert_eq!(prompts.len(), 4 + 4 + 3, "{prompts:?}"); + for d in &prompts { + assert!(dao.run.ablation_dir(1).join(format!("{d}.prompt")).is_file()); + } + // Answer every ask from a fake and read; then again over the same files. + for d in &prompts { + let text = std::fs::read_to_string(dao.run.ablation_dir(1).join(format!("{d}.prompt"))).unwrap(); + std::fs::write( + dao.run.ablation_dir(1).join(format!("{d}.answer")), + serde_json::json!({ "text": format!("answer {}", &text[..8]) }).to_string(), + ) + .unwrap(); + } + let first = read(go().unwrap()); + let again = read(go().unwrap()); + assert_eq!(first, again); + let after = (snapshot(&round1), snapshot(&dao.root), dao.run.rounds().unwrap(), dao.run.decisions().unwrap()); + assert!(before.0 == after.0, "round-1 is unchanged byte for byte"); + assert!(before.1 == after.1, "the run root's files are unchanged"); + assert_eq!(before.2, after.2); + assert_eq!(before.3, after.3); +} + +#[test] +fn a_stance_ablation_refuses_a_cycle_round_and_missing_round_answers() { + use weksa::model::RecordedAnswers; + let (dao, _) = recorded_dao(|_| {}); + // No round-2, and round 1 holds no answer files. + let mut model = Lowerer::new(reader); + let mut rec = RecordedAnswers::new(dao.root.join("round-1"), "fake-model".into()); + assert!(matches!( + dao.run.stance_ablation(2, &mut rec, &mut model, &Scorer::new(Bag)), + Err(RunError::Malformed("round")) + )); + let Err(missing) = ablate(&dao, &mut rec, &mut model) else { + panic!("a missing round answer is refused") + }; + assert!(matches!(missing, RunError::Unrecorded)); + // It says what happened, not that a store would not read. + let said = missing.to_string(); + assert!(said.contains("holds no answer") && said.contains("digest") && said.contains("head inventory"), "{said}"); + assert!(!said.contains("would not read"), "{said}"); + assert!( + std::fs::read_dir(dao.root.join("round-1")) + .unwrap() + .all(|e| !e.unwrap().file_name().to_string_lossy().ends_with(".prompt")), + "no prompt is written into the round" + ); + // Drift is refused as a round refuses it. + struct Named(Lowerer); + impl ModelPort for Named { + fn model(&self) -> &str { + "other-model" + } + fn answer(&mut self, p: &Prompt) -> Result { + self.0.answer(p) + } + } + let mut named = Named(Lowerer::new(reader)); + let mut rec2 = Lowerer::new(reader); + assert!(matches!( + ablate(&dao, &mut rec2, &mut named), + Err(RunError::Drift(DriftOf::Model)) + )); + assert!(matches!( + dao.run.stance_ablation(1, &mut rec2, &mut Lowerer::new(reader), &Scorer::new(OtherEmbedder)), + Err(RunError::Drift(DriftOf::Embedder)) + )); +} + +#[test] +fn a_stance_ablation_refuses_a_cycle_round() { + let mut dao = dao(); + two_rounds(&mut dao); + run_cycle(&mut dao, &mut cycle_fake(ONE_CONCEPT)); + let rounds = dao.run.rounds().unwrap(); + let candidate = rounds.iter().find(|r| r.cycle().is_some()).expect("the cycle's round"); + let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(reader)); + assert!(matches!( + dao.run.stance_ablation(candidate.n(), &mut rec, &mut model, &Scorer::new(Bag)), + Err(RunError::Malformed("round")) + )); + assert!(model.seen.is_empty() && rec.seen.is_empty()); +} + +/// The real sentence for the item's own stance, an empty text (not a usable +/// lowering) for any other. +fn mute_unless_own(key: &str, text: &str) -> String { + let said = helped(key, text); + if said == "zzz qqq xxx" { String::new() } else { said } +} + +#[test] +fn unusable_variant_answers_are_counted_and_read_as_none_not_zero() { + let (dao, _) = recorded_dao(|_| {}); + let (mut rec, mut model) = (Lowerer::new(mute_unless_own), Lowerer::new(mute_unless_own)); + let reading = read(ablate(&dao, &mut rec, &mut model).unwrap()); + let (train, validation) = (&reading[0].1, &reading[1].1); + // Every variant answer is unusable: Train has one drop, Validation three + // drops and three permutes. The items themselves (replay) still read. + assert_eq!((train.n, train.unusable), (1, 1)); + assert_eq!((validation.n, validation.unusable), (3, 6)); + for r in [train, validation] { + assert!((r.replay.chrf.unwrap() - 100.0).abs() < 1e-9); + assert_eq!(r.drop.n, 0); + assert_eq!(r.permute.n, 0); + assert_eq!(r.drop, weksa::run::VariantReading::default()); + assert_eq!(r.permute, weksa::run::VariantReading::default()); + } + assert_eq!( + train.line(), + "ablation n=1 unusable=1 replay chrf=100.00 meaning=1.000 | drop n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none | permute n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none" + ); +} + +#[test] +fn the_line_names_each_variant_with_its_own_n_and_the_splits_n() { + use weksa::run::{AblationReading, Agreement, VariantReading}; + let reading = AblationReading { + n: 7, + unusable: 2, + replay: Agreement { chrf: Some(91.5), meaning: Some(0.9) }, + drop: VariantReading { + n: 5, + chrf: Some(80.25), + meaning: Some(0.8), + moved: 4, + ref_gain_chrf: Some(3.5), + ref_gain_meaning: Some(-0.125), + }, + permute: VariantReading { + n: 3, + chrf: Some(70.0), + meaning: None, + moved: 1, + ref_gain_chrf: None, + ref_gain_meaning: Some(0.5), + }, + }; + assert_eq!( + reading.line(), + "ablation n=7 unusable=2 replay chrf=91.50 meaning=0.900 | drop n=5 chrf=80.25 meaning=0.800 moved=4 ref_gain_chrf=3.50 ref_gain_meaning=-0.125 | permute n=3 chrf=70.00 meaning=none moved=1 ref_gain_chrf=none ref_gain_meaning=0.500" + ); +} + +fn nothing_usable(_: &str, _: &str) -> String { + String::new() +} + +#[test] +fn a_split_where_no_replay_is_usable_reads_none_throughout() { + let (dao, _) = recorded_dao(|_| {}); + let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(nothing_usable)); + let reading = read(ablate(&dao, &mut rec, &mut model).unwrap()); + // Train: the item and its drop; Validation: three items, three drops, three permutes. + assert_eq!((reading[0].1.n, reading[0].1.unusable), (0, 2)); + assert_eq!((reading[1].1.n, reading[1].1.unusable), (0, 9)); + for (_, r) in &reading { + assert_eq!(r.replay, weksa::run::Agreement::default()); + assert_eq!(r.drop, weksa::run::VariantReading::default()); + assert_eq!(r.permute, weksa::run::VariantReading::default()); + } + assert_eq!( + reading[0].1.line(), + "ablation n=0 unusable=2 replay chrf=none meaning=none | drop n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none | permute n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none" + ); +}