From fd0c3b0edb4e4835f2386f2b3644f02efcbd6d51 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 04:14:39 +0000 Subject: [PATCH] Round: means count a refused item as 0, a failed round discards its packets, the round line is pinned Validation and Train means are over every item of the split with scored counts beside them, so refusing a weak item cannot raise a mean. A round that fails before its record is put (embedder down) removes its packets.cc, and a round starts from none, so a rerun with revised answers is not refused as a changed packet. RoundRecord::summary_line owns the printed line and is pinned with exact values. The RoundRecord compile_fail doctest gives every field and fails on E0451, with its twin literal in a lib unit test. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/main.rs | 33 +--- crates/weksa/src/run.rs | 154 +++++++++++++++++-- crates/weksa/tests/cli.rs | 2 +- crates/weksa/tests/golf_loop.rs | 143 ++++++++++++++--- schemas/cultnet/weksa.loop_round.schema.json | 18 ++- 5 files changed, 286 insertions(+), 64 deletions(-) diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 6ec2ce8..5e7eb28 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -11,7 +11,7 @@ use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn}; use weksa::entity::{Entity, EntityOrigin, LanguageForms, Pronunciation}; use weksa::inventory::{proof_revision, seed}; use weksa::packet::proof_packet; -use weksa::run::{Outcome, RoundOutcome, RoundRecord, RunPaths, RunStores, round}; +use weksa::run::{RoundOutcome, RunPaths, RunStores, round}; use weksa::score::ScoreInput; use weksa::{ AnswerFiles, CorpusStore, EntityId, EntityStore, InventoryStore, OllamaEmbedder, PacketStore, @@ -522,35 +522,6 @@ fn refused(e: StoreError) -> anyhow::Error { anyhow::anyhow!("{e}") } -/// `none` for an absent mean. -fn mean_text(value: Option, places: usize) -> String { - value.map_or("none".to_string(), |v| format!("{v:.places$}")) -} - -/// The one line a recorded round prints. -fn round_line(r: &RoundRecord) -> String { - let refused = r - .train() - .iter() - .filter(|row| matches!(row.outcome, Outcome::Refused(_))) - .count(); - let (t, v, x) = (r.train_mean(), r.validation_mean(), r.validation_refused()); - format!( - "round {} rev {} train n={} chrf={} meaning={} refused={refused} validation n={} chrf={} meaning={} refused parse={} admit={} lower={}", - r.n(), - r.inventory_rev(), - t.count, - mean_text(t.chrf, 2), - mean_text(t.meaning, 4), - v.count, - mean_text(v.chrf, 2), - mean_text(v.meaning, 4), - x.parse, - x.admit, - x.lower, - ) -} - fn main() -> Result<()> { let cli = parse_cli(); match cli.command { @@ -784,7 +755,7 @@ fn main() -> Result<()> { println!("unanswered {} of {asked}", prompts.len()); std::process::exit(3); } - RoundOutcome::Recorded(r) => println!("{}", round_line(&r)), + RoundOutcome::Recorded(r) => println!("{}", r.summary_line()), } } Command::Schema { out } => { diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index a73cadf..de6d846 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -113,9 +113,31 @@ pub struct RefusalCounts { /// fn f(r: &weksa::run::RoundRecord) -> u32 { r.n() } /// ``` /// -/// ```compile_fail -/// fn f() -> weksa::run::RoundRecord { -/// weksa::run::RoundRecord { n: 1 } +/// Every field is given, so the literal below fails only on privacy; its twin +/// is the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`. +/// +/// ```compile_fail,E0451 +/// use weksa::run::{RefusalCounts, RoundRecord}; +/// use weksa::score::SplitMean; +/// fn f() -> RoundRecord { +/// RoundRecord { +/// n: 1, +/// inventory_rev: 1, +/// prompt_digest: String::new(), +/// lowerer_digest: String::new(), +/// corpus_digest: String::new(), +/// entity_digest: String::new(), +/// model: String::new(), +/// embedder: String::new(), +/// chrf_version: String::new(), +/// calls: vec![], +/// train: vec![], +/// train_mean: SplitMean::default(), +/// train_scored: 0, +/// validation_mean: SplitMean::default(), +/// validation_scored: 0, +/// validation_refused: RefusalCounts::default(), +/// } /// } /// ``` #[derive(Clone, Debug, PartialEq, JsonSchema, DatabaseEntry)] @@ -149,6 +171,15 @@ pub struct RoundRecord { validation_mean: SplitMean, #[cultcache(key = 13)] validation_refused: RefusalCounts, + /// Train items that lowered and scored; `train_mean` is over all of them + /// and counts a refused item as 0. + #[cultcache(key = 14)] + train_scored: u32, + /// Validation items that lowered and scored; `validation_mean` is over all + /// of them and counts a refused item as 0, so refusing a weak item cannot + /// raise it. + #[cultcache(key = 15)] + validation_scored: u32, } impl RoundRecord { @@ -194,6 +225,46 @@ impl RoundRecord { pub fn validation_refused(&self) -> RefusalCounts { self.validation_refused } + pub fn train_scored(&self) -> u32 { + self.train_scored + } + pub fn validation_scored(&self) -> u32 { + self.validation_scored + } + + /// The one line a recorded round prints. `n` is every item of the split, + /// `scored` those that lowered; a mean counts a refused item as 0. + pub fn summary_line(&self) -> String { + let mean = |value: Option, places: usize| { + value.map_or("none".to_string(), |v| format!("{v:.places$}")) + }; + let refused = self + .train + .iter() + .filter(|row| matches!(row.outcome, Outcome::Refused(_))) + .count(); + let (t, v, x) = ( + &self.train_mean, + &self.validation_mean, + self.validation_refused, + ); + format!( + "round {} rev {} train n={} scored={} chrf={} meaning={} refused={refused} validation n={} scored={} chrf={} meaning={} refused parse={} admit={} lower={}", + self.n, + self.inventory_rev, + t.count, + self.train_scored, + mean(t.chrf, 2), + mean(t.meaning, 4), + v.count, + self.validation_scored, + mean(v.chrf, 2), + mean(v.meaning, 4), + x.parse, + x.admit, + x.lower, + ) + } } /// What a pinned store or id drifted from. @@ -456,7 +527,11 @@ enum Fate { /// One round: encodes every live Train and Validation item of the run's pinned /// corpus through `model`, admits each packet into the round's own /// `/round-/packets.cc`, lowers, scores, and records the round. The -/// round is all or nothing: while any prompt is unanswered, nothing is written. +/// round is all or nothing: while any prompt is unanswered, nothing is written, +/// and a round that fails before its record is put (the embedder down) leaves no +/// `packets.cc` behind, so a rerun with revised answers starts clean. A mean +/// counts an item that was refused or did not lower as 0, so refusing a weak +/// item cannot raise it; `scored` counts beside it say how many were real. pub fn round( run: &mut RunStores, model: &mut M, @@ -496,6 +571,8 @@ pub fn round( let dir = run.next_round_dir().map_err(store)?; std::fs::create_dir_all(&dir).map_err(|e| RunError::Io(e.kind()))?; + // The directory of an unrecorded round holds only a failed attempt's packets. + discard_packets(&dir); let mut packets = PacketStore::open(&dir.join("packets.cc")).map_err(store)?; let encoder = encoder_id(model.model(), &run.record.prompt_digest); let mut items: Vec = Vec::new(); @@ -540,21 +617,33 @@ pub fn round( let scores = if inputs.is_empty() { Vec::new() } else { - scorer.score(&inputs).map_err(|_| RunError::Embed)? + match scorer.score(&inputs) { + Ok(scores) => scores, + Err(_) => { + discard_packets(&dir); + return Err(RunError::Embed); + } + } }; let mut score_at: Vec> = vec![None; items.len()]; for ((i, _), score) in lowered.iter().zip(scores) { score_at[*i] = Some(score); } + // An item that did not lower scores 0 in every mean. + let nothing = ItemScore { + chrf: 0.0, + meaning: Some(0.0), + }; let scored: Vec<(&CorpusItem, ItemScore)> = items .iter() .zip(&score_at) - .filter_map(|(item, score)| score.clone().map(|s| (item, s))) + .map(|(item, score)| (item, score.clone().unwrap_or_else(|| nothing.clone()))) .collect(); let means = aggregate(&scored); let mut train_rows = Vec::new(); let mut refused = RefusalCounts::default(); + let (mut train_scored, mut validation_scored) = (0, 0); for ((item, fate), score) in items.iter().zip(&fates).zip(&score_at) { let (outcome, packet) = match (fate, score) { (Fate::Lowered(_, key), Some(score)) => ( @@ -568,11 +657,15 @@ pub fn round( (Fate::Lowered(..), None) => return Err(RunError::Store), }; match (SplitScheme::V1.live(item), fate) { - (Some(Split::Train), _) => train_rows.push(TrainRow { - item: item.id.clone(), - packet, - outcome, - }), + (Some(Split::Train), fate) => { + train_scored += u32::from(matches!(fate, Fate::Lowered(..))); + train_rows.push(TrainRow { + item: item.id.clone(), + packet, + outcome, + }); + } + (Some(Split::Validation), Fate::Lowered(..)) => validation_scored += 1, (Some(Split::Validation), Fate::Refused(stage, _)) => match stage { Stage::Parse => refused.parse += 1, Stage::Admit => refused.admit += 1, @@ -598,9 +691,48 @@ pub fn round( train_mean: means.train, validation_mean: means.validation, validation_refused: refused, + train_scored, + validation_scored, }; run.cache .put(format!("run:{}:round:{n}", run.record.label), &record) .map_err(store)?; Ok(RoundOutcome::Recorded(record)) } + +/// Removes a round directory's packet store and its lock. +fn discard_packets(dir: &Path) { + for name in ["packets.cc", "packets.cc.lock"] { + let _ = std::fs::remove_file(dir.join(name)); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The twin of the `RoundRecord` literal in its compile_fail doctest: the + /// same fields, compiling here because the fields are private to this module. + #[test] + fn a_round_record_literal_compiles_in_the_lib() { + let r = RoundRecord { + n: 1, + inventory_rev: 1, + prompt_digest: String::new(), + lowerer_digest: String::new(), + corpus_digest: String::new(), + entity_digest: String::new(), + model: String::new(), + embedder: String::new(), + chrf_version: String::new(), + calls: vec![], + train: vec![], + train_mean: SplitMean::default(), + train_scored: 0, + validation_mean: SplitMean::default(), + validation_scored: 0, + validation_refused: RefusalCounts::default(), + }; + assert_eq!(r.n(), 1); + } +} diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 0fae4df..8b0dd30 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -1392,7 +1392,7 @@ fn loop_open_and_unanswered_round() { assert!(second.status.success(), "{}", out(&second)); assert_eq!( out(&second), - "round 1 rev 3 train n=0 chrf=none meaning=none refused=1 validation n=0 chrf=none meaning=none refused parse=2 admit=0 lower=0\n" + "round 1 rev 3 train n=1 scored=0 chrf=0.00 meaning=0.0000 refused=1 validation n=2 scored=0 chrf=0.00 meaning=0.0000 refused parse=2 admit=0 lower=0\n" ); // The global store flags are not read by loop round: a drifted store is refused. diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 5cba1c3..5037622 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -732,41 +732,146 @@ fn refusal_stages_are_counted() { ]); assert_eq!(r.train()[0].outcome, Outcome::Refused(Stage::Parse)); assert_eq!(r.train()[0].packet, None); - assert_eq!((r.train_mean().count, r.train_mean().chrf), (0, None)); + // A refused item is in the mean as 0: n counts every item, scored the real ones. + let m = r.train_mean(); + assert_eq!((m.count, m.chrf, m.meaning), (1, Some(0.0), Some(0.0))); + assert_eq!((r.train_scored(), r.validation_scored()), (0, 0)); assert_eq!(r.validation_refused().parse, 0); assert_eq!( (r.validation_refused().admit, r.validation_refused().lower), (1, 1) ); - assert_eq!( - (r.validation_mean().count, r.validation_mean().chrf), - (0, None) - ); + let m = r.validation_mean(); + assert_eq!((m.count, m.chrf, m.meaning), (2, Some(0.0), Some(0.0))); - let r = run([ + // Validation items 2 and 3 score a and b when both lower (the Dao bed: + // 82.12 and 74.99). Refusing one halves the mean; it cannot raise it. + let full = run(good_answers().map(|(_, a)| a).try_into().unwrap()); + let b_only = run([ edited(1, unknown_concept), junk.clone(), good_answers()[2].1.clone(), ]); - assert_eq!(r.train()[0].outcome, Outcome::Refused(Stage::Admit)); - assert_eq!(r.train()[0].packet, None, "a refused packet is not stored"); - let v = r.validation_refused(); + assert_eq!(b_only.train()[0].outcome, Outcome::Refused(Stage::Admit)); + assert_eq!( + b_only.train()[0].packet, + None, + "a refused packet is not stored" + ); + let v = b_only.validation_refused(); assert_eq!((v.parse, v.admit, v.lower), (1, 0, 0)); assert_eq!( - r.validation_mean().count, - 1, - "only the scored item is counted" + b_only.validation_mean().count, + 2, + "every Validation item counts" ); - assert_eq!(two_dp(r.validation_mean().chrf.unwrap()), 74.99); - - let r = run([edited(1, with_relation), good_answers()[1].1.clone(), junk]); - assert_eq!(r.train()[0].outcome, Outcome::Refused(Stage::Lower)); + assert_eq!(b_only.validation_scored(), 1); + let b_half = b_only.validation_mean().chrf.unwrap(); assert!( - r.train()[0].packet.is_some(), + (b_half - 74.99 / 2.0).abs() < 0.01, + "(74.99 + 0) / 2 = {b_half}" + ); + + let a_only = run([edited(1, with_relation), good_answers()[1].1.clone(), junk]); + assert_eq!(a_only.train()[0].outcome, Outcome::Refused(Stage::Lower)); + assert!( + a_only.train()[0].packet.is_some(), "a packet that lowers badly was admitted" ); - assert_eq!(r.validation_refused().parse, 1); - assert_eq!(two_dp(r.validation_mean().chrf.unwrap()), 82.12); + assert_eq!(a_only.validation_refused().parse, 1); + let a_half = a_only.validation_mean().chrf.unwrap(); + assert!( + (a_half - 82.12 / 2.0).abs() < 0.01, + "(82.12 + 0) / 2 = {a_half}" + ); + // The halves add to the full mean: (a + 0) / 2 + (0 + b) / 2 = (a + b) / 2. + let whole = full.validation_mean().chrf.unwrap(); + assert!( + (a_half + b_half - whole).abs() < 1e-9, + "{a_half} + {b_half} vs {whole}" + ); + assert!( + b_half < whole && a_half < whole, + "refusing never raises the mean" + ); + assert_eq!(full.validation_scored(), 2); +} + +/// An embedder that is down. +struct Down; + +impl Embedder for Down { + fn model(&self) -> &str { + "bag" + } + fn embed(&self, _: &[&str]) -> anyhow::Result>> { + anyhow::bail!("down") + } +} + +#[test] +fn a_failed_round_leaves_nothing_a_rerun_collides_with() { + let mut dao = dao(); + let mut model = fake(&good_answers()); + assert_eq!( + round(&mut dao.run, &mut model, &Scorer::new(Down)).err(), + Some(RunError::Embed) + ); + assert!(dao.run.rounds().unwrap().is_empty()); + assert!( + !dao.root.join("round-1/packets.cc").exists(), + "the failed attempt's packets are discarded" + ); + + // The session revises item 1's answer and reruns: its packet is admitted + // as the revised one, not refused as a changed packet under the same key. + let mut revised = draft_of(1); + revised.confidence = weksa::packet::Grade::new(123).unwrap(); + let mut answers = good_answers(); + answers[0].1 = json(&revised); + let mut model = fake(&answers); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + assert!( + matches!(r.train()[0].outcome, Outcome::Scored { .. }), + "{:?}", + r.train()[0] + ); + assert_eq!(r.validation_refused(), Default::default()); + let stored = PacketStore::open(&dao.root.join("round-1/packets.cc")) + .unwrap() + .all() + .unwrap(); + assert_eq!(stored.len(), 3); + assert!(stored.iter().any(|p| p.provenance.confidence.get() == 123)); +} + +#[test] +fn the_round_line_prints_each_split_under_its_own_name() { + let mut dao = dao(); + let mut model = fake(&good_answers()); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + let line = r.summary_line(); + // Train is item 1 alone, Validation items 2 and 3; the numbers differ. + let (t, v) = (r.train_mean(), r.validation_mean()); + assert_eq!( + line, + format!( + "round 1 rev 3 train n=1 scored=1 chrf={:.2} meaning={:.4} refused=0 validation n=2 scored=2 chrf={:.2} meaning={:.4} refused parse=0 admit=0 lower=0", + t.chrf.unwrap(), + t.meaning.unwrap(), + v.chrf.unwrap(), + v.meaning.unwrap() + ) + ); + assert!( + line.contains("train n=1 scored=1 chrf=44.18 meaning=0."), + "{line}" + ); + assert!( + line.contains("validation n=2 scored=2 chrf=78.56 meaning=0."), + "{line}" + ); + assert_ne!(t.meaning, v.meaning); } #[test] diff --git a/schemas/cultnet/weksa.loop_round.schema.json b/schemas/cultnet/weksa.loop_round.schema.json index cb94831..7cde9c2 100644 --- a/schemas/cultnet/weksa.loop_round.schema.json +++ b/schemas/cultnet/weksa.loop_round.schema.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "RoundRecord", - "description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\n```compile_fail\nfn f() -> weksa::run::RoundRecord {\n weksa::run::RoundRecord { n: 1 }\n}\n```", + "description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin\nis the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::{RefusalCounts, RoundRecord};\nuse weksa::score::SplitMean;\nfn f() -> RoundRecord {\n RoundRecord {\n n: 1,\n inventory_rev: 1,\n prompt_digest: String::new(),\n lowerer_digest: String::new(),\n corpus_digest: String::new(),\n entity_digest: String::new(),\n model: String::new(),\n embedder: String::new(),\n chrf_version: String::new(),\n calls: vec![],\n train: vec![],\n train_mean: SplitMean::default(),\n train_scored: 0,\n validation_mean: SplitMean::default(),\n validation_scored: 0,\n validation_refused: RefusalCounts::default(),\n }\n}\n```", "type": "object", "properties": { "calls": { @@ -50,11 +50,23 @@ "train_mean": { "$ref": "#/$defs/SplitMean" }, + "train_scored": { + "description": "Train items that lowered and scored; `train_mean` is over all of them\nand counts a refused item as 0.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, "validation_mean": { "$ref": "#/$defs/SplitMean" }, "validation_refused": { "$ref": "#/$defs/RefusalCounts" + }, + "validation_scored": { + "description": "Validation items that lowered and scored; `validation_mean` is over all\nof them and counts a refused item as 0, so refusing a weak item cannot\nraise it.", + "type": "integer", + "format": "uint32", + "minimum": 0 } }, "required": [ @@ -71,7 +83,9 @@ "train", "train_mean", "validation_mean", - "validation_refused" + "validation_refused", + "train_scored", + "validation_scored" ], "$defs": { "Call": {