Round: means count a refused item as 0, a failed round discards its packets, the round line is pinned

Validation and Train means are over every item of the split with scored counts
beside them, so refusing a weak item cannot raise a mean. A round that fails
before its record is put (embedder down) removes its packets.cc, and a round
starts from none, so a rerun with revised answers is not refused as a changed
packet. RoundRecord::summary_line owns the printed line and is pinned with exact
values. The RoundRecord compile_fail doctest gives every field and fails on
E0451, with its twin literal in a lib unit test.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Weksa 2026-11-10 04:14:39 +00:00
commit fd0c3b0edb
5 changed files with 285 additions and 63 deletions

View file

@ -11,7 +11,7 @@ use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn};
use weksa::entity::{Entity, EntityOrigin, LanguageForms, Pronunciation};
use weksa::inventory::{proof_revision, seed};
use weksa::packet::proof_packet;
use weksa::run::{Outcome, RoundOutcome, RoundRecord, RunPaths, RunStores, round};
use weksa::run::{RoundOutcome, RunPaths, RunStores, round};
use weksa::score::ScoreInput;
use weksa::{
AnswerFiles, CorpusStore, EntityId, EntityStore, InventoryStore, OllamaEmbedder, PacketStore,
@ -522,35 +522,6 @@ fn refused(e: StoreError) -> anyhow::Error {
anyhow::anyhow!("{e}")
}
/// `none` for an absent mean.
fn mean_text(value: Option<f64>, places: usize) -> String {
value.map_or("none".to_string(), |v| format!("{v:.places$}"))
}
/// The one line a recorded round prints.
fn round_line(r: &RoundRecord) -> String {
let refused = r
.train()
.iter()
.filter(|row| matches!(row.outcome, Outcome::Refused(_)))
.count();
let (t, v, x) = (r.train_mean(), r.validation_mean(), r.validation_refused());
format!(
"round {} rev {} train n={} chrf={} meaning={} refused={refused} validation n={} chrf={} meaning={} refused parse={} admit={} lower={}",
r.n(),
r.inventory_rev(),
t.count,
mean_text(t.chrf, 2),
mean_text(t.meaning, 4),
v.count,
mean_text(v.chrf, 2),
mean_text(v.meaning, 4),
x.parse,
x.admit,
x.lower,
)
}
fn main() -> Result<()> {
let cli = parse_cli();
match cli.command {
@ -784,7 +755,7 @@ fn main() -> Result<()> {
println!("unanswered {} of {asked}", prompts.len());
std::process::exit(3);
}
RoundOutcome::Recorded(r) => println!("{}", round_line(&r)),
RoundOutcome::Recorded(r) => println!("{}", r.summary_line()),
}
}
Command::Schema { out } => {

View file

@ -113,9 +113,31 @@ pub struct RefusalCounts {
/// fn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }
/// ```
///
/// ```compile_fail
/// fn f() -> weksa::run::RoundRecord {
/// weksa::run::RoundRecord { n: 1 }
/// Every field is given, so the literal below fails only on privacy; its twin
/// is the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`.
///
/// ```compile_fail,E0451
/// use weksa::run::{RefusalCounts, RoundRecord};
/// use weksa::score::SplitMean;
/// fn f() -> RoundRecord {
/// RoundRecord {
/// n: 1,
/// inventory_rev: 1,
/// prompt_digest: String::new(),
/// lowerer_digest: String::new(),
/// corpus_digest: String::new(),
/// entity_digest: String::new(),
/// model: String::new(),
/// embedder: String::new(),
/// chrf_version: String::new(),
/// calls: vec![],
/// train: vec![],
/// train_mean: SplitMean::default(),
/// train_scored: 0,
/// validation_mean: SplitMean::default(),
/// validation_scored: 0,
/// validation_refused: RefusalCounts::default(),
/// }
/// }
/// ```
#[derive(Clone, Debug, PartialEq, JsonSchema, DatabaseEntry)]
@ -149,6 +171,15 @@ pub struct RoundRecord {
validation_mean: SplitMean,
#[cultcache(key = 13)]
validation_refused: RefusalCounts,
/// Train items that lowered and scored; `train_mean` is over all of them
/// and counts a refused item as 0.
#[cultcache(key = 14)]
train_scored: u32,
/// Validation items that lowered and scored; `validation_mean` is over all
/// of them and counts a refused item as 0, so refusing a weak item cannot
/// raise it.
#[cultcache(key = 15)]
validation_scored: u32,
}
impl RoundRecord {
@ -194,6 +225,46 @@ impl RoundRecord {
pub fn validation_refused(&self) -> RefusalCounts {
self.validation_refused
}
pub fn train_scored(&self) -> u32 {
self.train_scored
}
pub fn validation_scored(&self) -> u32 {
self.validation_scored
}
/// The one line a recorded round prints. `n` is every item of the split,
/// `scored` those that lowered; a mean counts a refused item as 0.
pub fn summary_line(&self) -> String {
let mean = |value: Option<f64>, places: usize| {
value.map_or("none".to_string(), |v| format!("{v:.places$}"))
};
let refused = self
.train
.iter()
.filter(|row| matches!(row.outcome, Outcome::Refused(_)))
.count();
let (t, v, x) = (
&self.train_mean,
&self.validation_mean,
self.validation_refused,
);
format!(
"round {} rev {} train n={} scored={} chrf={} meaning={} refused={refused} validation n={} scored={} chrf={} meaning={} refused parse={} admit={} lower={}",
self.n,
self.inventory_rev,
t.count,
self.train_scored,
mean(t.chrf, 2),
mean(t.meaning, 4),
v.count,
self.validation_scored,
mean(v.chrf, 2),
mean(v.meaning, 4),
x.parse,
x.admit,
x.lower,
)
}
}
/// What a pinned store or id drifted from.
@ -456,7 +527,11 @@ enum Fate {
/// One round: encodes every live Train and Validation item of the run's pinned
/// corpus through `model`, admits each packet into the round's own
/// `<root>/round-<n>/packets.cc`, lowers, scores, and records the round. The
/// round is all or nothing: while any prompt is unanswered, nothing is written.
/// round is all or nothing: while any prompt is unanswered, nothing is written,
/// and a round that fails before its record is put (the embedder down) leaves no
/// `packets.cc` behind, so a rerun with revised answers starts clean. A mean
/// counts an item that was refused or did not lower as 0, so refusing a weak
/// item cannot raise it; `scored` counts beside it say how many were real.
pub fn round<M: ModelPort, E: Embedder>(
run: &mut RunStores,
model: &mut M,
@ -496,6 +571,8 @@ pub fn round<M: ModelPort, E: Embedder>(
let dir = run.next_round_dir().map_err(store)?;
std::fs::create_dir_all(&dir).map_err(|e| RunError::Io(e.kind()))?;
// The directory of an unrecorded round holds only a failed attempt's packets.
discard_packets(&dir);
let mut packets = PacketStore::open(&dir.join("packets.cc")).map_err(store)?;
let encoder = encoder_id(model.model(), &run.record.prompt_digest);
let mut items: Vec<CorpusItem> = Vec::new();
@ -540,21 +617,33 @@ pub fn round<M: ModelPort, E: Embedder>(
let scores = if inputs.is_empty() {
Vec::new()
} else {
scorer.score(&inputs).map_err(|_| RunError::Embed)?
match scorer.score(&inputs) {
Ok(scores) => scores,
Err(_) => {
discard_packets(&dir);
return Err(RunError::Embed);
}
}
};
let mut score_at: Vec<Option<ItemScore>> = vec![None; items.len()];
for ((i, _), score) in lowered.iter().zip(scores) {
score_at[*i] = Some(score);
}
// An item that did not lower scores 0 in every mean.
let nothing = ItemScore {
chrf: 0.0,
meaning: Some(0.0),
};
let scored: Vec<(&CorpusItem, ItemScore)> = items
.iter()
.zip(&score_at)
.filter_map(|(item, score)| score.clone().map(|s| (item, s)))
.map(|(item, score)| (item, score.clone().unwrap_or_else(|| nothing.clone())))
.collect();
let means = aggregate(&scored);
let mut train_rows = Vec::new();
let mut refused = RefusalCounts::default();
let (mut train_scored, mut validation_scored) = (0, 0);
for ((item, fate), score) in items.iter().zip(&fates).zip(&score_at) {
let (outcome, packet) = match (fate, score) {
(Fate::Lowered(_, key), Some(score)) => (
@ -568,11 +657,15 @@ pub fn round<M: ModelPort, E: Embedder>(
(Fate::Lowered(..), None) => return Err(RunError::Store),
};
match (SplitScheme::V1.live(item), fate) {
(Some(Split::Train), _) => train_rows.push(TrainRow {
item: item.id.clone(),
packet,
outcome,
}),
(Some(Split::Train), fate) => {
train_scored += u32::from(matches!(fate, Fate::Lowered(..)));
train_rows.push(TrainRow {
item: item.id.clone(),
packet,
outcome,
});
}
(Some(Split::Validation), Fate::Lowered(..)) => validation_scored += 1,
(Some(Split::Validation), Fate::Refused(stage, _)) => match stage {
Stage::Parse => refused.parse += 1,
Stage::Admit => refused.admit += 1,
@ -598,9 +691,48 @@ pub fn round<M: ModelPort, E: Embedder>(
train_mean: means.train,
validation_mean: means.validation,
validation_refused: refused,
train_scored,
validation_scored,
};
run.cache
.put(format!("run:{}:round:{n}", run.record.label), &record)
.map_err(store)?;
Ok(RoundOutcome::Recorded(record))
}
/// Removes a round directory's packet store and its lock.
fn discard_packets(dir: &Path) {
for name in ["packets.cc", "packets.cc.lock"] {
let _ = std::fs::remove_file(dir.join(name));
}
}
#[cfg(test)]
mod tests {
use super::*;
/// The twin of the `RoundRecord` literal in its compile_fail doctest: the
/// same fields, compiling here because the fields are private to this module.
#[test]
fn a_round_record_literal_compiles_in_the_lib() {
let r = RoundRecord {
n: 1,
inventory_rev: 1,
prompt_digest: String::new(),
lowerer_digest: String::new(),
corpus_digest: String::new(),
entity_digest: String::new(),
model: String::new(),
embedder: String::new(),
chrf_version: String::new(),
calls: vec![],
train: vec![],
train_mean: SplitMean::default(),
train_scored: 0,
validation_mean: SplitMean::default(),
validation_scored: 0,
validation_refused: RefusalCounts::default(),
};
assert_eq!(r.n(), 1);
}
}

View file

@ -1392,7 +1392,7 @@ fn loop_open_and_unanswered_round() {
assert!(second.status.success(), "{}", out(&second));
assert_eq!(
out(&second),
"round 1 rev 3 train n=0 chrf=none meaning=none refused=1 validation n=0 chrf=none meaning=none refused parse=2 admit=0 lower=0\n"
"round 1 rev 3 train n=1 scored=0 chrf=0.00 meaning=0.0000 refused=1 validation n=2 scored=0 chrf=0.00 meaning=0.0000 refused parse=2 admit=0 lower=0\n"
);
// The global store flags are not read by loop round: a drifted store is refused.

View file

@ -732,41 +732,146 @@ fn refusal_stages_are_counted() {
]);
assert_eq!(r.train()[0].outcome, Outcome::Refused(Stage::Parse));
assert_eq!(r.train()[0].packet, None);
assert_eq!((r.train_mean().count, r.train_mean().chrf), (0, None));
// A refused item is in the mean as 0: n counts every item, scored the real ones.
let m = r.train_mean();
assert_eq!((m.count, m.chrf, m.meaning), (1, Some(0.0), Some(0.0)));
assert_eq!((r.train_scored(), r.validation_scored()), (0, 0));
assert_eq!(r.validation_refused().parse, 0);
assert_eq!(
(r.validation_refused().admit, r.validation_refused().lower),
(1, 1)
);
assert_eq!(
(r.validation_mean().count, r.validation_mean().chrf),
(0, None)
);
let m = r.validation_mean();
assert_eq!((m.count, m.chrf, m.meaning), (2, Some(0.0), Some(0.0)));
let r = run([
// Validation items 2 and 3 score a and b when both lower (the Dao bed:
// 82.12 and 74.99). Refusing one halves the mean; it cannot raise it.
let full = run(good_answers().map(|(_, a)| a).try_into().unwrap());
let b_only = run([
edited(1, unknown_concept),
junk.clone(),
good_answers()[2].1.clone(),
]);
assert_eq!(r.train()[0].outcome, Outcome::Refused(Stage::Admit));
assert_eq!(r.train()[0].packet, None, "a refused packet is not stored");
let v = r.validation_refused();
assert_eq!(b_only.train()[0].outcome, Outcome::Refused(Stage::Admit));
assert_eq!(
b_only.train()[0].packet,
None,
"a refused packet is not stored"
);
let v = b_only.validation_refused();
assert_eq!((v.parse, v.admit, v.lower), (1, 0, 0));
assert_eq!(
r.validation_mean().count,
1,
"only the scored item is counted"
b_only.validation_mean().count,
2,
"every Validation item counts"
);
assert_eq!(two_dp(r.validation_mean().chrf.unwrap()), 74.99);
let r = run([edited(1, with_relation), good_answers()[1].1.clone(), junk]);
assert_eq!(r.train()[0].outcome, Outcome::Refused(Stage::Lower));
assert_eq!(b_only.validation_scored(), 1);
let b_half = b_only.validation_mean().chrf.unwrap();
assert!(
r.train()[0].packet.is_some(),
(b_half - 74.99 / 2.0).abs() < 0.01,
"(74.99 + 0) / 2 = {b_half}"
);
let a_only = run([edited(1, with_relation), good_answers()[1].1.clone(), junk]);
assert_eq!(a_only.train()[0].outcome, Outcome::Refused(Stage::Lower));
assert!(
a_only.train()[0].packet.is_some(),
"a packet that lowers badly was admitted"
);
assert_eq!(r.validation_refused().parse, 1);
assert_eq!(two_dp(r.validation_mean().chrf.unwrap()), 82.12);
assert_eq!(a_only.validation_refused().parse, 1);
let a_half = a_only.validation_mean().chrf.unwrap();
assert!(
(a_half - 82.12 / 2.0).abs() < 0.01,
"(82.12 + 0) / 2 = {a_half}"
);
// The halves add to the full mean: (a + 0) / 2 + (0 + b) / 2 = (a + b) / 2.
let whole = full.validation_mean().chrf.unwrap();
assert!(
(a_half + b_half - whole).abs() < 1e-9,
"{a_half} + {b_half} vs {whole}"
);
assert!(
b_half < whole && a_half < whole,
"refusing never raises the mean"
);
assert_eq!(full.validation_scored(), 2);
}
/// An embedder that is down.
struct Down;
impl Embedder for Down {
fn model(&self) -> &str {
"bag"
}
fn embed(&self, _: &[&str]) -> anyhow::Result<Vec<Vec<f32>>> {
anyhow::bail!("down")
}
}
#[test]
fn a_failed_round_leaves_nothing_a_rerun_collides_with() {
let mut dao = dao();
let mut model = fake(&good_answers());
assert_eq!(
round(&mut dao.run, &mut model, &Scorer::new(Down)).err(),
Some(RunError::Embed)
);
assert!(dao.run.rounds().unwrap().is_empty());
assert!(
!dao.root.join("round-1/packets.cc").exists(),
"the failed attempt's packets are discarded"
);
// The session revises item 1's answer and reruns: its packet is admitted
// as the revised one, not refused as a changed packet under the same key.
let mut revised = draft_of(1);
revised.confidence = weksa::packet::Grade::new(123).unwrap();
let mut answers = good_answers();
answers[0].1 = json(&revised);
let mut model = fake(&answers);
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
assert!(
matches!(r.train()[0].outcome, Outcome::Scored { .. }),
"{:?}",
r.train()[0]
);
assert_eq!(r.validation_refused(), Default::default());
let stored = PacketStore::open(&dao.root.join("round-1/packets.cc"))
.unwrap()
.all()
.unwrap();
assert_eq!(stored.len(), 3);
assert!(stored.iter().any(|p| p.provenance.confidence.get() == 123));
}
#[test]
fn the_round_line_prints_each_split_under_its_own_name() {
let mut dao = dao();
let mut model = fake(&good_answers());
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
let line = r.summary_line();
// Train is item 1 alone, Validation items 2 and 3; the numbers differ.
let (t, v) = (r.train_mean(), r.validation_mean());
assert_eq!(
line,
format!(
"round 1 rev 3 train n=1 scored=1 chrf={:.2} meaning={:.4} refused=0 validation n=2 scored=2 chrf={:.2} meaning={:.4} refused parse=0 admit=0 lower=0",
t.chrf.unwrap(),
t.meaning.unwrap(),
v.chrf.unwrap(),
v.meaning.unwrap()
)
);
assert!(
line.contains("train n=1 scored=1 chrf=44.18 meaning=0."),
"{line}"
);
assert!(
line.contains("validation n=2 scored=2 chrf=78.56 meaning=0."),
"{line}"
);
assert_ne!(t.meaning, v.meaning);
}
#[test]

View file

@ -1,7 +1,7 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "RoundRecord",
"description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\n```compile_fail\nfn f() -> weksa::run::RoundRecord {\n weksa::run::RoundRecord { n: 1 }\n}\n```",
"description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin\nis the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::{RefusalCounts, RoundRecord};\nuse weksa::score::SplitMean;\nfn f() -> RoundRecord {\n RoundRecord {\n n: 1,\n inventory_rev: 1,\n prompt_digest: String::new(),\n lowerer_digest: String::new(),\n corpus_digest: String::new(),\n entity_digest: String::new(),\n model: String::new(),\n embedder: String::new(),\n chrf_version: String::new(),\n calls: vec![],\n train: vec![],\n train_mean: SplitMean::default(),\n train_scored: 0,\n validation_mean: SplitMean::default(),\n validation_scored: 0,\n validation_refused: RefusalCounts::default(),\n }\n}\n```",
"type": "object",
"properties": {
"calls": {
@ -50,11 +50,23 @@
"train_mean": {
"$ref": "#/$defs/SplitMean"
},
"train_scored": {
"description": "Train items that lowered and scored; `train_mean` is over all of them\nand counts a refused item as 0.",
"type": "integer",
"format": "uint32",
"minimum": 0
},
"validation_mean": {
"$ref": "#/$defs/SplitMean"
},
"validation_refused": {
"$ref": "#/$defs/RefusalCounts"
},
"validation_scored": {
"description": "Validation items that lowered and scored; `validation_mean` is over all\nof them and counts a refused item as 0, so refusing a weak item cannot\nraise it.",
"type": "integer",
"format": "uint32",
"minimum": 0
}
},
"required": [
@ -71,7 +83,9 @@
"train",
"train_mean",
"validation_mean",
"validation_refused"
"validation_refused",
"train_scored",
"validation_scored"
],
"$defs": {
"Call": {