diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index a9062a1..62a8815 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -46,7 +46,7 @@ use crate::encode::{ use crate::form::{is_attestation, is_language_tag, is_slug}; use crate::model::{AnswerFiles, ModelPort, PortError}; use crate::packet::{Packet, Stance}; -use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate}; +use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate, validation_items}; use crate::split::{Split, SplitScheme, fnv1a64, fnv1a64_of, hex16}; use crate::store::{ CorpusStore, EntityStore, InventoryStore, OntologyStore, PacketStore, SpeakerStateStore, @@ -246,9 +246,18 @@ pub struct LanguageRound { pub validation_refused: RefusalCounts, } +/// One panel language's Validation scores in a round, in corpus order: numbers +/// only, no item key and no text. Only `run.rs` reads them (the paired floor). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct LanguageScores { + pub language: String, + pub scores: Vec, +} + /// One scored round. It holds no text and no Test reading: per-item rows for -/// Train only, Validation only as means and refusal counts by stage. Only -/// `run::round` builds one. +/// Train only; Validation as means and refusal counts by stage, and as +/// per-item scores (numbers only, private to this module). Only `run::round` +/// builds one. /// /// ``` /// fn f(r: &weksa::run::RoundRecord) -> u32 { r.n() } @@ -277,6 +286,7 @@ pub struct LanguageRound { /// parse_reasons: vec![], /// own_family: 0, /// languages: vec![], +/// validation_scores: vec![], /// } /// } /// ``` @@ -336,6 +346,12 @@ pub struct RoundRecord { // Key 20 is `own_family`; the per-language readings take 21. #[cultcache(key = 21)] languages: Vec, + /// Each panel language's live Validation scores in corpus order, a refused + /// item as 0. Private with no getter: only `run.rs` (the floor) reads them, + /// so a leak outside it is a compile error. A run from before has none. + #[cultcache(key = 22, default)] + #[schemars(default)] + validation_scores: Vec, } impl RoundRecord { @@ -2069,6 +2085,7 @@ fn score_round( let mut train_rows = Vec::new(); let mut languages: Vec = Vec::new(); + let mut validation_scores: Vec = Vec::new(); for tag in &run.record.panel { let of: Vec<(&CorpusItem, ItemScore)> = scored .iter() @@ -2076,6 +2093,10 @@ fn score_round( .cloned() .collect(); let means = aggregate(&of); + validation_scores.push(LanguageScores { + language: tag.clone(), + scores: validation_items(&of), + }); languages.push(LanguageRound { language: tag.clone(), train_mean: means.train, @@ -2153,6 +2174,7 @@ fn score_round( .collect(), own_family, languages, + validation_scores, }; Ok(RoundOutcome::Recorded(record)) } @@ -2771,6 +2793,7 @@ mod tests { parse_reasons: vec![], own_family: 0, languages: vec![], + validation_scores: vec![], }; assert_eq!(r.n(), 1); } @@ -3286,6 +3309,7 @@ mod tests { validation_scored: 2, validation_refused: RefusalCounts::default(), }], + validation_scores: vec![], } } diff --git a/crates/weksa/src/score/mod.rs b/crates/weksa/src/score/mod.rs index c1033a4..e411d7c 100644 --- a/crates/weksa/src/score/mod.rs +++ b/crates/weksa/src/score/mod.rs @@ -28,7 +28,7 @@ pub struct ScoreInput<'a> { pub output: &'a str, } -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct ItemScore { /// chrF of the output against the reference. pub chrf: f64, @@ -131,13 +131,18 @@ pub struct Aggregate { pub validation: SplitMean, } -/// The mean of the scores of the items read in split `want`. -fn split_mean(scored: &[(&CorpusItem, ItemScore)], want: Split) -> SplitMean { - let of: Vec<&ItemScore> = scored +/// The scores of the items read in split `want`, in list order. +fn scores_in<'a>(scored: &'a [(&CorpusItem, ItemScore)], want: Split) -> Vec<&'a ItemScore> { + scored .iter() .filter(|(item, _)| SplitScheme::V1.live(item) == Some(want)) .map(|(_, score)| score) - .collect(); + .collect() +} + +/// The mean of the scores of the items read in split `want`. +fn split_mean(scored: &[(&CorpusItem, ItemScore)], want: Split) -> SplitMean { + let of = scores_in(scored, want); let average = |values: Vec| { (!values.is_empty()).then(|| values.iter().sum::() / values.len() as f64) }; @@ -157,6 +162,12 @@ pub fn aggregate(scored: &[(&CorpusItem, ItemScore)]) -> Aggregate { } } +/// The Validation items' scores in list order, numbers only: no key, no text. +/// A refused item arrives scored 0 and keeps its position. +pub fn validation_items(scored: &[(&CorpusItem, ItemScore)]) -> Vec { + scores_in(scored, Split::Validation).into_iter().cloned().collect() +} + /// The Test split's readings, scalars only: no item key, no minted string, no /// pair. The checkpoint decision (convergence-test) needs numbers; a key beside /// a minted string is the handle an edit step would act on. The only producer @@ -249,6 +260,31 @@ mod tests { item(key, text, &group_in(split), model) } + #[test] + fn validation_items_keeps_validation_only_in_order() { + let train = in_split("item:t:1", UNRELATED, Split::Train, false); + let first = in_split("item:v:1", "first words", Split::Validation, false); + let refused = in_split("item:v:2", "second words", Split::Validation, false); + let last = in_split("item:v:3", "third words", Split::Validation, false); + let mut gone = in_split("item:v:4", "withdrawn words", Split::Validation, false); + gone.withdrawn = true; + let test = in_split("item:c:1", HELD_OUT, Split::Test, false); + let scored = [ + (&train, score(10.0, Some(0.1))), + (&first, score(40.0, Some(0.4))), + (&test, score(60.0, Some(0.6))), + (&refused, score(0.0, Some(0.0))), + (&gone, score(90.0, Some(0.9))), + (&last, score(80.0, None)), + ]; + let got = validation_items(&scored); + assert_eq!( + got, + [score(40.0, Some(0.4)), score(0.0, Some(0.0)), score(80.0, None)] + ); + assert_eq!(got.len(), aggregate(&scored).validation.count); + } + #[test] fn test_readings_at_checkpoint() { let train_item = in_split("item:t:1", UNRELATED, Split::Train, false); diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 2d0320a..635065e 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -2645,6 +2645,9 @@ fn source_tripwires() { "Aggregate", "validation_mean", "validation_refused", + "ItemScore", + "LanguageScores", + "validation_scores", ] { assert!(!context.contains(banned), "context.rs names {banned}"); } @@ -3364,6 +3367,234 @@ fn a_run_from_before_does_not_open() { assert!(reopened.rounds().is_err()); } +/// A round as the file holds it, with key 22 the lib keeps private. +#[derive(Clone, Debug, schemars::JsonSchema, cultcache_rs::DatabaseEntry)] +#[cultcache(type = "weksa.loop_round")] +struct ScoredRound { + #[cultcache(key = 0)] + n: u32, + #[cultcache(key = 1)] + inventory_rev: u32, + #[cultcache(key = 2)] + prompt_digest: String, + #[cultcache(key = 4)] + corpus_digest: String, + #[cultcache(key = 5)] + entity_digest: String, + #[cultcache(key = 6)] + model: String, + #[cultcache(key = 7)] + embedder: String, + #[cultcache(key = 8)] + chrf_version: String, + #[cultcache(key = 9)] + calls: Vec, + #[cultcache(key = 10)] + train: Vec, + #[cultcache(key = 16)] + inventory_digest: String, + #[cultcache(key = 17)] + cycle: Option, + #[cultcache(key = 18)] + admit_rules: Vec, + #[cultcache(key = 19)] + parse_reasons: Vec, + #[cultcache(key = 20)] + own_family: u32, + #[cultcache(key = 21)] + languages: Vec, + #[cultcache(key = 22)] + validation_scores: Vec, +} + +/// A round as written before key 22 existed. +#[derive(Clone, Debug, schemars::JsonSchema, cultcache_rs::DatabaseEntry)] +#[cultcache(type = "weksa.loop_round")] +struct PreScoresRound { + #[cultcache(key = 0)] + n: u32, + #[cultcache(key = 1)] + inventory_rev: u32, + #[cultcache(key = 2)] + prompt_digest: String, + #[cultcache(key = 4)] + corpus_digest: String, + #[cultcache(key = 5)] + entity_digest: String, + #[cultcache(key = 6)] + model: String, + #[cultcache(key = 7)] + embedder: String, + #[cultcache(key = 8)] + chrf_version: String, + #[cultcache(key = 9)] + calls: Vec, + #[cultcache(key = 10)] + train: Vec, + #[cultcache(key = 16)] + inventory_digest: String, + #[cultcache(key = 17)] + cycle: Option, + #[cultcache(key = 18)] + admit_rules: Vec, + #[cultcache(key = 19)] + parse_reasons: Vec, + #[cultcache(key = 20)] + own_family: u32, + #[cultcache(key = 21)] + languages: Vec, +} + +/// The scores a round put in `run.cc`, read through the mirror (the lib's field +/// is private). The run must be dropped first. +fn stored_scores(root: &Path) -> Vec { + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache + .add_generic_backing_store(SingleFileMessagePackBackingStore::new(root.join("run.cc"))) + .unwrap(); + cache.pull_all_backing_stores().unwrap(); + cache.get_all::().unwrap().remove(0).validation_scores +} + +/// The stored scores agree with the round's readings: one entry per language +/// in the record's order, as many scores as Validation items (scored plus +/// refused), the same means, and at least the refused items as exact zeros. +fn assert_scores_match_readings(r: &RoundRecord, stored: &[weksa::run::LanguageScores]) { + let tags: Vec<&str> = r.languages().iter().map(|l| l.language.as_str()).collect(); + let got: Vec<&str> = stored.iter().map(|l| l.language.as_str()).collect(); + assert_eq!(got, tags, "one entry per panel language, in panel order"); + for (reading, entry) in r.languages().iter().zip(stored) { + let m = &reading.validation_mean; + let v = reading.validation_refused; + let refused = (v.parse + v.admit + v.lower) as usize; + assert_eq!(entry.scores.len(), m.count, "{}", reading.language); + assert_eq!(entry.scores.len(), reading.validation_scored as usize + refused); + let n = entry.scores.len() as f64; + let chrf = entry.scores.iter().map(|s| s.chrf).sum::() / n; + assert!((chrf - m.chrf.unwrap()).abs() < 1e-9, "{}", reading.language); + let zeros = entry + .scores + .iter() + .filter(|s| s.chrf == 0.0 && s.meaning == Some(0.0)) + .count(); + assert!(zeros >= refused, "{}: refused items are stored as zeros", reading.language); + } +} + +#[test] +fn a_round_stores_one_score_per_validation_item() { + let mut dao = dao(); + let mut model = fake(&good_answers()); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + let root = dao.root.clone(); + drop(dao.run); + let stored = stored_scores(&root); + assert_eq!(stored.len(), 1); + assert_eq!(stored[0].language, "en-US"); + assert_scores_match_readings(&r, &stored); +} + +#[test] +fn a_two_language_round_stores_each_languages_scores_in_panel_order() { + let mut dao = bilingual(LAOZI_JA); + let mut model = bilingual_fake(); + // The ja-JP Validation item 13 is not an encodable draft: it scores 0. + model.answers.insert("item:daodejing:13".into(), "not a draft".into()); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + let root = dao.root.clone(); + drop(dao.run); + let stored = stored_scores(&root); + assert_scores_match_readings(&r, &stored); + let tags: Vec<&str> = stored.iter().map(|l| l.language.as_str()).collect(); + assert_eq!(tags, ["en-US", "ja-JP"]); + // ja-JP: item 12 reads perfectly, then item 13 (the refused one), in + // stored item order; none of en-US's items is among them. + let ja: Vec<(f64, Option)> = stored[1].scores.iter().map(|s| (s.chrf, s.meaning)).collect(); + assert_eq!(ja, [(100.0, Some(1.0)), (0.0, Some(0.0))]); + assert_eq!(stored[0].scores.len(), 2); + assert!(stored[0].scores.iter().all(|s| s.chrf < 100.0), "en-US holds its own items"); +} + +#[test] +fn refused_validation_items_are_stored_as_zeros_at_every_stage() { + // Lower: the ja-JP items cite an entity with no ja-JP form. + let mut dao = bilingual(None); + let mut model = bilingual_fake(); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + let root = dao.root.clone(); + drop(dao.run); + let stored = stored_scores(&root); + assert_eq!(r.language("ja-JP").unwrap().validation_refused.lower, 2); + assert_scores_match_readings(&r, &stored); + assert!(stored[1].scores.iter().all(|s| s.chrf == 0.0 && s.meaning == Some(0.0))); + + // Parse (item 3 is Validation) and Admit (item 2), one run each. + for (parse, admit) in [(true, false), (false, true)] { + let mut dao = self::dao(); + let mut answers = good_answers(); + if parse { + answers[2].1 = "not json at all".into(); + } + if admit { + answers[1].1 = edited(2, unknown_concept); + } + let mut model = fake(&answers); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + let root = dao.root.clone(); + drop(dao.run); + let stored = stored_scores(&root); + let v = r.language("en-US").unwrap().validation_refused; + assert_eq!((v.parse, v.admit), (u32::from(parse), u32::from(admit))); + assert_scores_match_readings(&r, &stored); + } +} + +#[test] +fn an_old_run_without_scores_still_opens() { + let mut dao = dao(); + let mut model = fake(&good_answers()); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + let rec = dao.run.record().clone(); + let root = dao.root.clone(); + drop(dao.run); + let old = PreScoresRound { + n: r.n(), + inventory_rev: r.inventory_rev(), + prompt_digest: r.prompt_digest().into(), + corpus_digest: r.corpus_digest().into(), + entity_digest: r.entity_digest().into(), + model: r.model().into(), + embedder: r.embedder().into(), + chrf_version: r.chrf_version().into(), + calls: r.calls().to_vec(), + train: r.train().to_vec(), + inventory_digest: r.inventory_digest().into(), + cycle: r.cycle(), + admit_rules: vec![], + parse_reasons: vec![], + own_family: 0, + languages: vec![r.language("en-US").unwrap().clone()], + }; + let _ = std::fs::remove_file(root.join("run.cc")); + { + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache + .add_generic_backing_store(SingleFileMessagePackBackingStore::new(root.join("run.cc"))) + .unwrap(); + cache.pull_all_backing_stores().unwrap(); + cache.put("run:dao".to_string(), &rec).unwrap(); + cache.put("run:dao:round:1".to_string(), &old).unwrap(); + } + let reopened = RunStores::open(&root).unwrap(); + let rounds = reopened.rounds().unwrap(); + assert_eq!(rounds.len(), 1); + assert_eq!(rounds[0].language("en-US"), r.language("en-US")); +} + #[test] fn the_published_run_and_round_schemas_require_none_of_the_new_fields() { let published = weksa::schema::published(); diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index fbf2a50..9766d35 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -40,7 +40,7 @@ means no store holds it and nothing writes it. | Scene (cut `speaker-scene`) | Key `scene:@`, the item's split group; type `weksa.scene`: participants (speaker slugs) and, per ordered pair, relation features of slot `Scene` (relative status, intimacy, in-group, audience and whatever the loop adds), at an inventory revision (provenance; a run reads the features at its own head), and an `Annotator` | As Speaker. Keyed by the group, so a scene never straddles the split. `data/original/speaker-state.cc` holds Model-annotated keeper scenes, one per Train or Validation group of both languages (`weksa scene put|dump`); a Test group has none and reads `NoScene` | As Speaker; a model-annotated scene is never Test evidence | | Corpus item | Key `item::`, `n` canonical decimal >= 1 (no sign, no leading zero, from cut 2b); type `weksa.corpus_item` with text, language, speaker, split group, prior turns, source. Source `ModelWritten` carries a `ModelFamily` enum, the generator string as the fixture wrote it, and a prompt digest (cut 2b). The item carries no speaker state of its own: its speaker slug and group name the Speaker and Scene documents | Admitted once; the whole item is immutable (re-adding an identical item is a no-op, any difference is refused; a correction is a new item that `supersedes` the old); text already held under another key is refused unless the new item supersedes that key (cut 2b); withdrawn, never deleted, so old scores stay explainable | Importers and `weksa corpus add`, under `corpus-cast-plus-public-domain`; the source enum has no variant for Discord or any training-forbidden source (`corpus-clean`) | | Split assignment | Derived: `split(scheme, item.group)` | Fixed for a scheme; a new scheme is a new constant, never a reassignment. A model-written item whose group derives test is in no split; a superseding item keeps its predecessor's group | The hash function. Nobody assigns by hand, so there is no writer to forbid | -| Loop round | Key `run: